rmbackup is a shell script that uses rsync to do incremental centralized backups of remote servers. It can be configured by config files.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

380 lines
13 KiB

  1. #!/usr/bin/env sh
  2. ## rmbackup.sh
  3. # Create incremental backups of remote servers with ssh, rsync and config files
  4. # Copyright 2013 Christian Baer
  5. # http://github.com/chrisb86/
  6. # Permission is hereby granted, free of charge, to any person obtaining
  7. # a copy of this software and associated documentation files (the
  8. # "Software"), to deal in the Software without restriction, including
  9. # without limitation the rights to use, copy, modify, merge, publish,
  10. # distribute, sublicense, and/or sell copies of the Software, and to
  11. # permit persons to whom the Software is furnished to do so, subject to
  12. # the following conditions:
  13. # The above copyright notice and this permission notice shall be
  14. # included in all copies or substantial portions of the Software.
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16. # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  17. # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18. # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  19. # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  20. # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  21. # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  22. rmbackup=`basename -- $0`
  23. rmbackup_pid=$$
  24. rmbackup_conf_file="/usr/local/etc/rmbackup.conf"
  25. VERBOSE="${VERBOSE:-false}"
  26. DEBUG="${DEBUG:-false}"
  27. log_date_format="%Y-%m-%d %H:%M:%S" # For logging purposses when config file isn't there
  28. rmbackup_usage_backup="Usage: $rmbackup backup [-f CONFIGFILE] [-vd]"
  29. rmbackup_usage_cleanup="Usage: $rmbackup cleanup [-f CONFIGFILE] [-vd]"
  30. # Show help screen
  31. # Usage: help exitcode
  32. help () {
  33. echo "Usage: $rmbackup command {params}"
  34. echo
  35. echo "backup Backup all defined hosts."
  36. echo " [-f CONFIGFILE] Backup only specified host."
  37. echo " [-v] Turn on verbose mode."
  38. echo " [-d] Turn on debug mode (includes -v)."
  39. echo "cleanup Cleanup old backups."
  40. echo " [-f CONFIGFILE] Cleanup only specified hosts backups."
  41. echo " [-v] Turn on verbose mode."
  42. echo " [-d] Turn on debug mode (includes -v)."
  43. echo "help Show this screen"
  44. exit $1
  45. }
  46. # Print and log messages when verbose mode is on
  47. # Usage: chat [0|1|2|3] MESSAGE
  48. ## 0 = regular output
  49. ## 1 = error messages
  50. ## 2 = verbose messages
  51. ## 3 = debug messages
  52. chat () {
  53. messagetype=$1
  54. message=$2
  55. log=$log_dir/$log_file
  56. log_date=$(date "+$log_date_format")
  57. if [ $messagetype = 0 ]; then
  58. echo "[$log_date] [INFO] $message" | tee -a $log ;
  59. fi
  60. #
  61. if [ $messagetype = 1 ]; then
  62. echo "[$log_date] [ERROR] $message" | tee -a $log ; exit 1;
  63. fi
  64. if [ $messagetype = 2 ] && [ "$VERBOSE" = true ]; then
  65. echo "[$log_date] [INFO] $message" | tee -a $log
  66. fi
  67. if [ $messagetype = 3 ] && [ "$DEBUG" = true ]; then
  68. echo "[$log_date] [DEBUG] $message" | tee -a $log
  69. fi
  70. }
  71. # Load config file and set default variables
  72. # Usage: init [CONFIGFILE]
  73. init () {
  74. [ ! -f "$rmbackup_conf_file" ] && chat 1 "Config file $rmbackup_conf_file not found. Exiting."
  75. . $rmbackup_conf_file
  76. rmbackup_conf_location="${CONF_LOCATION:-/usr/local/etc/rmbackup.d}"
  77. rmbackup_pid_file="${PID_FILE:-/var/run/rmbackup.pid}"
  78. backup_timestamp_format="${BACKUP_TIMESTAMP_FORMAT:-%Y-%m-%d}"
  79. log_dir="${LOG_DIR:-/var/log}"
  80. log_file="${LOG_FILE:-rmbackup.log}"
  81. log_date_format="${LOG_DATE_FORMAT:-%Y-%m-%d %H:%M:%S}"
  82. backups_dir="$BACKUPS_DIR"
  83. backups_path_last="${LAST:-last}"
  84. backups_path_inprog="${INPROG:-inProgress}"
  85. rsync="${RSYNC_PATH:-/usr/local/bin/rsync}"
  86. rsync_conf_default="${RSYNC_CONF_DEFAULT:---del --quiet}"
  87. ssh="${SSH_PATH:-/usr/bin/ssh}"
  88. remote_privileges_path_default="${REMOTE_PRIVILEGES_PATH:-/usr/local/bin/doas}"
  89. remote_rsync_path_default="${REMOTE_RSYNC_PATH:-/usr/local/bin/rsync}"
  90. global_keep="${GLOBAL_KEEP_DAYS:-14}"
  91. chat 2 "Starting $rmbackup with PID $rmbackup_pid."
  92. chat 3 "rmbackup: $rmbackup"
  93. chat 3 "rmbackup_pid: $rmbackup_pid"
  94. chat 3 "rmbackup_conf_file: $rmbackup_conf_file"
  95. chat 3 "rmbackup_conf_location: $rmbackup_conf_location"
  96. chat 3 "rmbackup_pid_file: $rmbackup_pid_file"
  97. chat 3 "backup_timestamp_format: $backup_timestamp_format"
  98. chat 3 "log_dir: $log_dir"
  99. chat 3 "log_file: $log_file"
  100. chat 3 "log_date_format: $log_date_format"
  101. chat 3 "backups_dir: $backups_dir"
  102. chat 3 "backups_path_last: $backups_path_last"
  103. chat 3 "backups_path_inprog: $backups_path_inprog"
  104. chat 3 "rsync: $rsync"
  105. chat 3 "rsync_conf_default: $rsync_conf_default"
  106. chat 3 "ssh: $ssh"
  107. chat 3 "remote_privileges_path_default: $remote_privileges_path_default"
  108. chat 3 "remote_rsync_path_default: $remote_rsync_path_default"
  109. chat 3 "global_keep :$global_keep"
  110. chat 3 "VERBOSE: $VERBOSE"
  111. chat 3 "DEBUG: $DEBUG"
  112. if [ -z "$backups_dir" ]; then chat 1 "Target directory for backups not set. Please set it in $rmbackup_conf_file!"; fi
  113. }
  114. # Check if script is already running
  115. # Usage: checkpid
  116. checkPID () {
  117. touch $rmbackup_pid_file
  118. # Get stored PID from file
  119. rmbackup_stored_pid=`cat $rmbackup_pid_file`
  120. # Check if stored PID is in use
  121. rmbackup_pid_is_running=`ps aux | awk '{print $2}' | grep $rmbackup_stored_pid`
  122. chat 3 "rmbackup_pid: $rmbackup_pid"
  123. chat 3 "rmbackup_stored_pid: $rmbackup_stored_pid"
  124. chat 3 "rmbackup_pid_is_running: $rmbackup_pid_is_running"
  125. if [ "$rmbackup_pid_is_running" ]; then
  126. # If stored PID is already in use, skip execution
  127. chat 1 "Skipping because $rmbackup is running (PID: $rmbackup_stored_pid)."
  128. else
  129. # Update PID file
  130. echo $rmbackup_pid > $rmbackup_pid_file
  131. chat 0 "Starting work."
  132. fi
  133. }
  134. # Link the specified directory to $backups_path_last
  135. # Usage: backup_host DIRECTORY
  136. link_last () {
  137. link_source_dir=$1
  138. chat 2 "Symlinking $link_source_dir to $backup_target/$backups_path_last."
  139. chat 3 "ln -nsf $link_source_dir $backup_target/$backups_path_last"
  140. ln -nsf "$link_source_dir" "$backup_target/$backups_path_last"
  141. unset link_source_dir
  142. }
  143. # Unsets the host specific variables
  144. # Usage: unset_config
  145. unset_config() {
  146. chat 2 "Cleaning up variables."
  147. chat 3 "unset ssh_port privileges remote_rsync_path ssh_server ssh_alias ssh_user ssh_args rsync_conf backup_target backup_last backup_timestamp remote_sources latest_backup keep SSH_PORT PRIVILEGES_PATH RSYNC_PATH SSH_SERVER SSH_ALIAS SSH_USER SSH_ARGS RSYNC_CONF REMOTE_SOURCES ERROR KEEP"
  148. unset ssh_port privileges remote_rsync_path ssh_server ssh_alias ssh_user ssh_args rsync_conf backup_target backup_last backup_timestamp remote_sources latest_backup keep SSH_PORT PRIVILEGES_PATH RSYNC_PATH SSH_SERVER SSH_ALIAS SSH_USER SSH_ARGS RSYNC_CONF REMOTE_SOURCES ERROR KEEP
  149. }
  150. # Loads a config file and backups the host
  151. # Usage: backup_host CONFIGFILE
  152. backup_host () {
  153. config_file=$1
  154. # Check if config file exists and exit if not.
  155. if [ ! -f "$1" ] ; then chat 1 "Config file $config_file not found."; fi
  156. chat 2 "Loading Config file $config_file"
  157. chat 3 "config_file: $config_file"
  158. # Load config file
  159. . $config_file
  160. ssh_port="${SSH_PORT:-22}"
  161. privileges="${PRIVILEGES_PATH:-$remote_privileges_path_default}"
  162. remote_rsync_path="${RSYNC_PATH:-$remote_rsync_path_default}"
  163. ssh_server="$SSH_SERVER"
  164. ssh_alias="${SSH_ALIAS:-$SSH_SERVER}"
  165. ssh_user="$SSH_USER"
  166. ssh_args="$SSH_ARGS"
  167. rsync_conf="${rsync_conf_default} ${RSYNC_CONF}"
  168. backup_target="$backups_dir/$ssh_alias"
  169. backup_last="$backup_target/$backups_path_last"
  170. backup_timestamp=$(date "+$backup_timestamp_format")
  171. remote_sources=${REMOTE_SOURCES}
  172. latest_backup=`find $backup_target -type d \( ! -iname "$backups_path_inprog" ! -iname "$backups_path_last" ! -iname ".*" \) -maxdepth 1 -print | sort -r | head -n 1`
  173. chat 3 "ssh_port: $ssh_port"
  174. chat 3 "privileges $privileges"
  175. chat 3 "remote_rsync_path: $remote_rsync_path"
  176. chat 3 "ssh_server: $ssh_server"
  177. chat 3 "ssh_alias: $ssh_alias"
  178. chat 3 "ssh_user: $ssh_user"
  179. chat 3 "ssh_args: $ssh_args"
  180. chat 3 "rsync_conf: ${rsync_conf}"
  181. chat 3 "backup_target: $backup_target"
  182. chat 3 "backup_last: $backup_last"
  183. chat 3 "backup_target: $backup_target"
  184. chat 3 "backup_last: $backup_last"
  185. chat 3 "backup_timestamp: $backup_timestamp"
  186. chat 3 "remote_sources: ${remote_sources}"
  187. chat 3 "latest_backup: $latest_backup"
  188. chat 0 "Starting backup for $ssh_alias (using config file $config_file)"
  189. ## Prepare the target directory
  190. if [ ! -d "$backup_target/$backups_path_inprog" ]; then
  191. chat 2 "Preparing target directory."
  192. if [ -d "$backup_target/$backup_timestamp" ]; then
  193. chat 2 "Current backup directory exists. Moving $backup_target/$backup_timestamp to $backup_target/$backups_path_inprog."
  194. chat 3 "mv $backup_target/$backup_timestamp $backup_target/$backups_path_inprog"
  195. mv "$backup_target/$backup_timestamp" "$backup_target/$backups_path_inprog"
  196. link_last "$backup_target/$backups_path_inprog"
  197. else
  198. chat 2 "Creating $backup_target/$backups_path_inprog."
  199. chat 3 "mkdir -p $backup_target/$backups_path_inprog"
  200. mkdir -p "$backup_target/$backups_path_inprog"
  201. fi
  202. else
  203. chat 2 "Target directory $backup_target/$backups_path_inprog exists"
  204. fi
  205. ## Check if $backups_path_last symlink exists in backup folder.
  206. ## If not, symlink the last complete backup to $backups_path_last
  207. ## If no backup exists, link $backups_path_inprog to $backups_path_last
  208. ## Create target folder if it doesn't exist
  209. if [ ! -L "$backup_target/$backups_path_last" ]; then
  210. chat 2 "$backup_target/$backups_path_last not found."
  211. if [ -d "$latest_backup" ] && [ "$latest_backup" -ne "$backup_target" ]; then
  212. chat 2 "Latest backup is $latest_backup."
  213. link_last "$latest_backup"
  214. else
  215. chat 2 "Latest backup not found."
  216. link_last "$backup_target/$backups_path_inprog"
  217. fi
  218. fi
  219. for source in ${remote_sources}
  220. do
  221. chat 2 "Backing up $ssh_alias:$source to $backup_target/$backups_path_inprog."
  222. chat 3 "$rsync -e \"$ssh -p $ssh_port -l $ssh_user $ssh_args\" --link-dest=\"$backup_last\" -aR $ssh_server:$source --rsync-path=$privileges $remote_rsync_path ${rsync_conf} \"$backup_target/$backups_path_inprog\""
  223. $rsync -e "$ssh -p $ssh_port -l $ssh_user $ssh_args" --link-dest="$backup_last" -aR "$ssh_server:$source" --rsync-path="$privileges $remote_rsync_path" ${rsync_conf} "$backup_target/$backups_path_inprog"
  224. done
  225. if [ $? -ne 0 ]; then
  226. ERROR=1
  227. #[TODO] Implement error Handling. Mail?
  228. else
  229. chat 2 "Finishing backup."
  230. ## Rename transfer dir to backup_timestamp
  231. chat 3 "mv $backup_target/$backups_path_inprog $backup_target/$backup_timestamp"
  232. mv "$backup_target"/"$backups_path_inprog" "$backup_target"/"$backup_timestamp"
  233. link_last "$backup_target/$backup_timestamp"
  234. chat 0 "Finished backup for $ssh_alias."
  235. unset_config
  236. fi
  237. }
  238. # Sources a config file and cleans up the backups for this specific host that are older than $keep days
  239. # Usage: backup_host CONFIGFILE
  240. cleanup_host () {
  241. #[TODO] What if all backups are older than $keep days? Prevent from deleting the last backups!
  242. config_file=$1
  243. # Check if config file exists and exit if not.
  244. if [ ! -f "$1" ] ; then chat 1 "Config file $config_file not found."; fi
  245. chat 2 "Loading Config file $config_file"
  246. chat 3 "config_file: $config_file"
  247. # Load config file
  248. . $config_file
  249. ssh_server="$SSH_SERVER"
  250. ssh_alias="${SSH_ALIAS:-$SSH_SERVER}"
  251. backup_target="$backups_dir/$ssh_alias"
  252. keep="${KEEP_DAYS:-$global_keep}"
  253. chat 3 "ssh_server: $ssh_server"
  254. chat 3 "ssh_alias: $ssh_alias"
  255. chat 3 "backup_target: $backup_target"
  256. chat 3 "keep: $keep"
  257. chat 2 "Deleting backups in $backup_target that are older than $keep days."
  258. chat 3 "find $backup_target -type d \( ! -iname \"$backups_path_inprog\" ! -iname \"$backup_last\" ! -iname \".*\" \) -maxdepth 1 -mtime +$keep -exec rm -r '{}' '+'"
  259. find $backup_target -type d \( ! -iname "$backups_path_inprog" ! -iname "$backup_last" ! -iname ".*" \) -maxdepth 1 -mtime +$keep -exec rm -r '{}' '+'
  260. unset_config
  261. }
  262. case "$1" in
  263. ######################## rmbackup.sh HELP ########################
  264. help)
  265. help 0
  266. ;;
  267. ######################## rmbackup.sh BACKUP ########################
  268. backup)
  269. shift; while getopts :f:vd arg; do case ${arg} in
  270. f) config_file=${OPTARG};;
  271. v) VERBOSE=true;;
  272. d) DEBUG=true; VERBOSE=true;;
  273. ?) init; chat 1 "$rmbackup_usage_backup";;
  274. :) init; chat 1 "$rmbackup_usage_backup";;
  275. esac; done; shift $(( ${OPTIND} - 1 ))
  276. init
  277. checkPID
  278. if [ -n "$config_file" ]; then
  279. backup_host $config_file
  280. else
  281. for f in $rmbackup_conf_location/*.conf
  282. do
  283. backup_host $f
  284. done
  285. fi
  286. chat 2 "All jobs done. Exiting."
  287. ;;
  288. ######################## rmbackup.sh CLEANUP ########################
  289. cleanup)
  290. shift; while getopts :f:vd arg; do case ${arg} in
  291. f) config_file=${OPTARG};;
  292. v) VERBOSE=true;;
  293. d) DEBUG=true; VERBOSE=true;;
  294. ?) init; chat 1 "$rmbackup_usage_cleanup";;
  295. :) init; chat 1 "$rmbackup_usage_cleanup";;
  296. esac; done; shift $(( ${OPTIND} - 1 ))
  297. init
  298. checkPID
  299. if [ -n "$config_file" ]; then
  300. cleanup_host $config_file
  301. else
  302. for f in $rmbackup_conf_location/*.conf
  303. do
  304. cleanup_host $f
  305. done
  306. fi
  307. chat 2 "All jobs done. Exiting."
  308. ;;
  309. *)
  310. help 1
  311. ;;
  312. esac