37a8b1e9d0
Borg 1.2 exits 0 for success, exists 1 for backup completion with warnings and exits >1 for proper errors. The most common cause of a warning appear to be a file changing while it is backed up. This causes paste02 backups to consistently email us because our borg backup script exits 1 which we treat as a failure. Update the script so that when borg 1.2.8 is in use we treat rc 1 as a success with warnings rather than failure with warnings. This should make our cron job quieter and match the old 1.1.18 behavior. To better test this we also drop the backup exclusion for the borg backup log file (which we write to as we backup so it is very likely to change during backups). Change-Id: Iab69f0d5951247897d204dcb0a2face424472db0
92 lines
3.4 KiB
Django/Jinja
92 lines
3.4 KiB
Django/Jinja
#!/bin/bash
|
|
|
|
# Flags based on
|
|
# https://borgbackup.readthedocs.io/en/stable/quickstart.html
|
|
|
|
if [ -z "$1" ]; then
|
|
echo "Must specify backup host"
|
|
exit 1
|
|
fi
|
|
|
|
BORG="/opt/borg/bin/borg"
|
|
BORG_CREATE="${BORG} create --verbose --filter AME --list --stats --show-rc --compression lz4 --exclude-caches "
|
|
|
|
# Setting this, so the repo does not need to be given on the commandline:
|
|
export BORG_REPO="ssh://{{ borg_username}}@${1}/opt/backups/{{ borg_username }}/backup"
|
|
|
|
# some helpers and error handling:
|
|
info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; }
|
|
trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM
|
|
|
|
info "Starting backup"
|
|
|
|
# This avoids UI prompts when first accessing the remote repository
|
|
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=1
|
|
|
|
# Backup the most important directories into an archive named after
|
|
# the machine this script is currently running on:
|
|
# TODO(clarkb) Borg 1.2 deprecated exclude paths starting with a leading /
|
|
# Borg 1.2 should strip them off for us, but we should clean up our excludes
|
|
# after everything is running 1.2 or newer.
|
|
${BORG_CREATE} \
|
|
{% for item in borg_backup_excludes + borg_backup_excludes_extra -%}
|
|
--exclude '{{ item }}' \
|
|
{% endfor -%}
|
|
::'{hostname}-filesystem-{now}' \
|
|
{% for item in borg_backup_dirs + borg_backup_dirs_extra -%}
|
|
{{ item }} {{ '\\' if not loop.last }}
|
|
{% endfor -%}
|
|
|
|
backup_exit=$?
|
|
|
|
# Default stream_exit to success as we may not backup any streams
|
|
stream_exit=0
|
|
for f in $(shopt -s nullglob; echo /etc/borg-streams/*)
|
|
do
|
|
stream_name=$(basename $f)
|
|
info "Backing up stream archive $stream_name"
|
|
bash $f | ${BORG_CREATE} --stdin-name ${stream_name} \
|
|
::"{hostname}-${stream_name}-{now}" -
|
|
|
|
_status=( "${PIPESTATUS[@]}" )
|
|
if [[ ${_status[0]} -ne 0 ]]; then
|
|
info "Streaming script ${f} failed!"
|
|
info "Note that problems in the ssh connectivity might cause the streaming script to fail. You may need to check both halves of the streaming backup."
|
|
stream_exit=${_status[0]}
|
|
elif [[ ${_status[1]} -ne 0 ]]; then
|
|
# We don't check BORG_VERSION here because streaming backups should
|
|
# never have the file change under them while backing up.
|
|
info "Borg failed (rc: ${_status[1]})!"
|
|
stream_exit=${_status[1]}
|
|
fi
|
|
done
|
|
|
|
BORG_VERSION=$(${BORG} --version)
|
|
# Default to failure
|
|
final_exit=1
|
|
|
|
if [ ${backup_exit} -eq 0 ] && [ ${stream_exit} -eq 0 ] ; then
|
|
info "Backup finished successfully"
|
|
final_exit=0
|
|
elif [ ${backup_exit} -eq 1 ] && [ ${stream_exit} -eq 0 ] && \
|
|
[ ! "${BORG_VERSION}" \< "borg 1.2" ] ; then
|
|
# Borg 1.2 and newer exit with rc 1 if warnings occur. The most common
|
|
# warning is for files that change while being backed up. We treat that
|
|
# as a successful backup if it occurs.
|
|
# Note: Use not less than so that all variants of 1.1.xy don't match.
|
|
# Using greater than we'd get comparisons like 1.1.8 is greater
|
|
# than 1.1.18.
|
|
info "Backup finished with warnings."
|
|
final_exit=0
|
|
else
|
|
info "Backup finished with errors"
|
|
if [ ${BORG_UNDER_CRON:-0} -eq 1 ]; then
|
|
echo "Backups failed on host $(hostname) at $(date)." | \
|
|
mail -s "ACTION REQUIRED: Backup failed on $(hostname)" infra-root@openstack.org
|
|
fi
|
|
# Attempt to preserve as much information about the error code as possible.
|
|
(( final_exit = backup_exit || stream_exit ))
|
|
fi
|
|
|
|
exit ${final_exit}
|