03ad7d6ace
As database backups can grow substantially in size, compressing backups helps to preserve disk space. While the mariabackup utility offers no compression by itself, we can stream the backup into a compression tool to create an archive [1]. The xtrabackup_checkpoints file, which contains metadata on a backup, gets stored alongside the archive, allowing to create incremental backups from non-compressed backups and vice-versa [2]. One thing to note, is that compressed backups cannot be prepared in advance, this step must be manually carried out by the user. Backup compression is disabled by default and different compressors can be chosen (zstd, xz, ...), with gzip being the default. [1] https://mariadb.com/kb/en/using-encryption-and-compression-tools-with-mariabackup/ [2] https://mariadb.com/kb/en/incremental-backup-and-restore-with-mariabackup/#combining-with-stream-output Depends-On: https://review.opendev.org/c/openstack/openstack-ansible/+/888437 Change-Id: I28c6a0e0b41d4d29c3e79e601de45ea373dee4fb Signed-off-by: Simon Hensel <simon.hensel@inovex.de> (cherry picked from commit 60009ed7cebe9c082592fd564b1577068ef94b6c) (cherry picked from commit 41801ab04d3c64d61b881d410691e05e0312114d)
291 lines
11 KiB
Django/Jinja
Executable File
291 lines
11 KiB
Django/Jinja
Executable File
#!/usr/bin/python3
|
|
# {{ ansible_managed }}
|
|
from subprocess import Popen, PIPE, check_output, run
|
|
from argparse import ArgumentParser
|
|
from shutil import rmtree
|
|
from time import strftime, mktime, sleep
|
|
from datetime import datetime, timedelta
|
|
import os
|
|
|
|
def get_opts():
|
|
parser = ArgumentParser(
|
|
usage="python3 mariabackup_script <destdir> [--full-backup][--increment] [--suffix=<suffix>] [--defaults-file=<defaults-file>]",
|
|
prog="Mariadb Backup Script",
|
|
description="""
|
|
This program makes a mariadb backup with Mariabackup
|
|
""",)
|
|
parser.add_argument(
|
|
"destdir",
|
|
help="Specifying directory for storing backups",
|
|
)
|
|
parser.add_argument(
|
|
"-f",
|
|
"--full-backup",
|
|
action="store_true",
|
|
dest="fullbackup_flag",
|
|
default=False,
|
|
help="Flag for creation of full backup",
|
|
)
|
|
parser.add_argument(
|
|
"-i",
|
|
"--increment",
|
|
action="store_true",
|
|
dest="increment_flag",
|
|
default=False,
|
|
help="Flag to make incremental backup, based on the latest backup",
|
|
)
|
|
parser.add_argument(
|
|
"--compress",
|
|
dest="compress_flag",
|
|
default=False,
|
|
type=eval,
|
|
choices=[True, False],
|
|
help="Flag to compress created backups",
|
|
)
|
|
parser.add_argument(
|
|
"--compressor",
|
|
dest="compressor",
|
|
default="gzip",
|
|
type=str,
|
|
help="The compressor to use when compressing backups (default: gzip)",
|
|
)
|
|
parser.add_argument(
|
|
"-c",
|
|
"--copies",
|
|
dest="copies_flag",
|
|
default=False,
|
|
type=int,
|
|
help="Specifying how much copies to rotate",
|
|
)
|
|
parser.add_argument(
|
|
"--check",
|
|
action="store_true",
|
|
dest="check_flag",
|
|
default=False,
|
|
help="Checking last mariadb full backup for their relevancy",
|
|
)
|
|
parser.add_argument(
|
|
"--warning",
|
|
dest="warning_value",
|
|
default=False,
|
|
type=int,
|
|
help="When to raise warning (for --check) in days",
|
|
)
|
|
parser.add_argument(
|
|
"--critical",
|
|
dest="critical_value",
|
|
default=False,
|
|
type=int,
|
|
help="When to raise critical (for --check) in days",
|
|
)
|
|
parser.add_argument(
|
|
"-s",
|
|
"--suffix",
|
|
dest="suffix",
|
|
default=False,
|
|
type=str,
|
|
help="Added to the filename of backups"
|
|
)
|
|
parser.add_argument(
|
|
"--defaults-file",
|
|
dest="defaults_file",
|
|
type=str,
|
|
help="A cnf file can specified to the mariabackup process"
|
|
)
|
|
opts = parser.parse_args()
|
|
return opts
|
|
|
|
|
|
def check_backups(dest, warning, critical, full_backup_filename):
|
|
try:
|
|
last_mariabackup_full = datetime.strptime(max([os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith(full_backup_filename)], key=os.path.getmtime).split(full_backup_filename)[1], '%Y%m%d-%H%M%S')
|
|
except ValueError:
|
|
print("No files found, you may need to check your destination directory or add a suffix.")
|
|
raise SystemExit()
|
|
warning_time = datetime.today() - timedelta(days=warning)
|
|
critical_time = datetime.today() - timedelta(days=critical)
|
|
print_info = "Last mariadb backup date "+str(last_mariabackup_full)
|
|
if last_mariabackup_full < critical_time:
|
|
print(print_info)
|
|
raise SystemExit(2)
|
|
elif last_mariabackup_full < warning_time:
|
|
print(print_info)
|
|
raise SystemExit(1)
|
|
else:
|
|
print(print_info)
|
|
raise SystemExit(0)
|
|
|
|
|
|
def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args, compress, compressor):
|
|
check_lock_file()
|
|
get_lock_file()
|
|
try:
|
|
err = open(os.path.normpath(dest+"/backup.log"), "w")
|
|
if compress:
|
|
#Creating compressed full backup
|
|
os.makedirs(dest+"/"+full_backup_filename+curtime, exist_ok=True)
|
|
mariabackup_run = Popen(
|
|
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--extra-lsndir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=PIPE, stderr=err
|
|
)
|
|
compressed_backup = open(os.path.normpath(dest+"/"+full_backup_filename+curtime+"/"+full_backup_filename+curtime), "wb")
|
|
run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup)
|
|
mariabackup_run.wait()
|
|
mariabackup_res = mariabackup_run.communicate()
|
|
if mariabackup_run.returncode:
|
|
print(mariabackup_res[1])
|
|
compressed_backup.close()
|
|
else:
|
|
#Creating full backup
|
|
mariabackup_run = Popen(
|
|
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err
|
|
)
|
|
mariabackup_run.wait()
|
|
mariabackup_res = mariabackup_run.communicate()
|
|
if mariabackup_run.returncode:
|
|
print(mariabackup_res[1])
|
|
#Preparing full backup
|
|
err_p = open(os.path.normpath(dest+"/prepare.log"), "w")
|
|
mariabackup_prep = Popen(
|
|
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p
|
|
)
|
|
mariabackup_prep.wait()
|
|
mariabackup_prep_res = mariabackup_prep.communicate()
|
|
if mariabackup_prep.returncode:
|
|
print(mariabackup_prep_res[1])
|
|
err_p.close()
|
|
err.close()
|
|
except OSError:
|
|
print("Please, check that Mariabackup is installed")
|
|
except Exception as e:
|
|
print(e)
|
|
finally:
|
|
os.unlink("/var/run/mariabackup-galera/db_backup.pid")
|
|
|
|
|
|
def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args, compress, compressor):
|
|
check_lock_file()
|
|
get_lock_file()
|
|
try:
|
|
basedir = max([ os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith('mariabackup-')], key=os.path.getmtime)
|
|
except ValueError:
|
|
print("No full backup found, cannot create incremental backup.")
|
|
os.unlink("/var/run/mariabackup-galera/db_backup.pid")
|
|
raise SystemExit(1)
|
|
try:
|
|
err = open(os.path.normpath(dest+"/increment.err"), "w")
|
|
if compress:
|
|
#Creating compressed incremental backup
|
|
os.makedirs(dest+"/"+increment_backup_filename+curtime, exist_ok=True)
|
|
mariabackup_run = Popen(
|
|
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--incremental-basedir="+basedir, "--extra-lsndir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime)], stdout=PIPE, stderr=err
|
|
)
|
|
compressed_backup = open(os.path.normpath(dest+"/"+increment_backup_filename+curtime+"/"+increment_backup_filename+curtime), "wb")
|
|
run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup)
|
|
mariabackup_run.wait()
|
|
mariabackup_res = mariabackup_run.communicate()
|
|
if mariabackup_run.returncode:
|
|
print(mariabackup_res[1])
|
|
compressed_backup.close()
|
|
else:
|
|
#Creating incremental backup
|
|
mariabackup_run = Popen(
|
|
["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err
|
|
)
|
|
mariabackup_run.wait()
|
|
mariabackup_res = mariabackup_run.communicate()
|
|
if mariabackup_run.returncode:
|
|
print(mariabackup_res[1])
|
|
err.close()
|
|
except OSError:
|
|
print("Please, check that Mariabackup is installed")
|
|
except Exception as e:
|
|
print(e)
|
|
finally:
|
|
os.unlink("/var/run/mariabackup-galera/db_backup.pid")
|
|
|
|
|
|
def rotate_backups(dest, copies, full_backup_filename, increment_backup_filename):
|
|
check_lock_file()
|
|
get_lock_file()
|
|
full_list = [os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith(full_backup_filename)]
|
|
increment_list = [ os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith(increment_backup_filename)]
|
|
# Rotate full backups
|
|
if len(full_list) > copies:
|
|
full_list.sort()
|
|
while len(full_list) > copies:
|
|
oldest_full_backup = min(full_list, key=os.path.getmtime)
|
|
full_list.remove(oldest_full_backup)
|
|
rmtree(oldest_full_backup)
|
|
# Remove all incremental backups older than the oldest full backup
|
|
oldest_full_backup_timestamp = parsedate(oldest_full_backup.split(full_backup_filename)[1])
|
|
for increment in increment_list:
|
|
increment_timestamp = parsedate(increment.split(increment_backup_filename)[1])
|
|
if increment_timestamp < oldest_full_backup_timestamp:
|
|
rmtree(increment)
|
|
os.unlink("/var/run/mariabackup-galera/db_backup.pid")
|
|
|
|
|
|
def parsedate(s):
|
|
return mktime(datetime.strptime(s, '%Y%m%d-%H%M%S').timetuple())
|
|
|
|
|
|
def check_lock_file():
|
|
timer = 0
|
|
while os.path.isfile("/var/run/mariabackup-galera/db_backup.pid"):
|
|
sleep(60)
|
|
timer += 1
|
|
if timer == 120:
|
|
print("timeout of waiting another process is reached")
|
|
raise SystemExit(1)
|
|
|
|
|
|
def get_lock_file():
|
|
try:
|
|
pid = open('/var/run/mariabackup-galera/db_backup.pid', 'w')
|
|
pid.write(str(os.getpid()))
|
|
pid.close()
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
|
|
def main():
|
|
opts = get_opts()
|
|
curtime = strftime("%Y%m%d-%H%M%S")
|
|
|
|
if not opts.copies_flag and opts.fullbackup_flag:
|
|
raise NameError("--copies flag is required for running full backup.")
|
|
|
|
full_backup_filename = "mariabackup-full_"
|
|
increment_backup_filename = "mariabackup-increment_"
|
|
if opts.suffix:
|
|
full_backup_filename = ("mariabackup-full-" + opts.suffix + "_")
|
|
increment_backup_filename = ("mariabackup-increment-" + opts.suffix + "_")
|
|
|
|
extra_mariabackup_args = []
|
|
# --defaults-file must be specified straight after the process
|
|
if opts.defaults_file:
|
|
extra_mariabackup_args = ["--defaults-file=" + opts.defaults_file] + extra_mariabackup_args
|
|
|
|
if opts.fullbackup_flag and opts.increment_flag:
|
|
raise NameError("Only one flag can be specified per operation")
|
|
elif opts.fullbackup_flag:
|
|
create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor)
|
|
rotate_backups(opts.destdir, opts.copies_flag, full_backup_filename, increment_backup_filename)
|
|
raise SystemExit()
|
|
elif opts.increment_flag:
|
|
create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor)
|
|
raise SystemExit()
|
|
elif opts.check_flag:
|
|
pass
|
|
else:
|
|
raise NameError("either --increment or --full-backup flag is required")
|
|
|
|
if opts.check_flag and (opts.warning_value and opts.critical_value):
|
|
check_backups(warning = opts.warning_value, critical = opts.critical_value, dest = opts.destdir, full_backup_filename = full_backup_filename)
|
|
else:
|
|
raise NameError("--warning and --critical thresholds should be specified for check")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|