diff --git a/defaults/main.yml b/defaults/main.yml index 2361f9f3..b6555678 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -319,6 +319,8 @@ galera_mariadb_backups_user: galera_mariadb_backup galera_mariadb_backups_suffix: "{{ inventory_hostname }}" galera_mariadb_backups_cnf_file: "/etc/mysql/mariabackup.cnf" galera_mariadb_backups_nodes: ["{{ galera_cluster_members[0] }}"] +galera_mariadb_backups_compress: False +galera_mariadb_backups_compressor: gzip galera_mariadb_encryption_enabled: false galera_mariadb_encryption_plugin: "file_key_management" diff --git a/releasenotes/notes/mariabackup-compression-337b04c68f370c1d.yaml b/releasenotes/notes/mariabackup-compression-337b04c68f370c1d.yaml new file mode 100644 index 00000000..f51a8348 --- /dev/null +++ b/releasenotes/notes/mariabackup-compression-337b04c68f370c1d.yaml @@ -0,0 +1,28 @@ +--- +features: + - | + Adds optional compression for backups created with mariabackup. Adds two + new CLI parameters to the mariabackup script that are used to enable + compression and to choose a compression tool. + + * ``--compress=True|False`` + * ``--compressor=`` + + Also introduces new Ansible variables that control the above mentioned + parameters. + + * ``galera_mariadb_backups_compress`` + * ``galera_mariadb_backups_compressor`` + + Each backup archive is stored in a dedicated directory, alongside the + backup metadata. +upgrade: + - | + Backup compression is disabled by default, so no changes need to be made + for existing deployments. Should compression be desired, set + ``galera_mariadb_backups_compress`` to ``True``. Choose a compression tool + with ``galera_mariadb_backups_compressor``, default is ``gzip``. +others: + - | + Compressed backups cannot be prepared in advance, this step must be + manually carried out by the user before importing it into MariaDB. diff --git a/tasks/galera_server_backups.yml b/tasks/galera_server_backups.yml index 940c0e2e..91a3c314 100644 --- a/tasks/galera_server_backups.yml +++ b/tasks/galera_server_backups.yml @@ -39,6 +39,7 @@ - /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }} --full-backup --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }} --defaults-file={{ galera_mariadb_backups_cnf_file }} + --compress={{ galera_mariadb_backups_compress }} --compressor={{ galera_mariadb_backups_compressor }} environment: UMASK: '0640' UMASK_DIR: '0750' @@ -66,6 +67,7 @@ - /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }} --increment --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }} --defaults-file={{ galera_mariadb_backups_cnf_file }} + --compress={{ galera_mariadb_backups_compress }} --compressor={{ galera_mariadb_backups_compressor }} environment: UMASK: '0640' UMASK_DIR: '0750' diff --git a/templates/mariabackup_script.py.j2 b/templates/mariabackup_script.py.j2 index 6a480e1c..8659700f 100755 --- a/templates/mariabackup_script.py.j2 +++ b/templates/mariabackup_script.py.j2 @@ -1,11 +1,10 @@ #!/usr/bin/python3 # {{ ansible_managed }} -from subprocess import Popen, PIPE +from subprocess import Popen, PIPE, check_output, run from argparse import ArgumentParser from shutil import rmtree from time import strftime, mktime, sleep from datetime import datetime, timedelta -import socket import os def get_opts(): @@ -35,6 +34,21 @@ def get_opts(): default=False, help="Flag to make incremental backup, based on the latest backup", ) + parser.add_argument( + "--compress", + dest="compress_flag", + default=False, + type=eval, + choices=[True, False], + help="Flag to compress created backups", + ) + parser.add_argument( + "--compressor", + dest="compressor", + default="gzip", + type=str, + help="The compressor to use when compressing backups (default: gzip)", + ) parser.add_argument( "-c", "--copies", @@ -102,30 +116,44 @@ def check_backups(dest, warning, critical, full_backup_filename): raise SystemExit(0) -def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args): +def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args, compress, compressor): check_lock_file() get_lock_file() try: - #Creating full backup err = open(os.path.normpath(dest+"/backup.log"), "w") - mariabackup_run = Popen( - ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err - ) - mariabackup_run.wait() - mariabackup_res = mariabackup_run.communicate() - if mariabackup_run.returncode: - print(mariabackup_res[1]) + if compress: + #Creating compressed full backup + os.makedirs(dest+"/"+full_backup_filename+curtime, exist_ok=True) + mariabackup_run = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--extra-lsndir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=PIPE, stderr=err + ) + compressed_backup = open(os.path.normpath(dest+"/"+full_backup_filename+curtime+"/"+full_backup_filename+curtime), "wb") + run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup) + mariabackup_run.wait() + mariabackup_res = mariabackup_run.communicate() + if mariabackup_run.returncode: + print(mariabackup_res[1]) + compressed_backup.close() + else: + #Creating full backup + mariabackup_run = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err + ) + mariabackup_run.wait() + mariabackup_res = mariabackup_run.communicate() + if mariabackup_run.returncode: + print(mariabackup_res[1]) + #Preparing full backup + err_p = open(os.path.normpath(dest+"/prepare.log"), "w") + mariabackup_prep = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p + ) + mariabackup_prep.wait() + mariabackup_prep_res = mariabackup_prep.communicate() + if mariabackup_prep.returncode: + print(mariabackup_prep_res[1]) + err_p.close() err.close() - #Preparing full backup - err_p = open(os.path.normpath(dest+"/prepare.log"), "w") - mariabackup_prep = Popen( - ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p - ) - mariabackup_prep.wait() - mariabackup_prep_res = mariabackup_prep.communicate() - if mariabackup_prep.returncode: - print(mariabackup_prep_res[1]) - err_p.close() except OSError: print("Please, check that Mariabackup is installed") except Exception as e: @@ -134,7 +162,7 @@ def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_ar os.unlink("/var/run/mariabackup-galera/db_backup.pid") -def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args): +def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args, compress, compressor): check_lock_file() get_lock_file() try: @@ -145,14 +173,28 @@ def create_increment_backup(dest, curtime, increment_backup_filename, extra_mari raise SystemExit(1) try: err = open(os.path.normpath(dest+"/increment.err"), "w") - #Creating incremental backup - mariabackup_run = Popen( - ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err - ) - mariabackup_run.wait() - mariabackup_res = mariabackup_run.communicate() - if mariabackup_run.returncode: - print(mariabackup_res[1]) + if compress: + #Creating compressed incremental backup + os.makedirs(dest+"/"+increment_backup_filename+curtime, exist_ok=True) + mariabackup_run = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--stream=xbstream", "--incremental-basedir="+basedir, "--extra-lsndir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime)], stdout=PIPE, stderr=err + ) + compressed_backup = open(os.path.normpath(dest+"/"+increment_backup_filename+curtime+"/"+increment_backup_filename+curtime), "wb") + run([compressor], stdin=mariabackup_run.stdout, stdout=compressed_backup) + mariabackup_run.wait() + mariabackup_res = mariabackup_run.communicate() + if mariabackup_run.returncode: + print(mariabackup_res[1]) + compressed_backup.close() + else: + #Creating incremental backup + mariabackup_run = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err + ) + mariabackup_run.wait() + mariabackup_res = mariabackup_run.communicate() + if mariabackup_run.returncode: + print(mariabackup_res[1]) err.close() except OSError: print("Please, check that Mariabackup is installed") @@ -227,11 +269,11 @@ def main(): if opts.fullbackup_flag and opts.increment_flag: raise NameError("Only one flag can be specified per operation") elif opts.fullbackup_flag: - create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args) + create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor) rotate_backups(opts.destdir, opts.copies_flag, full_backup_filename, increment_backup_filename) raise SystemExit() elif opts.increment_flag: - create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args) + create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args, opts.compress_flag, opts.compressor) raise SystemExit() elif opts.check_flag: pass