From 9a8ee0293b2053938365cff6fd8b04e2c66d5aeb Mon Sep 17 00:00:00 2001 From: Georgina Date: Wed, 30 Sep 2020 11:46:36 +0000 Subject: [PATCH] Ability to take mariadb backups using mariabackup This patch allows a user to specify a directory they would like their database backups to be put into. A number of full backup copies will be kept alongside their corresponding increments (if any). Users can specify multiple systemd timer OnCalendar directives for taking full back ups and incremental backups. Incremental backups are optional. Depends-On: https://review.opendev.org/759146/ Change-Id: Id78151a23ec5fcc424bfba669673a4a2df83ef23 --- defaults/main.yml | 16 ++ tasks/galera_server_backups.yml | 82 ++++++++++ tasks/galera_server_main.yml | 7 + templates/mariabackup.cnf.j2 | 3 + templates/mariabackup_script.py.j2 | 245 +++++++++++++++++++++++++++++ 5 files changed, 353 insertions(+) create mode 100644 tasks/galera_server_backups.yml create mode 100644 templates/mariabackup.cnf.j2 create mode 100755 templates/mariabackup_script.py.j2 diff --git a/defaults/main.yml b/defaults/main.yml index f2c55e71..18de8049 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -206,3 +206,19 @@ galera_client_my_cnf_overrides: {} # defined key is not present. By default this will try and pull from the # "galera_server" group and fall back to localhost. galera_ssl_server: "{{ (galera_cluster_members | default(['localhost']))[0] }}" + +# Configure backups of database +# copies is the number of full backups to be kept, the corresponding +# incremental backups will also be kept. Uses systemd timer instead of cron. +galera_mariadb_backups_enabled: false +galera_mariadb_backups_path: "/var/backup/mariadb_backups" +galera_mariadb_backups_full_copies: 2 +galera_mariadb_backups_full_on_calendar: "*-*-* 00:00:00" +galera_mariadb_backups_increment_on_calendar: + - "*-*-* 06:00:00" + - "*-*-* 12:00:00" + - "*-*-* 18:00:00" +galera_mariadb_backups_user: galera_mariadb_backup +galera_mariadb_backups_suffix: "{{ inventory_hostname }}" +galera_mariadb_backups_cnf_file: "/etc/mysql/mariabackup.cnf" +galera_mariadb_backups_nodes: ["{{ galera_cluster_members[0] }}"] diff --git a/tasks/galera_server_backups.yml b/tasks/galera_server_backups.yml new file mode 100644 index 00000000..f9621d26 --- /dev/null +++ b/tasks/galera_server_backups.yml @@ -0,0 +1,82 @@ +--- + +- name: Create mariadb back up directory + file: + path: "{{ galera_mariadb_backups_path }}" + state: "directory" + group: "root" + owner: "root" + mode: "0755" + +- name: Template out mariadb backup script + template: + src: "mariabackup_script.py.j2" + dest: "{{ galera_mariadb_backups_path }}/mariabackup_script.py" + mode: "0755" + +- name: Template out mariabackup cnf file + template: + src: "mariabackup.cnf.j2" + dest: "{{ galera_mariadb_backups_cnf_file }}" + mode: "0644" + +- name: Create service and timer for full backups + import_role: + name: systemd_service + vars: + systemd_service_enabled: true + systemd_service_restart_changed: false + systemd_user_name: "root" + systemd_group_name: "root" + systemd_services: + - service_name: "mariabackup-full" + execstarts: + - /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }} + --full-backup --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }} + --defaults-file={{ galera_mariadb_backups_cnf_file }} + timer: + state: "started" + options: + OnCalendar: "{{ galera_mariadb_backups_full_on_calendar }}" + Persistent: true + Unit: "mariabackup-full.service" + +- name: Create service and timer for incremental backups + import_role: + name: systemd_service + vars: + systemd_service_enabled: true + systemd_service_restart_changed: false + systemd_user_name: "root" + systemd_group_name: "root" + systemd_services: + - service_name: "mariabackup-increment" + execstarts: + - /usr/bin/python3 {{ galera_mariadb_backups_path }}/mariabackup_script.py {{ galera_mariadb_backups_path }} + --increment --copies={{ galera_mariadb_backups_full_copies }} --suffix={{ galera_mariadb_backups_suffix }} + --defaults-file={{ galera_mariadb_backups_cnf_file }} + timer: + state: "started" + options: + OnCalendar: "{{ galera_mariadb_backups_increment_on_calendar }}" + Persistent: true + Unit: "mariabackup-increment.service" + when: galera_mariadb_backups_increment_on_calendar is defined + +- name: Grant access to the database for the backup service + delegate_to: "{{ openstack_db_setup_host | default('localhost') }}" + vars: + galera_db_setup_host: "{{ openstack_db_setup_host | default('localhost') }}" + ansible_python_interpreter: "{{ openstack_db_setup_python_interpreter | + default((galera_db_setup_host == 'localhost') | + ternary(ansible_playbook_python, ansible_python['executable'])) }}" + community.mysql.mysql_user: + name: "{{ galera_mariadb_backups_user }}" + password: "{{ galera_mariadb_backups_password }}" + host: "%" + priv: "*.*:RELOAD,PROCESS,LOCK TABLES,REPLICATION CLIENT" + append_privs: yes + login_host: "{{ galera_address }}" + login_port: 3306 + no_log: true + run_once: true diff --git a/tasks/galera_server_main.yml b/tasks/galera_server_main.yml index ff907e37..e0b07642 100644 --- a/tasks/galera_server_main.yml +++ b/tasks/galera_server_main.yml @@ -91,3 +91,10 @@ when: inventory_hostname == galera_server_bootstrap_node tags: - galera_server-config + +- include_tasks: galera_server_backups.yml + when: + - galera_mariadb_backups_enabled | bool + - inventory_hostname in galera_mariadb_backups_nodes + tags: + - galera_server-backups diff --git a/templates/mariabackup.cnf.j2 b/templates/mariabackup.cnf.j2 new file mode 100644 index 00000000..167041ce --- /dev/null +++ b/templates/mariabackup.cnf.j2 @@ -0,0 +1,3 @@ +[mariabackup] +user = {{ galera_mariadb_backups_user }} +password = {{ galera_mariadb_backups_password }} diff --git a/templates/mariabackup_script.py.j2 b/templates/mariabackup_script.py.j2 new file mode 100755 index 00000000..0ffc9672 --- /dev/null +++ b/templates/mariabackup_script.py.j2 @@ -0,0 +1,245 @@ +#!/usr/bin/python3 +# {{ ansible_managed }} +from subprocess import Popen, PIPE +from argparse import ArgumentParser +from shutil import rmtree +from time import strftime, mktime, sleep +from datetime import datetime, timedelta +import socket +import os + +def get_opts(): + parser = ArgumentParser( + usage="python3 mariabackup_script [--full-backup][--increment] [--suffix=] [--defaults-file=]", + prog="Mariadb Backup Script", + description=""" + This program makes a mariadb backup with Mariabackup + """,) + parser.add_argument( + "destdir", + help="Specifying directory for storing backups", + ) + parser.add_argument( + "-f", + "--full-backup", + action="store_true", + dest="fullbackup_flag", + default=False, + help="Flag for creation of full backup", + ) + parser.add_argument( + "-i", + "--increment", + action="store_true", + dest="increment_flag", + default=False, + help="Flag to make incremental backup, based on the latest backup", + ) + parser.add_argument( + "-c", + "--copies", + dest="copies_flag", + default=False, + type=int, + help="Specifying how much copies to rotate", + ) + parser.add_argument( + "--check", + action="store_true", + dest="check_flag", + default=False, + help="Checking last mariadb full backup for their relevancy", + ) + parser.add_argument( + "--warning", + dest="warning_value", + default=False, + type=int, + help="When to raise warning (for --check) in days", + ) + parser.add_argument( + "--critical", + dest="critical_value", + default=False, + type=int, + help="When to raise critical (for --check) in days", + ) + parser.add_argument( + "-s", + "--suffix", + dest="suffix", + default=False, + type=str, + help="Added to the filename of backups" + ) + parser.add_argument( + "--defaults-file", + dest="defaults_file", + type=str, + help="A cnf file can specified to the mariabackup process" + ) + opts = parser.parse_args() + return opts + + +def check_backups(dest, warning, critical, full_backup_filename): + try: + last_mariabackup_full = datetime.strptime(max([os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith(full_backup_filename)], key=os.path.getmtime).split(full_backup_filename)[1], '%Y%m%d-%H%M%S') + except ValueError: + print("No files found, you may need to check your destination directory or add a suffix.") + raise SystemExit() + warning_time = datetime.today() - timedelta(days=warning) + critical_time = datetime.today() - timedelta(days=critical) + print_info = "Last mariadb backup date "+str(last_mariabackup_full) + if last_mariabackup_full < critical_time: + print(print_info) + raise SystemExit(2) + elif last_mariabackup_full < warning_time: + print(print_info) + raise SystemExit(1) + else: + print(print_info) + raise SystemExit(0) + + +def create_full_backup(dest, curtime, full_backup_filename, extra_mariabackup_args): + check_lock_file() + get_lock_file() + try: + #Creating full backup + err = open(os.path.normpath(dest+"/backup.log"), "w") + mariabackup_run = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err + ) + mariabackup_run.wait() + mariabackup_res = mariabackup_run.communicate() + if mariabackup_run.returncode: + print(mariabackup_res[1]) + err.close() + #Preparing full backup + err_p = open(os.path.normpath(dest+"/prepare.log"), "w") + mariabackup_prep = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--prepare", "--apply-log-only", "--target-dir="+os.path.normpath(dest+"/"+full_backup_filename+curtime)], stdout=None, stderr=err_p + ) + mariabackup_prep.wait() + mariabackup_prep_res = mariabackup_prep.communicate() + if mariabackup_prep.returncode: + print(mariabackup_prep_res[1]) + err_p.close() + except OSError: + print("Please, check that Mariabackup is installed") + except Exception as e: + print(e) + finally: + os.unlink("/var/run/db_backup.pid") + + +def create_increment_backup(dest, curtime, increment_backup_filename, extra_mariabackup_args): + check_lock_file() + get_lock_file() + try: + basedir = max([ os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith('mariabackup-')], key=os.path.getmtime) + except(OSError): + basedir="./" + try: + err = open(os.path.normpath(dest+"/increment.err"), "w") + #Creating incremental backup + mariabackup_run = Popen( + ["/usr/bin/mariabackup"] + extra_mariabackup_args + ["--backup", "--target-dir="+os.path.normpath(dest+"/"+increment_backup_filename+curtime), "--incremental-basedir="+basedir], stdout=None, stderr=err + ) + mariabackup_run.wait() + mariabackup_res = mariabackup_run.communicate() + if mariabackup_run.returncode: + print(mariabackup_res[1]) + err.close() + except OSError: + print("Please, check that Mariabackup is installed") + except Exception as e: + print(e) + finally: + os.unlink("/var/run/db_backup.pid") + + +def rotate_backups(dest, copies, full_backup_filename, increment_backup_filename): + check_lock_file() + get_lock_file() + full_list = [os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith(full_backup_filename)] + increment_list = [ os.path.normpath(dest+'/'+f) for f in os.listdir(dest) if f.startswith(increment_backup_filename)] + if len(full_list) > copies: + full_list.sort() + left = parsedate(full_list[0].split(full_backup_filename)[1]) + right = parsedate(full_list[1].split(full_backup_filename)[1]) + for files in increment_list: + stamp = parsedate(files.split(increment_backup_filename)[1]) + if stamp > left and stamp < right: + rmtree(files) + while len(full_list) > copies: + folder = min(full_list, key=os.path.getmtime) + full_list.remove(folder) + rmtree(folder) + os.unlink("/var/run/db_backup.pid") + + +def parsedate(s): + return mktime(datetime.strptime(s, '%Y%m%d-%H%M%S').timetuple()) + + +def check_lock_file(): + timer = 0 + while os.path.isfile("/var/run/db_backup.pid"): + sleep(60) + timer += 1 + if timer == 120: + print("timeout of waiting another process is reached") + raise SystemExit(1) + + +def get_lock_file(): + try: + pid = open('/var/run/db_backup.pid', 'w') + pid.write(str(os.getpid())) + pid.close() + except Exception as e: + print(e) + + +def main(): + opts = get_opts() + curtime = strftime("%Y%m%d-%H%M%S") + + if not opts.copies_flag and opts.fullbackup_flag: + raise NameError("--copies flag is required for running full backup.") + + full_backup_filename = "mariabackup-full_" + increment_backup_filename = "mariabackup-increment_" + if opts.suffix: + full_backup_filename = ("mariabackup-full-" + opts.suffix + "_") + increment_backup_filename = ("mariabackup-increment-" + opts.suffix + "_") + + extra_mariabackup_args = [] + # --defaults-file must be specified straight after the process + if opts.defaults_file: + extra_mariabackup_args = ["--defaults-file=" + opts.defaults_file] + extra_mariabackup_args + + if opts.fullbackup_flag and opts.increment_flag: + raise NameError("Only one flag can be specified per operation") + elif opts.fullbackup_flag: + create_full_backup(opts.destdir, curtime, full_backup_filename, extra_mariabackup_args) + rotate_backups(opts.destdir, opts.copies_flag, full_backup_filename, increment_backup_filename) + raise SystemExit() + elif opts.increment_flag: + create_increment_backup(opts.destdir, curtime, increment_backup_filename, extra_mariabackup_args) + raise SystemExit() + elif opts.check_flag: + pass + else: + raise NameError("either --increment or --full-backup flag is required") + + if opts.check_flag and (opts.warning_value and opts.critical_value): + check_backups(warning = opts.warning_value, critical = opts.critical_value, dest = opts.destdir, full_backup_filename = full_backup_filename) + else: + raise NameError("--warning and --critical thresholds should be specified for check") + + +if __name__ == "__main__": + main()