From 0d01d941b1049590e54df1d783015e3af9dfbd19 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Wed, 10 Feb 2021 12:25:21 +1100
Subject: [PATCH] borg-backup-server: run a weekly backup verification

This checks the backup archives and alerts us if anything seems wrong.
This will take a few hours, so we run once a week.

Change-Id: I832c0d29a37df94d4bf2704c59bb3f8d855c3cc8
---
 .../files/verify-borg-backups.sh              | 22 ++++++++++++++++++
 .../roles/borg-backup-server/tasks/main.yaml  | 23 +++++++++++++++++++
 testinfra/test_borg_backups.py                |  8 +++++++
 zuul.d/system-config-run.yaml                 |  1 +
 4 files changed, 54 insertions(+)
 create mode 100644 playbooks/roles/borg-backup-server/files/verify-borg-backups.sh

diff --git a/playbooks/roles/borg-backup-server/files/verify-borg-backups.sh b/playbooks/roles/borg-backup-server/files/verify-borg-backups.sh
new file mode 100644
index 0000000000..88061e3b2f
--- /dev/null
+++ b/playbooks/roles/borg-backup-server/files/verify-borg-backups.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+pushd /opt/backups
+
+for u in borg-*; do
+    BORG_REPO=/opt/backups/$u/backup
+
+    sudo BORG_RELOCATED_REPO_ACCESS_IS_OK=y BORG_REPO=${BORG_REPO} -u ${u} -s  <<'EOF'
+
+    echo "$(date) Verifying ${BORG_REPO} ..."
+    /opt/borg/bin/borg check --verify-data
+    if [[ $? -ne 0 ]]; then
+        echo "$(date) *** Verification failed"
+        echo "Inconsistency found in backup ${BORG_REPO} on $(hostname) at $(date)" |
+            mail -s "ACTION REQUIRED: Backup inconsistency: ${BORG_REPO}" infra-root@openstack.org
+    else
+        echo "$(date) ... done"
+        echo
+    fi
+
+EOF
+done
diff --git a/playbooks/roles/borg-backup-server/tasks/main.yaml b/playbooks/roles/borg-backup-server/tasks/main.yaml
index fa38c961ce..88eb372e46 100644
--- a/playbooks/roles/borg-backup-server/tasks/main.yaml
+++ b/playbooks/roles/borg-backup-server/tasks/main.yaml
@@ -31,6 +31,29 @@
     minute: '0'
     hour: '0'
 
+- name: Install backup verification
+  copy:
+    src: 'verify-borg-backups.sh'
+    dest: '/usr/local/bin/verify-borg-backups'
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Run backup verification
+  cron:
+    name: verify-borg-backups
+    state: present
+    job: '/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log'
+    minute: '0'
+    hour: '0'
+    weekday: '0'
+
+- name: Rotate verification logs
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: '/var/log/verify-borg-backups.log'
+
 - name: Build all borg users from backup hosts
   set_fact:
     borg_users: '{{ borg_users }} + [ {{ hostvars[item]["borg_user"] }} ]'
diff --git a/testinfra/test_borg_backups.py b/testinfra/test_borg_backups.py
index 683b5c85c9..d395abe047 100644
--- a/testinfra/test_borg_backups.py
+++ b/testinfra/test_borg_backups.py
@@ -96,3 +96,11 @@ def test_borg_server_prune(host):
 
     cmd = host.run('echo "prune" | /usr/local/bin/prune-borg-backups &> /var/log/prune-borg-backups.log')
     assert cmd.succeeded
+
+def test_borg_server_verify(host):
+    hostname = host.backend.get_hostname()
+    if hostname.startswith('borg-backup-test'):
+        pytest.skip()
+
+    cmd = host.run('/usr/local/bin/verify-borg-backups &> /var/log/verify-borg-backups.log')
+    assert cmd.succeeded
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index 83fe0e50e2..ccf1727564 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -360,6 +360,7 @@
       borg-backup01.region.provider.opendev.org:
         host_copy_output:
           '/var/log/prune-borg-backups.log': logs
+          '/var/log/verify-borg-backups.log': logs
       borg-backup-test01.opendev.org:
         host_copy_output:
           '/var/log/borg-backup-borg-backup01.region.provider.opendev.org.log': logs