Accomodate zuul's new key management system

Zuul is changing the way its key management system work from implicit
"backups" to explicit exports that can be used for backups. Additionally
to rename projects we will need to update those keys in zk which can be
done with copy and delete commands. We update the rename playbook to use
these.

Depends-On: https://review.opendev.org/c/zuul/zuul/+/803973
Change-Id: I2ba8015392f22ea615bcba7fb0d73a138dc77034

playbooks/rename_repos.yaml

@@ -67,25 +67,14 @@
   gather_facts: False
   remote_user: root
   tasks:
-
     - include_vars: "{{ repolist }}"
 
-    - name: Make organization dirs
-      file:
-        state: directory
-        path: "{{ (item[0] + item[1].new) | dirname }}"
-        group: zuuld
-        owner: zuuld
-        mode: 0755
-      with_nested:
-        - [ '/var/lib/zuul/keys/secrets/project/gerrit/', '/var/lib/zuul/keys/ssh/project/gerrit/' ]
-        - "{{ repos }}"
+    - name: Backup the zuul keys before we modify them
+      command: "docker exec zuul-scheduler_scheduler_1 zuul export-keys /var/lib/zuul/zuul-keys-backup.json"
 
-    - name: Rename keys on Zuul
-      shell: mv {{ item[0] }}{{ item[1].old }} {{ item[0] }}{{ item[1].new }}
-      with_nested:
-        - [ '/var/lib/zuul/keys/secrets/project/gerrit/', '/var/lib/zuul/keys/ssh/project/gerrit/' ]
-        - "{{ repos }}"
+    - name: Copy zuul keys from old to new project
+      command: "docker exec zuul-scheduler_scheduler_1 zuul copy-keys gerrit {{ item.old }} gerrit {{ item.new }}"
+      loop: "{{ repos }}"
 
 - hosts: review
   gather_facts: False
@@ -143,3 +132,13 @@
 
     - name: Start online reindex of changes
       shell: "{{ gerrit_ssh_command }} index start changes --force"
+
+- hosts: zuul-scheduler
+  gather_facts: False
+  remote_user: root
+  tasks:
+    - include_vars: "{{ repolist }}"
+
+    - name: Remove old zuul project keys
+      command: "docker exec zuul-scheduler_scheduler_1 zuul delete-keys gerrit {{ item.old }}"
+      loop: "{{ repos }}"

playbooks/roles/zuul-scheduler/tasks/main.yaml

@@ -49,6 +49,15 @@
   vars:
     tenant: pypa
 
+- name: Backup zk key data daily
+  cron:
+    name: Backup zuul zk key data
+    minute: 0
+    hour: 0
+    # Note we backup to /var/lib/zuul because that is what we have bind
+    # mounted in the container and we are operating relative to that context
+    job: docker exec zuul-scheduler_scheduler_1 zuul export-keys /var/lib/zuul/zuul-keys-backup.json
+
 - name: Install logging config
   copy:
     src: logging.conf