mirror-update: export mirroring logs

This adds a periodic job to copy logs to a mirror volume, and export
it via the usual mirror http.

I have precreated the log volume; just as a R/W volume because this is
expected to be very low volume access.

Change-Id: I67870f6d439af2d2a63a5048ef52cecff3e75275

playbooks/roles/mirror-update/files/publish-mirror-logs

@@ -0,0 +1,24 @@
+#!/bin/bash -xe
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+DEST="/afs/.openstack.org/mirror/logs/"
+
+K5START="k5start -t -f /etc/logs.keytab service/logs-mirror --"
+RSYNC="rsync -avz --no-perms --no-owner --no-group"
+
+$K5START $RSYNC /var/log/rsync-mirrors $DEST
+
+# NOTE(ianw) : r/w volume as this is infrequently accessed; thus no
+# replications and no "vos release" etc required.

playbooks/roles/mirror-update/tasks/log_publish.yaml

@@ -0,0 +1,32 @@
+- name: Install log publisher keytab
+  shell: 'echo "{{ mirror_update_keytab_logs }}" | base64 -d > /etc/logs.keytab'
+  args:
+    creates: /etc/logs.keytab
+  no_log: True
+
+- name: Ensure permissions on log publisher keytab
+  file:
+    path: '/etc/logs.keytab'
+    owner: root
+    group: root
+    mode: '0400'
+
+- name: Copy log publisher script
+  copy:
+    src: 'publish-mirror-logs'
+    dest: '/usr/local/bin/publish-mirror-logs'
+    mode: '0755'
+
+- name: Install log publisher cron job
+  cron:
+    name: 'Publish mirror logs'
+    state: present
+    job: 'publish-mirror-logs >> /var/log/publish-mirror-logs.log 2>&1'
+    hour: '*'
+    minute: '30'
+
+- name: Install logrotate rules
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: '/var/log/publish-mirror-logs.log'

playbooks/roles/mirror-update/tasks/main.yaml

@@ -27,6 +27,9 @@
     env: yes
     value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
+- name: Setup log publisher script
+  include_tasks: log_publish.yaml
+
 - name: Setup rsync mirror scripts
   include_tasks: rsync.yaml
 

playbooks/roles/mirror/tasks/main.yaml

@@ -88,6 +88,7 @@
     - debian-openstack
     - epel
     - fedora
+    - logs
     - opensuse
     - ubuntu
     - ubuntu-ports

playbooks/zuul/templates/host_vars/mirror-update01.opendev.org.yaml.j2

@@ -1,6 +1,9 @@
 mirror_update_keytab_afsadmin: |-
   aEkRPhZllm2F2y71Zgf3X9NjyHT7/sS8bd/vXt9oG1PKkUmpeBXprFnrxzMuKiupHwwTa09w5LuB
   blLvBOC8W5Miz1u6TkRe+/jLQurLpzYHwk3bJCJ6s3WwPKDej54TDVgrVQ==
+mirror_update_keytab_logs: |-
+  aEkRPhZllm2F2y71Zgf3X9NjyHT7/sS8bd/vXt9oG1PKkUmpeBXprFnrxzMuKiupHwwTa09w5LuB
+  blLvBOC8W5Miz1u6TkRe+/jLQurLpzYHwk3bJCJ6s3WwPKDej54TDVgrVQ==
 mirror_update_keytab_centos: |-
   aEkRPhZllm2F2y71Zgf3X9NjyHT7/sS8bd/vXt9oG1PKkUmpeBXprFnrxzMuKiupHwwTa09w5LuB
   blLvBOC8W5Miz1u6TkRe+/jLQurLpzYHwk3bJCJ6s3WwPKDej54TDVgrVQ==

testinfra/test_mirror-update.py

@@ -35,8 +35,13 @@ def test_rsync_scripts(host):
         f = host.file('/usr/local/bin/%s-mirror-update' % script)
         assert f.exists
 
+def test_publisher_script(host):
+    f = host.file('/usr/local/bin/publish-mirror-logs')
+    assert f.exists
+
 def test_keytabs(host):
     for keytab in ['/etc/afsadmin.keytab',
+                   '/etc/logs.keytab',
                    '/etc/centos.keytab',
                    '/etc/epel.keytab',
                    '/etc/fedora.keytab',