Merge "Add AFS apt mirror"

manifests/site.pp

@@ -533,6 +533,7 @@ node 'mirror-update.openstack.org' {
   class { 'openstack_project::mirror_update':
     bandersnatch_keytab => hiera('bandersnatch_keytab'),
     admin_keytab        => hiera('afsadmin_keytab'),
+    reprepro_keytab     => hiera('reprepro_keytab'),
     sysadmins           => hiera('sysadmins', []),
   }
 }

modules/openstack_project/files/reprepro/reprepro-mirror-update.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2016 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+UNREF_FILE=/var/run/reprepro/unreferenced-files
+
+echo "Obtaining reprepro tokens and running reprepro update"
+k5start -t -f /etc/reprepro.keytab service/reprepro -- timeout -k 2m 30m reprepro update
+
+if [ -f $UNREF_FILE ] ; then
+    echo "Cleaning up files made unreferenced on the last run"
+    k5start -t -f /etc/reprepro.keytab service/reprepro -- timeout -k 2m 30m reprepro deleteifunreferenced < $UNREF_FILE
+fi
+
+echo "Saving list of newly unreferenced files for next time"
+reprepro dumpunreferenced > $UNREF_FILE
+
+echo "Checking state of mirror"
+reprepro checkpool fast
+reprepro check
+
+echo "reprepro completed successfully, running reprepro export."
+k5start -t -f /etc/afsadmin.keytab service/afsadmin -- vos release -v mirror.apt
+
+echo "Done."

modules/openstack_project/files/reprepro/updates

@@ -0,0 +1,31 @@
+Name: ubuntu
+Method: http://us.archive.ubuntu.com/ubuntu
+Components: main universe restricted multiverse
+UDebComponents: main
+Architectures: amd64
+GetInRelease: no
+VerifyRelease: 437D05B5
+
+Name: ubuntu-security
+Method: http://security.ubuntu.com/ubuntu
+Components: main universe restricted multiverse
+UDebComponents: main
+Architectures: amd64
+GetInRelease: no
+VerifyRelease: 437D05B5
+
+Name: debian
+Method: http://ftp.us.debian.org/debian/
+Components: main non-free contrib
+UDebComponents: main
+Architectures: amd64
+GetInRelease: no
+VerifyRelease: 55BE302B
+
+Name: debian-security
+Method: http://security.debian.org/
+Components: main non-free contrib
+UDebComponents: main
+Architectures: amd64
+GetInRelease: no
+VerifyRelease: 55BE302B

modules/openstack_project/manifests/mirror_update.pp

@@ -3,6 +3,7 @@
 class openstack_project::mirror_update (
   $sysadmins = [],
   $bandersnatch_keytab = '',
+  $reprepro_keytab = '',
   $admin_keytab = '',
 ) {
 
@@ -56,4 +57,36 @@ class openstack_project::mirror_update (
        Class['bandersnatch::mirror']
     ]
   }
+
+  class { '::openstack_project::reprepro':
+    ubuntu_releases => ['trusty'],
+  }
+
+  file { '/etc/reprepro.keytab':
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0400',
+    content => $reprepro_keytab,
+  }
+
+  file { '/usr/local/bin/reprepro-mirror-update':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    source  => 'puppet:///modules/openstack_project/reprepro/reprepro-mirror-update.sh',
+  }
+
+  cron { 'reprepro':
+    user        => $user,
+    hour        => '*/2',
+    command     => 'flock -n /var/run/reprepro/mirror.lock reprepro-mirror-update >>/var/log/reprepro/mirror.log 2>&1',
+    environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
+    require     => [
+       File['/usr/local/bin/reprepro-mirror-update'],
+       File['/etc/afsadmin.keytab'],
+       File['/etc/reprepro.keytab'],
+       Class['::openstack_project::reprepro']
+    ]
+  }
 }

modules/openstack_project/manifests/reprepro.pp

@@ -0,0 +1,52 @@
+# == Class: openstack_project::mirror_update
+#
+class openstack_project::reprepro (
+  $outdir = '/afs/.openstack.org/mirror/apt',
+  $logdir = '/var/log/reprepro',
+  $updates_file = 'puppet:///modules/openstack_project/reprepro/updates',
+  $options_template = 'openstack_project/reprepro/options.erb',
+  $distributions_template = 'openstack_project/reprepro/distributions.erb',
+  $ubuntu_releases = [],
+  $debian_releases = [],
+) {
+
+  package { 'reprepro':
+    ensure => present,
+  }
+
+  file { $logdir:
+    ensure => directory,
+  }
+
+  file { '/etc/reprepro':
+    ensure => directory,
+  }
+
+  file { '/var/run/reprepro':
+    ensure => directory,
+  }
+
+  file { '/etc/reprepro/updates':
+    ensure => present,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0755',
+    source => $updates_file,
+  }
+
+  file { '/etc/reprepro/options':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template($options_template),
+  }
+
+  file { '/etc/reprepro/distributions':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template($distributions_template),
+  }
+}

modules/openstack_project/templates/reprepro/distributions.erb

@@ -0,0 +1,83 @@
+<% @ubuntu_releases.each do |release| -%>
+Origin: Ubuntu
+Codename: <%= release %>
+Description: OpenStack Ubuntu <%= release.capitalize %> mirror
+Architectures: amd64
+Components: main universe restricted multiverse
+UDebComponents: main
+Contents: .gz
+Update: ubuntu
+Log: <%= @logdir %>/ubuntu-<%= release %>.log
+
+Origin: Ubuntu
+Codename: <%= release %>-updates
+Description: OpenStack Ubuntu <%= release.capitalize %> Updates mirror
+Architectures: amd64
+Components: main universe restricted multiverse
+UDebComponents: main
+Contents: .gz
+Update: ubuntu
+Log: <%= @logdir %>/ubuntu-<%= release %>-updates.log
+
+Origin: Ubuntu
+Codename: <%= release %>-backports
+Description: OpenStack Ubuntu <%= release.capitalize %> Backports mirror
+Architectures: amd64
+Components: main universe restricted multiverse
+UDebComponents: main
+Contents: .gz
+Update: ubuntu
+Log: <%= @logdir %>/ubuntu-<%= release %>-backports.log
+
+Origin: Ubuntu
+Codename: <%= release %>-security
+Description: OpenStack Ubuntu <%= release.capitalize %> Security mirror
+Architectures: amd64
+Components: main universe restricted multiverse
+UDebComponents: main
+Contents: .gz
+Update: ubuntu-security
+Log: <%= @logdir %>/ubuntu-<%= release %>-security.log
+<% end -%>
+
+<% @debian_releases.each do |release| -%>
+Origin: Debian
+Codename: <%= release %>
+Description: OpenStack Debian <%= release.capitalize %> mirror
+Architectures: amd64
+Components: main non-free contrib
+UDebComponents: main
+Contents: .gz
+Update: debian
+Log: <%= @logdir %>/debian-<%= release %>.log
+
+Origin: Debian
+Codename: <%= release %>-updates
+Description: OpenStack Debian <%= release.capitalize %> Updates mirror
+Architectures: amd64
+Components: main non-free contrib
+UDebComponents: main
+Contents: .gz
+Update: debian
+Log: <%= @logdir %>/debian-<%= release %>-updates.log
+
+Origin: Debian
+Codename: <%= release %>-backports
+Description: OpenStack Debian <%= release.capitalize %> Backports mirror
+Architectures: amd64
+Components: main non-free contrib
+UDebComponents: main
+Contents: .gz
+Update: debian
+Log: <%= @logdir %>/debian-<%= release %>-backports.log
+
+Origin: Debian
+Codename: <%= release %>/updates
+Description: OpenStack Debian <%= release.capitalize %> Security mirror
+Architectures: amd64
+Components: main non-free contrib
+UDebComponents: main
+Contents: .gz
+Update: debian-security
+Log: <%= @logdir %>/debian-<%= release %>-security.log
+<% end -%>

modules/openstack_project/templates/reprepro/options.erb

@@ -0,0 +1,4 @@
+outdir <%= @outdir %>
+keepunreferencedfiles
+noskipold
+export=changed