From f1c86c66a340b957894a36321bc6fc7f5541f833 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 26 Jan 2016 10:19:53 -0500 Subject: [PATCH] Add AFS apt mirror The reprepro class in this is in-tree rather than in its own module purely for ease of getting started. It's also highly hard-coded rather than flexible. This change will need a mirror.apt volume and service/reprepro principal and keytab to be created before it lands. Allow for pool trimming after a 2 hour delay. Each devstack run of apt-get update should be able to be assumed to be valid for the length of the devstack. For that reason, only delete files that are unreferenced during the subsequent mirror run, ensuring at least a 2 hour delay between becoming unreferenced and going away. Local testing indicates that a trusty mirror is 86G. Change-Id: I84f6a0391f80e6bf567c4bfc18a41bd270fe8c01 --- manifests/site.pp | 1 + .../files/reprepro/reprepro-mirror-update.sh | 39 +++++++++ .../openstack_project/files/reprepro/updates | 31 +++++++ .../manifests/mirror_update.pp | 33 ++++++++ .../openstack_project/manifests/reprepro.pp | 52 ++++++++++++ .../templates/reprepro/distributions.erb | 83 +++++++++++++++++++ .../templates/reprepro/options.erb | 4 + 7 files changed, 243 insertions(+) create mode 100644 modules/openstack_project/files/reprepro/reprepro-mirror-update.sh create mode 100644 modules/openstack_project/files/reprepro/updates create mode 100644 modules/openstack_project/manifests/reprepro.pp create mode 100644 modules/openstack_project/templates/reprepro/distributions.erb create mode 100644 modules/openstack_project/templates/reprepro/options.erb diff --git a/manifests/site.pp b/manifests/site.pp index e7ae8661f2..fe913c100c 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -531,6 +531,7 @@ node 'mirror-update.openstack.org' { class { 'openstack_project::mirror_update': bandersnatch_keytab => hiera('bandersnatch_keytab'), admin_keytab => hiera('afsadmin_keytab'), + reprepro_keytab => hiera('reprepro_keytab'), sysadmins => hiera('sysadmins', []), } } diff --git a/modules/openstack_project/files/reprepro/reprepro-mirror-update.sh b/modules/openstack_project/files/reprepro/reprepro-mirror-update.sh new file mode 100644 index 0000000000..75191c021f --- /dev/null +++ b/modules/openstack_project/files/reprepro/reprepro-mirror-update.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright 2016 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -e + +UNREF_FILE=/var/run/reprepro/unreferenced-files + +echo "Obtaining reprepro tokens and running reprepro update" +k5start -t -f /etc/reprepro.keytab service/reprepro -- timeout -k 2m 30m reprepro update + +if [ -f $UNREF_FILE ] ; then + echo "Cleaning up files made unreferenced on the last run" + k5start -t -f /etc/reprepro.keytab service/reprepro -- timeout -k 2m 30m reprepro deleteifunreferenced < $UNREF_FILE +fi + +echo "Saving list of newly unreferenced files for next time" +reprepro dumpunreferenced > $UNREF_FILE + +echo "Checking state of mirror" +reprepro checkpool fast +reprepro check + +echo "reprepro completed successfully, running reprepro export." +k5start -t -f /etc/afsadmin.keytab service/afsadmin -- vos release -v mirror.apt + +echo "Done." diff --git a/modules/openstack_project/files/reprepro/updates b/modules/openstack_project/files/reprepro/updates new file mode 100644 index 0000000000..d9a6d5118e --- /dev/null +++ b/modules/openstack_project/files/reprepro/updates @@ -0,0 +1,31 @@ +Name: ubuntu +Method: http://us.archive.ubuntu.com/ubuntu +Components: main universe restricted multiverse +UDebComponents: main +Architectures: amd64 +GetInRelease: no +VerifyRelease: 437D05B5 + +Name: ubuntu-security +Method: http://security.ubuntu.com/ubuntu +Components: main universe restricted multiverse +UDebComponents: main +Architectures: amd64 +GetInRelease: no +VerifyRelease: 437D05B5 + +Name: debian +Method: http://ftp.us.debian.org/debian/ +Components: main non-free contrib +UDebComponents: main +Architectures: amd64 +GetInRelease: no +VerifyRelease: 55BE302B + +Name: debian-security +Method: http://security.debian.org/ +Components: main non-free contrib +UDebComponents: main +Architectures: amd64 +GetInRelease: no +VerifyRelease: 55BE302B diff --git a/modules/openstack_project/manifests/mirror_update.pp b/modules/openstack_project/manifests/mirror_update.pp index 359d355261..58ad02c490 100644 --- a/modules/openstack_project/manifests/mirror_update.pp +++ b/modules/openstack_project/manifests/mirror_update.pp @@ -3,6 +3,7 @@ class openstack_project::mirror_update ( $sysadmins = [], $bandersnatch_keytab = '', + $reprepro_keytab = '', $admin_keytab = '', ) { @@ -56,4 +57,36 @@ class openstack_project::mirror_update ( Class['bandersnatch::mirror'] ] } + + class { '::openstack_project::reprepro': + ubuntu_releases => ['trusty'], + } + + file { '/etc/reprepro.keytab': + owner => 'root', + group => 'root', + mode => '0400', + content => $reprepro_keytab, + } + + file { '/usr/local/bin/reprepro-mirror-update': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/openstack_project/reprepro/reprepro-mirror-update.sh', + } + + cron { 'reprepro': + user => $user, + hour => '*/2', + command => 'flock -n /var/run/reprepro/mirror.lock reprepro-mirror-update >>/var/log/reprepro/mirror.log 2>&1', + environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', + require => [ + File['/usr/local/bin/reprepro-mirror-update'], + File['/etc/afsadmin.keytab'], + File['/etc/reprepro.keytab'], + Class['::openstack_project::reprepro'] + ] + } } diff --git a/modules/openstack_project/manifests/reprepro.pp b/modules/openstack_project/manifests/reprepro.pp new file mode 100644 index 0000000000..ec7a067cff --- /dev/null +++ b/modules/openstack_project/manifests/reprepro.pp @@ -0,0 +1,52 @@ +# == Class: openstack_project::mirror_update +# +class openstack_project::reprepro ( + $outdir = '/afs/.openstack.org/mirror/apt', + $logdir = '/var/log/reprepro', + $updates_file = 'puppet:///modules/openstack_project/reprepro/updates', + $options_template = 'openstack_project/reprepro/options.erb', + $distributions_template = 'openstack_project/reprepro/distributions.erb', + $ubuntu_releases = [], + $debian_releases = [], +) { + + package { 'reprepro': + ensure => present, + } + + file { $logdir: + ensure => directory, + } + + file { '/etc/reprepro': + ensure => directory, + } + + file { '/var/run/reprepro': + ensure => directory, + } + + file { '/etc/reprepro/updates': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => $updates_file, + } + + file { '/etc/reprepro/options': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + content => template($options_template), + } + + file { '/etc/reprepro/distributions': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + content => template($distributions_template), + } +} diff --git a/modules/openstack_project/templates/reprepro/distributions.erb b/modules/openstack_project/templates/reprepro/distributions.erb new file mode 100644 index 0000000000..49a279a475 --- /dev/null +++ b/modules/openstack_project/templates/reprepro/distributions.erb @@ -0,0 +1,83 @@ +<% @ubuntu_releases.each do |release| -%> +Origin: Ubuntu +Codename: <%= release %> +Description: OpenStack Ubuntu <%= release.capitalize %> mirror +Architectures: amd64 +Components: main universe restricted multiverse +UDebComponents: main +Contents: .gz +Update: ubuntu +Log: <%= @logdir %>/ubuntu-<%= release %>.log + +Origin: Ubuntu +Codename: <%= release %>-updates +Description: OpenStack Ubuntu <%= release.capitalize %> Updates mirror +Architectures: amd64 +Components: main universe restricted multiverse +UDebComponents: main +Contents: .gz +Update: ubuntu +Log: <%= @logdir %>/ubuntu-<%= release %>-updates.log + +Origin: Ubuntu +Codename: <%= release %>-backports +Description: OpenStack Ubuntu <%= release.capitalize %> Backports mirror +Architectures: amd64 +Components: main universe restricted multiverse +UDebComponents: main +Contents: .gz +Update: ubuntu +Log: <%= @logdir %>/ubuntu-<%= release %>-backports.log + +Origin: Ubuntu +Codename: <%= release %>-security +Description: OpenStack Ubuntu <%= release.capitalize %> Security mirror +Architectures: amd64 +Components: main universe restricted multiverse +UDebComponents: main +Contents: .gz +Update: ubuntu-security +Log: <%= @logdir %>/ubuntu-<%= release %>-security.log +<% end -%> + +<% @debian_releases.each do |release| -%> +Origin: Debian +Codename: <%= release %> +Description: OpenStack Debian <%= release.capitalize %> mirror +Architectures: amd64 +Components: main non-free contrib +UDebComponents: main +Contents: .gz +Update: debian +Log: <%= @logdir %>/debian-<%= release %>.log + +Origin: Debian +Codename: <%= release %>-updates +Description: OpenStack Debian <%= release.capitalize %> Updates mirror +Architectures: amd64 +Components: main non-free contrib +UDebComponents: main +Contents: .gz +Update: debian +Log: <%= @logdir %>/debian-<%= release %>-updates.log + +Origin: Debian +Codename: <%= release %>-backports +Description: OpenStack Debian <%= release.capitalize %> Backports mirror +Architectures: amd64 +Components: main non-free contrib +UDebComponents: main +Contents: .gz +Update: debian +Log: <%= @logdir %>/debian-<%= release %>-backports.log + +Origin: Debian +Codename: <%= release %>/updates +Description: OpenStack Debian <%= release.capitalize %> Security mirror +Architectures: amd64 +Components: main non-free contrib +UDebComponents: main +Contents: .gz +Update: debian-security +Log: <%= @logdir %>/debian-<%= release %>-security.log +<% end -%> diff --git a/modules/openstack_project/templates/reprepro/options.erb b/modules/openstack_project/templates/reprepro/options.erb new file mode 100644 index 0000000000..7b22020993 --- /dev/null +++ b/modules/openstack_project/templates/reprepro/options.erb @@ -0,0 +1,4 @@ +outdir <%= @outdir %> +keepunreferencedfiles +noskipold +export=changed