Add AFS apt mirror
The reprepro class in this is in-tree rather than in its own module purely for ease of getting started. It's also highly hard-coded rather than flexible. This change will need a mirror.apt volume and service/reprepro principal and keytab to be created before it lands. Allow for pool trimming after a 2 hour delay. Each devstack run of apt-get update should be able to be assumed to be valid for the length of the devstack. For that reason, only delete files that are unreferenced during the subsequent mirror run, ensuring at least a 2 hour delay between becoming unreferenced and going away. Local testing indicates that a trusty mirror is 86G. Change-Id: I84f6a0391f80e6bf567c4bfc18a41bd270fe8c01
This commit is contained in:
Monty Taylor
parent
16e99c61ad
commit
f1c86c66a3
@ -531,6 +531,7 @@ node 'mirror-update.openstack.org' {
|
||||
class { 'openstack_project::mirror_update':
|
||||
bandersnatch_keytab => hiera('bandersnatch_keytab'),
|
||||
admin_keytab => hiera('afsadmin_keytab'),
|
||||
reprepro_keytab => hiera('reprepro_keytab'),
|
||||
sysadmins => hiera('sysadmins', []),
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2016 IBM Corp.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -e
|
||||
|
||||
UNREF_FILE=/var/run/reprepro/unreferenced-files
|
||||
|
||||
echo "Obtaining reprepro tokens and running reprepro update"
|
||||
k5start -t -f /etc/reprepro.keytab service/reprepro -- timeout -k 2m 30m reprepro update
|
||||
|
||||
if [ -f $UNREF_FILE ] ; then
|
||||
echo "Cleaning up files made unreferenced on the last run"
|
||||
k5start -t -f /etc/reprepro.keytab service/reprepro -- timeout -k 2m 30m reprepro deleteifunreferenced < $UNREF_FILE
|
||||
fi
|
||||
|
||||
echo "Saving list of newly unreferenced files for next time"
|
||||
reprepro dumpunreferenced > $UNREF_FILE
|
||||
|
||||
echo "Checking state of mirror"
|
||||
reprepro checkpool fast
|
||||
reprepro check
|
||||
|
||||
echo "reprepro completed successfully, running reprepro export."
|
||||
k5start -t -f /etc/afsadmin.keytab service/afsadmin -- vos release -v mirror.apt
|
||||
|
||||
echo "Done."
|
||||
31
modules/openstack_project/files/reprepro/updates
Normal file
31
modules/openstack_project/files/reprepro/updates
Normal file
@ -0,0 +1,31 @@
|
||||
Name: ubuntu
|
||||
Method: http://us.archive.ubuntu.com/ubuntu
|
||||
Components: main universe restricted multiverse
|
||||
UDebComponents: main
|
||||
Architectures: amd64
|
||||
GetInRelease: no
|
||||
VerifyRelease: 437D05B5
|
||||
|
||||
Name: ubuntu-security
|
||||
Method: http://security.ubuntu.com/ubuntu
|
||||
Components: main universe restricted multiverse
|
||||
UDebComponents: main
|
||||
Architectures: amd64
|
||||
GetInRelease: no
|
||||
VerifyRelease: 437D05B5
|
||||
|
||||
Name: debian
|
||||
Method: http://ftp.us.debian.org/debian/
|
||||
Components: main non-free contrib
|
||||
UDebComponents: main
|
||||
Architectures: amd64
|
||||
GetInRelease: no
|
||||
VerifyRelease: 55BE302B
|
||||
|
||||
Name: debian-security
|
||||
Method: http://security.debian.org/
|
||||
Components: main non-free contrib
|
||||
UDebComponents: main
|
||||
Architectures: amd64
|
||||
GetInRelease: no
|
||||
VerifyRelease: 55BE302B
|
||||
@ -3,6 +3,7 @@
|
||||
class openstack_project::mirror_update (
|
||||
$sysadmins = [],
|
||||
$bandersnatch_keytab = '',
|
||||
$reprepro_keytab = '',
|
||||
$admin_keytab = '',
|
||||
) {
|
||||
|
||||
@ -56,4 +57,36 @@ class openstack_project::mirror_update (
|
||||
Class['bandersnatch::mirror']
|
||||
]
|
||||
}
|
||||
|
||||
class { '::openstack_project::reprepro':
|
||||
ubuntu_releases => ['trusty'],
|
||||
}
|
||||
|
||||
file { '/etc/reprepro.keytab':
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0400',
|
||||
content => $reprepro_keytab,
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/reprepro-mirror-update':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/reprepro/reprepro-mirror-update.sh',
|
||||
}
|
||||
|
||||
cron { 'reprepro':
|
||||
user => $user,
|
||||
hour => '*/2',
|
||||
command => 'flock -n /var/run/reprepro/mirror.lock reprepro-mirror-update >>/var/log/reprepro/mirror.log 2>&1',
|
||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
require => [
|
||||
File['/usr/local/bin/reprepro-mirror-update'],
|
||||
File['/etc/afsadmin.keytab'],
|
||||
File['/etc/reprepro.keytab'],
|
||||
Class['::openstack_project::reprepro']
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
52
modules/openstack_project/manifests/reprepro.pp
Normal file
52
modules/openstack_project/manifests/reprepro.pp
Normal file
@ -0,0 +1,52 @@
|
||||
# == Class: openstack_project::mirror_update
|
||||
#
|
||||
class openstack_project::reprepro (
|
||||
$outdir = '/afs/.openstack.org/mirror/apt',
|
||||
$logdir = '/var/log/reprepro',
|
||||
$updates_file = 'puppet:///modules/openstack_project/reprepro/updates',
|
||||
$options_template = 'openstack_project/reprepro/options.erb',
|
||||
$distributions_template = 'openstack_project/reprepro/distributions.erb',
|
||||
$ubuntu_releases = [],
|
||||
$debian_releases = [],
|
||||
) {
|
||||
|
||||
package { 'reprepro':
|
||||
ensure => present,
|
||||
}
|
||||
|
||||
file { $logdir:
|
||||
ensure => directory,
|
||||
}
|
||||
|
||||
file { '/etc/reprepro':
|
||||
ensure => directory,
|
||||
}
|
||||
|
||||
file { '/var/run/reprepro':
|
||||
ensure => directory,
|
||||
}
|
||||
|
||||
file { '/etc/reprepro/updates':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => $updates_file,
|
||||
}
|
||||
|
||||
file { '/etc/reprepro/options':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
content => template($options_template),
|
||||
}
|
||||
|
||||
file { '/etc/reprepro/distributions':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
content => template($distributions_template),
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,83 @@
|
||||
<% @ubuntu_releases.each do |release| -%>
|
||||
Origin: Ubuntu
|
||||
Codename: <%= release %>
|
||||
Description: OpenStack Ubuntu <%= release.capitalize %> mirror
|
||||
Architectures: amd64
|
||||
Components: main universe restricted multiverse
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: ubuntu
|
||||
Log: <%= @logdir %>/ubuntu-<%= release %>.log
|
||||
|
||||
Origin: Ubuntu
|
||||
Codename: <%= release %>-updates
|
||||
Description: OpenStack Ubuntu <%= release.capitalize %> Updates mirror
|
||||
Architectures: amd64
|
||||
Components: main universe restricted multiverse
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: ubuntu
|
||||
Log: <%= @logdir %>/ubuntu-<%= release %>-updates.log
|
||||
|
||||
Origin: Ubuntu
|
||||
Codename: <%= release %>-backports
|
||||
Description: OpenStack Ubuntu <%= release.capitalize %> Backports mirror
|
||||
Architectures: amd64
|
||||
Components: main universe restricted multiverse
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: ubuntu
|
||||
Log: <%= @logdir %>/ubuntu-<%= release %>-backports.log
|
||||
|
||||
Origin: Ubuntu
|
||||
Codename: <%= release %>-security
|
||||
Description: OpenStack Ubuntu <%= release.capitalize %> Security mirror
|
||||
Architectures: amd64
|
||||
Components: main universe restricted multiverse
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: ubuntu-security
|
||||
Log: <%= @logdir %>/ubuntu-<%= release %>-security.log
|
||||
<% end -%>
|
||||
|
||||
<% @debian_releases.each do |release| -%>
|
||||
Origin: Debian
|
||||
Codename: <%= release %>
|
||||
Description: OpenStack Debian <%= release.capitalize %> mirror
|
||||
Architectures: amd64
|
||||
Components: main non-free contrib
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: debian
|
||||
Log: <%= @logdir %>/debian-<%= release %>.log
|
||||
|
||||
Origin: Debian
|
||||
Codename: <%= release %>-updates
|
||||
Description: OpenStack Debian <%= release.capitalize %> Updates mirror
|
||||
Architectures: amd64
|
||||
Components: main non-free contrib
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: debian
|
||||
Log: <%= @logdir %>/debian-<%= release %>-updates.log
|
||||
|
||||
Origin: Debian
|
||||
Codename: <%= release %>-backports
|
||||
Description: OpenStack Debian <%= release.capitalize %> Backports mirror
|
||||
Architectures: amd64
|
||||
Components: main non-free contrib
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: debian
|
||||
Log: <%= @logdir %>/debian-<%= release %>-backports.log
|
||||
|
||||
Origin: Debian
|
||||
Codename: <%= release %>/updates
|
||||
Description: OpenStack Debian <%= release.capitalize %> Security mirror
|
||||
Architectures: amd64
|
||||
Components: main non-free contrib
|
||||
UDebComponents: main
|
||||
Contents: .gz
|
||||
Update: debian-security
|
||||
Log: <%= @logdir %>/debian-<%= release %>-security.log
|
||||
<% end -%>
|
||||
4
modules/openstack_project/templates/reprepro/options.erb
Normal file
4
modules/openstack_project/templates/reprepro/options.erb
Normal file
@ -0,0 +1,4 @@
|
||||
outdir <%= @outdir %>
|
||||
keepunreferencedfiles
|
||||
noskipold
|
||||
export=changed
|
||||
Loading…
x
Reference in New Issue
Block a user