From 008fb47583ef3b13ab07bd709f5e72cb5196fbcb Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Mon, 4 Jun 2012 09:24:07 -0400
Subject: [PATCH] Use pip cache instead of full mirror.
Change-Id: I14ae08eb6c5a402f26de6c20234ac434f8f4b2c9
---
manifests/site.pp | 27 ++++
modules/pypimirror/files/pull-repo.sh | 27 ++++
modules/pypimirror/manifests/init.pp | 115 ++++++++----------
modules/pypimirror/templates/config.erb | 62 ----------
.../pypimirror/templates/run-mirror.sh.erb | 12 ++
5 files changed, 118 insertions(+), 125 deletions(-)
create mode 100644 modules/pypimirror/files/pull-repo.sh
delete mode 100644 modules/pypimirror/templates/config.erb
create mode 100644 modules/pypimirror/templates/run-mirror.sh.erb
diff --git a/manifests/site.pp b/manifests/site.pp
index 6d0657d367..e6f8b03a54 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -376,11 +376,38 @@ node "eavesdrop.openstack.org" {
node "pypi.openstack.org" {
include openstack_cron
+
+ # include jenkins slave so that build deps are there for the pip download
+ class { 'jenkins_slave':
+ ssh_key => "",
+ user => false
+ }
+
class { 'openstack_server':
iptables_public_tcp_ports => [80]
}
+
class { "pypimirror":
base_url => "http://pypi.openstack.org",
+ projects => [
+ 'cinder',
+ 'glance',
+ 'horizon',
+ 'keystone',
+ 'melange',
+ 'nova',
+ 'openstack-common',
+ 'python-cinderclient',
+ 'python-glanceclient',
+ 'python-keystoneclient',
+ 'python-melangeclient',
+ 'python-novaclient',
+ 'python-openstackclient',
+ 'python-quantumclient',
+ 'python-swiftclient',
+ 'quantum',
+ 'swift'
+ ]
}
}
diff --git a/modules/pypimirror/files/pull-repo.sh b/modules/pypimirror/files/pull-repo.sh
new file mode 100644
index 0000000000..9730ff8ac5
--- /dev/null
+++ b/modules/pypimirror/files/pull-repo.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# This file is managed by puppet.
+# https://github.com/openstack/openstack-ci-puppet
+
+export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip}
+export PIP_TEMP_DOWNLOAD=${PIP_TEMP_DOWNLOAD:-/var/lib/pip-download}
+
+project=$1
+pip_command='pip install -M -U -I --exists-action=w --no-install'
+
+cd ${PIP_TEMP_DOWNLOAD}
+if [ ! -d ${project} ] ; then
+ git clone git://github.com/openstack/${project}.git ${project} >/dev/null 2>&1
+fi
+cd ${project}
+git fetch origin
+for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do
+ git reset --hard $branch
+ git clean -x -f -d -q
+ echo "*********************"
+ echo "Fetching pip requires for $project:$branch"
+ for requires_file in tools/pip-requires tools/test-requires ; do
+ if [ -f ${requires_file} ] ; then
+ $pip_command -r $requires_file
+ fi
+ done
+done
diff --git a/modules/pypimirror/manifests/init.pp b/modules/pypimirror/manifests/init.pp
index 5c41172cd7..f0f26da038 100644
--- a/modules/pypimirror/manifests/init.pp
+++ b/modules/pypimirror/manifests/init.pp
@@ -1,87 +1,76 @@
class pypimirror ( $base_url,
$log_filename = "/var/log/pypimirror.log",
$mirror_file_path = "/var/lib/pypimirror",
- $fetch_since_days = 1,
- $package_matches = ["*"],
- $external_links = true,
- $follow_external_index_pages = true )
+ $pip_download = "/var/lib/pip-download",
+ $pip_cache = "/var/cache/pip",
+ $projects = [] )
{
- if $external_links == true {
- $external_links_real = 'True'
- }
- else {
- $external_links_real = 'False'
- }
-
- if $follow_external_index_pages == true {
- $follow_external_index_pages_real = 'True'
- }
- else {
- $follow_external_index_pages_real = 'False'
- }
-
- $packages = [ 'nginx',
- 'python-pip' ]
-
- package { $packages:
+ package { 'nginx':
ensure => present,
}
- # Build the mirror config file based on options provided.
+ package { 'pip':
+ ensure => latest,
+ provider => 'pip',
+ require => Package['python-pip'],
+ }
- file { 'pypimirror.cfg':
- path => '/etc/pypimirror.cfg',
- ensure => present,
- mode => 644,
+ file { '/usr/local/mirror_scripts':
+ ensure => 'directory',
+ mode => 755,
owner => 'root',
group => 'root',
- content => template('pypimirror/config.erb'),
+ }
+
+ file { $pip_download:
+ ensure => 'directory',
+ mode => 755,
+ owner => 'root',
+ group => 'root',
+ }
+
+ file { $pip_cache:
+ ensure => 'directory',
+ mode => 755,
+ owner => 'root',
+ group => 'root',
+ }
+
+ file { '/usr/local/mirror_scripts/run-mirror.sh':
+ ensure => present,
+ mode => 755,
+ owner => 'root',
+ group => 'root',
+ content => template('pypimirror/run-mirror.sh.erb'),
+ require => File['/usr/local/mirror_scripts'],
}
- file { '/usr/local/z3c.pypimirror':
- ensure => absent,
+ file { '/usr/local/mirror_scripts/pull-repo.sh':
+ ensure => present,
+ mode => 755,
+ owner => 'root',
+ group => 'root',
+ source => "puppet:///modules/pypimirror/pull-repo.sh",
+ require => File['/usr/local/mirror_scripts'],
}
- # if we already have the repo the pull updates
-
- exec { "update_pypi_mirror":
- command => "git pull --ff-only",
- cwd => "/usr/local/pypi-mirror",
- path => "/bin:/usr/bin",
- onlyif => "test -d /usr/local/pypi-mirror",
- before => Exec["get_pypi_mirror"],
- }
-
- # otherwise get a new clone of it
-
- exec { "get_pypi_mirror":
- command => "git clone git://github.com/openstack-ci/pypi-mirror.git /usr/local/pypi-mirror",
- path => "/bin:/usr/bin",
- onlyif => "test ! -d /usr/local/pypi-mirror"
- }
-
- exec { "install_pypi_mirror":
- command => "python setup.py install",
- cwd => "/usr/local/pypi-mirror",
- path => "/bin:/usr/bin",
- subscribe => [ Exec["get_pypi_mirror"], Exec["update_pypi_mirror"] ],
- }
-
- exec { "initialize_mirror":
- command => "pypimirror --initial-fetch /etc/pypimirror.cfg",
- path => "/bin:/usr/bin:/usr/local/bin",
- onlyif => "test ! -d ${mirror_file_path}",
- require => [ Exec["get_pypi_mirror"], Exec["install_pypi_mirror"] ],
+ file { '/usr/local/mirror_scripts/process_cache.py':
+ ensure => present,
+ mode => 755,
+ owner => 'root',
+ group => 'root',
+ source => "puppet:///modules/pypimirror/process_cache.py",
+ require => File['/usr/local/mirror_scripts'],
}
# Add cron job to update the mirror
cron { "update_mirror":
user => root,
- hour => 0,
- command => '/usr/local/bin/pypimirror --initial-fetch /etc/pypimirror.cfg',
- require => Exec["install_pypi_mirror"],
+ minute => "0",
+ command => '/usr/local/mirror_scripts/run-mirror.sh',
+ require => File["/usr/local/mirror_scripts/run-mirror.sh"],
}
# Rotate the mirror log file
diff --git a/modules/pypimirror/templates/config.erb b/modules/pypimirror/templates/config.erb
deleted file mode 100644
index 4394a645a0..0000000000
--- a/modules/pypimirror/templates/config.erb
+++ /dev/null
@@ -1,62 +0,0 @@
-[DEFAULT]
-# the root folder of all mirrored packages.
-# if necessary it will be created for you
-mirror_file_path = <%= mirror_file_path %>
-
-# where's your mirror on the net?
-base_url = <%= base_url %>
-
-# lock file to avoid duplicate runs of the mirror script
-lock_file_name = /var/lock/pypi-poll-access.lock
-
-# days to fetch in past on update
-fetch_since_days = <%= fetch_since_days %>
-
-# Pattern for package files, only those matching will be mirrored
-filename_matches =
- *.zip
- *.tgz
- *.egg
- *.tar.gz
- *.tar.bz2
-
-# Pattern for package names; only packages having matching names will
-# be mirrored
-package_matches =
- <% package_matches.each do |match| -%>
- <%= match %>
- <% end -%>
-
-# remove packages not on pypi (or externals) anymore
-cleanup = True
-
-# create index.html files
-create_indexes = True
-
-# be more verbose
-verbose = True
-
-# resolve download_url links on pypi which point to files and download
-# the files from there (if they match filename_matches).
-# The filename and filesize (from the download header) are used
-# to find out if the file is already on the mirror. Not all servers
-# support the content-length header, so be prepared to download
-# a lot of data on each mirror update.
-# This is highly experimental and shouldn't be used right now.
-#
-# NOTE: This option should only be set to True if package_matches is not
-# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL
-# using this option!!!
-external_links = <%= external_links_real %>
-
-# similar to 'external_links' but also follows an index page if no
-# download links are available on the referenced download_url page
-# of a given package.
-#
-# NOTE: This option should only be set to True if package_matches is not
-# set to '*' - otherwise you will mirror a huge amount of data. BE CAREFUL
-# using this option!!!
-follow_external_index_pages = <%= follow_external_index_pages_real %>
-
-# logfile
-log_filename = <%= log_filename %>
diff --git a/modules/pypimirror/templates/run-mirror.sh.erb b/modules/pypimirror/templates/run-mirror.sh.erb
new file mode 100644
index 0000000000..451304dcf9
--- /dev/null
+++ b/modules/pypimirror/templates/run-mirror.sh.erb
@@ -0,0 +1,12 @@
+#!/bin/bash
+# This file is managed by puppet.
+# https://github.com/openstack/openstack-ci-puppet
+
+export PIP_DOWNLOAD_CACHE=<%= pip_cache %>
+export PIP_TEMP_DOWNLOAD=<%= pip_download %>
+export MIRROR_FILE_PATH=<%= mirror_file_path %>
+export LOG_FILENAME=<%= log_filename %>
+<% projects.each do |project| -%>
+/usr/local/mirror_scripts/pull-repo.sh <%= project %> >>$LOG_FILENAME 2>&1
+<% end -%>
+python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}