From 19e278dfb74fe2b9204b09e0d34095b5c14d10c9 Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Sun, 25 May 2014 11:01:12 -0400
Subject: [PATCH] Add support for disk-image-builder in nodepool
In order to use disk-image-builder to build nodepool images, we need
a few things. First, we need additional debs to be installed for
disk-image-builder to work. Then we need the image elements themselves
to describe the images we want to create. Also, a few helper scripts
for locally working with the disk images have been included.
Tested with build-image.sh script; devstack-gate-trusty.qcow2 created
correctly
Change-Id: I539743147341dc5b387d103fecd3e9ff0f01fdfc
---
.../files/nodepool/elements/README.rst | 54 +++++
.../elements/cache-devstack/README.rst | 1 +
.../elements/cache-devstack/element-deps | 1 +
.../extra-data.d/50-early-source-repo | 209 ++++++++++++++++++
.../extra-data.d/55-cache-devstack-repos | 185 ++++++++++++++++
.../extra-data.d/60-rm-early-source-repo | 25 +++
.../cache-devstack/install.d/50-download-pkgs | 30 +++
.../elements/node-devstack/README.rst | 1 +
.../elements/node-devstack/element-deps | 2 +
.../node-devstack/install.d/20-prepare-node | 28 +++
.../elements/nodepool-base/README.rst | 1 +
.../exta-data.d/50-copy-nodepool-scripts | 24 ++
.../nodepool-base/finalise.d/99-unbound | 54 +++++
.../nodepool-base/install.d/05-record-details | 18 ++
.../install.d/50-restrict-memory | 31 +++
.../nodepool-base/install.d/99-install-zuul | 22 ++
.../elements/openstack-repos/README.rst | 1 +
.../elements/openstack-repos/element-deps | 3 +
.../extra-data.d/50-create-repo-list | 55 +++++
.../install.d/95-chown-jenkins | 20 ++
.../files/nodepool/elements/puppet/README.rst | 1 +
.../nodepool/elements/puppet/bin/prepare-node | 55 +++++
.../nodepool/elements/puppet/element-deps | 3 +
.../elements/puppet/install.d/05-puppet | 27 +++
.../elements/puppet/install.d/95-clean-repos | 37 ++++
.../elements/puppet/install.d/96-clean-cron | 23 ++
.../elements/puppet/pre-install.d/10-preseed | 39 ++++
.../nodepool/elements/slave-db/README.rst | 1 +
.../manifests/nodepool_prod.pp | 12 +
.../manifests/slave_common.pp | 23 ++
.../templates/nodepool/nodepool.yaml.erb | 2 +
tools/build-dib-in-docker.sh | 16 ++
tools/build-image.sh | 28 +++
tools/mount-image.sh | 25 +++
tools/umount-image.sh | 19 ++
35 files changed, 1076 insertions(+)
create mode 100644 modules/openstack_project/files/nodepool/elements/README.rst
create mode 100644 modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst
create mode 100644 modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps
create mode 100755 modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo
create mode 100755 modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos
create mode 100755 modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo
create mode 100644 modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs
create mode 100644 modules/openstack_project/files/nodepool/elements/node-devstack/README.rst
create mode 100644 modules/openstack_project/files/nodepool/elements/node-devstack/element-deps
create mode 100755 modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node
create mode 100644 modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst
create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts
create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound
create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details
create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory
create mode 100755 modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul
create mode 100644 modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst
create mode 100644 modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps
create mode 100755 modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list
create mode 100755 modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins
create mode 100644 modules/openstack_project/files/nodepool/elements/puppet/README.rst
create mode 100644 modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node
create mode 100644 modules/openstack_project/files/nodepool/elements/puppet/element-deps
create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet
create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos
create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron
create mode 100755 modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed
create mode 100644 modules/openstack_project/files/nodepool/elements/slave-db/README.rst
create mode 100644 tools/build-dib-in-docker.sh
create mode 100755 tools/build-image.sh
create mode 100644 tools/mount-image.sh
create mode 100644 tools/umount-image.sh
diff --git a/modules/openstack_project/files/nodepool/elements/README.rst b/modules/openstack_project/files/nodepool/elements/README.rst
new file mode 100644
index 0000000000..cb98263bcc
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/README.rst
@@ -0,0 +1,54 @@
+Using diskimage-builder to build devstack-gate nodes
+====================================================
+
+In addition to being able to just download and consume images that are the
+same as what run devstack-gate, it's easy to make your own for local dev or
+testing - or just for fun.
+
+Install diskimage-builder
+-------------------------
+
+Install the dependencies:
+
+::
+
+ sudo apt-get install kpartx qemu-utils curl
+
+Install diskimage-builder:
+
+::
+
+ sudo pip install diskimage-builder
+
+
+Build an image
+--------------
+
+Building an image is simple, we have a script!
+
+::
+
+ bash tools/build-image.sh
+
+You should be left with a file called devstack-gate-precise.qcow2.
+
+Mounting the image
+------------------
+
+If you would like to examine the contents of the image, you can mount it on
+a loopback device using qemu-nbd.
+
+::
+
+ sudo apt-get install qemu-utils
+ sudo modprobe nbd max_part=16
+ sudo mkdir -p /tmp/newimage
+ sudo qemu-nbd -c /dev/nbd1 devstack-gate-precise.qcow2
+ sudo mount /dev/nbd1 /tmp/newimage
+
+Other things
+------------
+
+It's a qcow2 image, so you can do tons of things with it. You can upload it
+to glance, you can boot it using kvm, and you can even copy it to a cloud
+server, replace the contents of the server with it and kexec the new kernel.
diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst b/modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst
new file mode 100644
index 0000000000..1a568aceed
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/README.rst
@@ -0,0 +1 @@
+Pre-cache all of the things devstack might need
diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps b/modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps
new file mode 100644
index 0000000000..7b2a984ae5
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/element-deps
@@ -0,0 +1 @@
+openstack-repos
diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo
new file mode 100755
index 0000000000..c6ef8c2d12
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/50-early-source-repo
@@ -0,0 +1,209 @@
+#!/bin/bash
+
+# Copyright (c) 2012-2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We need a copy of the devstack repo so that we can read it so that
+# we can generate the list of repos that we need to get. We'd like to
+# use the normal repo fetching and caching routines for that, so use
+# a modified version.
+
+# The bulk of this code is copied from
+# elements/source-repositories/extra-data.d/98-source-repositories
+# Most of it should be removed when we can source and call the
+# functions directly
+
+set -eu
+
+# If the old cache exists, move it to the new name
+function make_new_cache(){
+ local OLD_CACHE_BASE=$1
+ local CACHE_BASE=$2
+
+ # If the old cache name exists, move it to the new cache name
+ if [ -e "$OLD_CACHE_BASE" ] ; then
+ if [ ! -e "$CACHE_BASE" ] ; then
+ mv -n $OLD_CACHE_BASE $CACHE_BASE
+ else
+ echo "Not replacing new cache location with old cache"
+ fi
+ fi
+}
+
+# Gets repositories or individual files listed in the a repository file
+# and places them in the specified destination path.
+# The format of the repository file is one or more lines matching
+# <name> <type> <destination> <location> [<ref>]
+function get_repos_for_element(){
+ local REPO_SOURCES=$1
+ local CACHE_URL=$TMP_HOOKS_PATH/bin/cache-url
+
+ local REGEX="^([^ ]+) (git|tar|file|package) ?(/[^ ]+)? ?([^ ]+)? ?([^ ]*)$"
+
+ while read line; do
+ # expand variables
+ line=$(eval echo $line)
+
+ # ignore blank lines and lines beginning in '#'
+ [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
+
+ if [[ "$line" =~ $REGEX ]] ; then
+ local REPONAME=${BASH_REMATCH[1]}
+ local REPOTYPE=${BASH_REMATCH[2]}
+ local REPOPATH=${BASH_REMATCH[3]}
+ local REPOLOCATION=${BASH_REMATCH[4]}
+ local REPO_ORIG_LOCATION=$REPOLOCATION
+ local REPOREF=${BASH_REMATCH[5]:-master}
+
+ local REPO_DEST=$TMP_MOUNT_PATH$REPOPATH
+ local REPO_SUB_DIRECTORY=$(dirname $REPO_DEST)
+
+ # REPOTYPE can be overridden with DIB_REPOTYPE_{name}
+ local REPOTYPE_OVERRIDE=DIB_REPOTYPE_${REPONAME//[^A-Za-z0-9]/_}
+ REPOTYPE=${!REPOTYPE_OVERRIDE:-$REPOTYPE}
+
+ # REPOLOCATION can be overridden with DIB_REPOLOCATION_{name}
+ local REPOLOCATION_OVERRIDE=DIB_REPOLOCATION_${REPONAME//[^A-Za-z0-9]/_}
+ REPOLOCATION=${!REPOLOCATION_OVERRIDE:-$REPOLOCATION}
+
+ # REPOREF can be overridden with DIB_REPOREF_{name}
+ local REPOREF_OVERRIDE=DIB_REPOREF_${REPONAME//[^A-Za-z0-9]/_}
+ REPOREF=${!REPOREF_OVERRIDE:-$REPOREF}
+
+ # Determine a unique cache path for this repo
+ CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
+ OLD_CACHE_PATH=${CACHE_BASE}/${CACHE_NAME}
+ # Add the repo name to the sha1sum for readability
+ CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME}
+ CACHE_PATH=${CACHE_BASE}/$CACHE_NAME
+ make_new_cache $OLD_CACHE_PATH $CACHE_PATH
+
+ # Return if install type is not source
+ local INSTALL_TYPE_VAR=DIB_INSTALLTYPE_${REPONAME//[^A-Za-z0-9]/_}
+ local INSTALL_TYPE=${!INSTALL_TYPE_VAR:-source}
+ if [ ! $INSTALL_TYPE = "source" ]; then
+ echo "$REPONAME install type not set to source"
+ continue
+ fi
+
+ case $REPOTYPE in
+ git)
+ if [ -z "${!REPOLOCATION_OVERRIDE:-""}" -a -n "${DIB_GITREPOBASE:-""}" ] ; then
+ # Transform the current repo base to the new one
+ local NEW_REPOLOCATION=$(echo $REPOLOCATION |\
+ sed "s,^[^:]\+://[^/]\+/\(~[^/]\+\)\?\(.*\)$,${DIB_GITREPOBASE}\2,g")
+ echo "Transformed ${REPOLOCATION} to ${NEW_REPOLOCATION}"
+ REPOLOCATION=$NEW_REPOLOCATION
+ # Also update the cache location
+ CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
+ CACHE_PATH=~/.cache/image-create/repository-sources/$CACHE_NAME
+ fi
+ sudo mkdir -p $REPO_SUB_DIRECTORY
+
+ if [ ! -e "$CACHE_PATH" ] ; then
+ echo "Caching $REPONAME from $REPOLOCATION in $CACHE_PATH"
+ git clone $REPOLOCATION $CACHE_PATH.tmp
+ mv ${CACHE_PATH}{.tmp,}
+ fi
+
+ HAS_REF=$(git --git-dir=$CACHE_PATH/.git name-rev $REPOREF 2>/dev/null || true)
+ if [ -z "$DIB_OFFLINE" -o -z "$HAS_REF" ] ; then
+ echo "Updating cache of $REPOLOCATION in $CACHE_PATH with ref $REPOREF"
+ git --git-dir=$CACHE_PATH/.git fetch --update-head-ok $REPOLOCATION +refs/heads/*:refs/heads/*
+ fi
+
+ echo "Cloning from $REPONAME cache and applying ref $REPOREF"
+ sudo git clone $CACHE_PATH $REPO_DEST
+ pushd $REPO_DEST
+ sudo git fetch $CACHE_PATH $REPOREF
+ sudo git reset --hard FETCH_HEAD
+ # Get the reference in use
+ git_ref=$(git rev-parse FETCH_HEAD)
+ popd
+
+ # Write the reference being used into the source-repositories manifest
+ echo "$REPONAME git $REPOPATH $REPOLOCATION $git_ref" >> $GIT_MANIFEST
+ ;;
+ tar)
+ # The top level directory of the tarball mightn't have a fixed name i.e.
+ # it could contain version numbers etc... so we write it to a tmpdir
+ # the then move the contents into the directory we want it in, this does
+ # assume the tarball only contains a single top level directory
+ local tmpdir=$(mktemp --tmpdir=$TMP_MOUNT_PATH/tmp -d)
+ if [ -n "$CACHE_PATH" ] ; then
+ echo "Caching $REPONAME tarball from $REPOLOCATION in $CACHE_PATH"
+ if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
+ $CACHE_URL $REPOLOCATION $CACHE_PATH
+ fi
+ tar -C $tmpdir -xzf $CACHE_PATH
+ else
+ echo "Fetching $REPONAME tarball from $REPOLOCATION"
+ curl $REPOLOCATION | tar -C $tmpdir -xzf -
+ fi
+ sudo mkdir -p $REPO_DEST
+ sudo mv $tmpdir/*/* $REPO_DEST
+ rm -rf $tmpdir
+ ;;
+ file)
+ sudo mkdir -p $REPO_SUB_DIRECTORY
+ if [ -n "$CACHE_PATH" ] ; then
+ echo "Caching $REPONAME file from $REPOLOCATION in $CACHE_PATH"
+ if [ ! -f "$CACHE_PATH" -o -z "$DIB_OFFLINE" ] ; then
+ $CACHE_URL $REPOLOCATION $CACHE_PATH
+ fi
+ sudo cp $CACHE_PATH $REPO_DEST
+ else
+ echo "Fetching $REPONAME file from $REPOLOCATION"
+ sudo curl $REPOLOCATION -o $REPO_DEST
+ fi
+ ;;
+ *)
+ echo "Unsupported repository type: $REPOTYPE"
+ return 1
+ ;;
+ esac
+
+ # Capture the in-instance repository path for later review / other
+ # elements (like a pypi dependency cache).
+ echo "$REPOPATH" | sudo dd of=$TMP_MOUNT_PATH/etc/dib-source-repositories oflag=append conv=notrunc
+
+ else
+ echo "Couldn't parse '$line' as a source repository"
+ return 1
+ fi
+ done < $REPO_SOURCES
+}
+
+CACHE_BASE=~/.cache/image-create/source-repositories
+OLD_CACHE_BASE=~/.cache/image-create/repository-sources
+make_new_cache $OLD_CACHE_BASE $CACHE_BASE
+mkdir -p $CACHE_BASE
+# Use the IMAGE_NAME from the calling script, and make it unique with the temporary path
+GIT_MANIFEST_NAME=dib-manifest-git-$(basename ${IMAGE_NAME})
+GIT_MANIFEST_CACHE_NAME=${GIT_MANIFEST_NAME}_$(dirname ${TMP_MOUNT_PATH##*.})
+GIT_MANIFEST=$CACHE_BASE/${GIT_MANIFEST_CACHE_NAME}
+rm -f $GIT_MANIFEST
+
+# Get source repositories for the target
+echo "devstack git /opt/git/openstack-dev/devstack git://git.openstack.org/openstack-dev/devstack.git" > $TMP_HOOKS_PATH/early-source-repository-config
+for _SOURCEREPO in $(find $TMP_HOOKS_PATH -maxdepth 1 -name "early-source-repository-*" -not -name '*~'); do
+ get_repos_for_element $_SOURCEREPO
+done
+
+# Copy the manifest into the image if it exists (there may be no git repositories used)
+if [ -e "$GIT_MANIFEST" ] ; then
+ sudo cp $GIT_MANIFEST $TMP_MOUNT_PATH/${DIB_MANIFEST_IMAGE_DIR}/$GIT_MANIFEST_NAME
+fi
diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos
new file mode 100755
index 0000000000..bb2afee4de
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+
+
+RELEASE = os.environ['DIB_RELEASE']
+TMP_MOUNT_PATH = os.environ['TMP_MOUNT_PATH']
+TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH']
+
+DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
+CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp')
+IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
+
+
+def run_local(cmd, status=False, cwd='.', env={}):
+ print "Running:", cmd
+ newenv = os.environ
+ newenv.update(env)
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
+ stderr=subprocess.STDOUT, env=newenv)
+ (out, nothing) = p.communicate()
+ if status:
+ return (p.returncode, out.strip())
+ return out.strip()
+
+
+def git_branches():
+ branches = []
+ for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
+ branch = branch.strip()
+ if not branch.startswith('remotes/origin'):
+ continue
+ branches.append(branch)
+ return branches
+
+
+def tokenize(fn, tokens, distribution, comment=None):
+ for line in open(fn):
+ if 'dist:' in line and ('dist:%s' % distribution not in line):
+ continue
+ if 'qpid' in line:
+ continue # TODO: explain why this is here
+ if comment and comment in line:
+ line = line[:line.rfind(comment)]
+ line = line.strip()
+ if line and line not in tokens:
+ tokens.append(line)
+
+
+def _legacy_find_images(basedir):
+ """Divine what images we should use based on parsing stackrc."""
+ images = []
+ for line in open(os.path.join(basedir, 'stackrc')):
+ line = line.strip()
+ if line.startswith('IMAGE_URLS'):
+ if '#' in line:
+ line = line[:line.rfind('#')]
+ if line.endswith(';;'):
+ line = line[:-2]
+ line = line.split('=', 1)[1].strip()
+ if line.startswith('${IMAGE_URLS:-'):
+ line = line[len('${IMAGE_URLS:-'):]
+ if line.endswith('}'):
+ line = line[:-1]
+ if not line:
+ continue
+ if line[0] == line[-1] == '"':
+ line = line[1:-1]
+ # Add image to the list to be downloaded, but
+ # skip downloading giant vmware images
+ images += [x.strip() for x in line.split(',')
+ if not x.strip().endswith('vmdk')]
+ return images
+
+
+def _find_images(basedir):
+ images = []
+ try:
+ image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
+ if os.path.exists(image_tool):
+ images = subprocess.check_output(image_tool).split('\n')
+ except subprocess.CalledProcessError as ce:
+ print "image_list.sh failed"
+ print "Exit: %s, Output: %s" % (ce.returncode, ce.output)
+ # reset images so we'll fall back
+ images = []
+ return images
+
+
+def local_prep(distribution):
+ branches = []
+ for branch in git_branches():
+ # Ignore branches of the form 'somestring -> someotherstring'
+ # as this denotes a symbolic reference and the entire string
+ # as is cannot be checked out. We can do this safely as the
+ # reference will refer to one of the other branches returned
+ # by git_branches.
+ if ' -> ' in branch:
+ continue
+ branch_data = {'name': branch}
+ print 'Branch: ', branch
+ run_local(['git', 'checkout', branch], cwd=DEVSTACK)
+ run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
+
+ if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')):
+ debs = []
+ debdir = os.path.join(DEVSTACK, 'files', 'apts')
+ for fn in os.listdir(debdir):
+ fn = os.path.join(debdir, fn)
+ tokenize(fn, debs, distribution, comment='#')
+ branch_data['debs'] = debs
+
+ if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/rpm')):
+ rpms = []
+ rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
+ for fn in os.listdir(rpmdir):
+ fn = os.path.join(rpmdir, fn)
+ tokenize(fn, rpms, distribution, comment='#')
+ branch_data['rpms'] = rpms
+
+ images = _find_images(DEVSTACK)
+ if not images:
+ images = _legacy_find_images(DEVSTACK)
+
+ branch_data['images'] = images
+ branches.append(branch_data)
+ return branches
+
+
+def main():
+
+ branches = local_prep(RELEASE)
+
+ with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs:
+ for branch_data in branches:
+ if branch_data.get('debs'):
+ pkgs.write(" ".join(branch_data['debs']) + "\n")
+ elif branch_data.get('rpms'):
+ pkgs.write(" ".join(branch_data['rpms']) + "\n")
+ else:
+ sys.exit('No supported package data found.')
+
+ image_filenames = []
+ line_template = "%(name)s file %(location)s %(url)s\n"
+ with open(IMAGES, 'w') as images_list:
+ for branch_data in branches:
+ for url in branch_data['images']:
+ fname = url.split('/')[-1].strip()
+ if fname == '':
+ continue
+ # TODO: There is an image in devstack that starts with
+ # $HEAT_FETCHED_TEST_IMAGE
+ if fname.startswith('$'):
+ continue
+ if fname in image_filenames:
+ continue
+ image_filenames.append(fname)
+ args = dict(
+ name=fname,
+ location=os.path.join('/home/jenkins/cache/files', fname),
+ url=url)
+
+ images_list.write(line_template % args)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo
new file mode 100755
index 0000000000..c757bc3c80
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/extra-data.d/60-rm-early-source-repo
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remove the copy of devstack that we cloned in for getting devstack
+# cache data so that the later cloning of all of the OpenStack repos
+# won't bomb out
+
+set -eu
+
+sudo rm -rf $TMP_MOUNT_PATH/opt/git/openstack-dev/devstack
diff --git a/modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs b/modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs
new file mode 100644
index 0000000000..b80311d434
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/cache-devstack/install.d/50-download-pkgs
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+while read line ; do
+
+ if [ -f /usr/bin/apt-get ] ; then
+ apt-get -y -d install $line
+ else
+ yum install -y --downloadonly $line
+ fi
+
+done < ~/tmp/pkgs-to-install
+
+rm /tmp/pkgs-to-install
diff --git a/modules/openstack_project/files/nodepool/elements/node-devstack/README.rst b/modules/openstack_project/files/nodepool/elements/node-devstack/README.rst
new file mode 100644
index 0000000000..455c31e147
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/node-devstack/README.rst
@@ -0,0 +1 @@
+Prepare a node to be a devstack slave
diff --git a/modules/openstack_project/files/nodepool/elements/node-devstack/element-deps b/modules/openstack_project/files/nodepool/elements/node-devstack/element-deps
new file mode 100644
index 0000000000..4a446a2f00
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/node-devstack/element-deps
@@ -0,0 +1,2 @@
+puppet
+cache-devstack
diff --git a/modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node b/modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node
new file mode 100755
index 0000000000..ebf2f32cb2
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/node-devstack/install.d/20-prepare-node
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export SUDO='true'
+export THIN='true'
+
+prepare-node
+
+mkdir -p ~jenkins/cache/files
+mkdir -p ~jenkins/cache/pip
+
+install-packages build-essential python-dev
diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst b/modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst
new file mode 100644
index 0000000000..ec1436dded
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/README.rst
@@ -0,0 +1 @@
+Tasks to deal with image metadata and other nodepool cloud specific tweaks.
diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts b/modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts
new file mode 100755
index 0000000000..e690500834
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/exta-data.d/50-copy-nodepool-scripts
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copy the nodepools scripts into the image
+
+set -eu
+
+sudo cp -a $NODEPOOL_SCRIPTDIR $TMP_MOUNT_PATH/opt/nodepool-scripts
+sudo chmod -R a+rx $TMP_MOUNT_PATH/opt/nodepool-scripts
diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound b/modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound
new file mode 100755
index 0000000000..ddf28346f1
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/finalise.d/99-unbound
@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# The puppet modules should install unbound.
+dd of=/tmp/forwarding.conf <<EOF
+forward-zone:
+ name: "."
+ forward-addr: 8.8.8.8
+EOF
+
+mv /tmp/forwarding.conf /etc/unbound/
+chown root:root /etc/unbound/forwarding.conf
+chmod a+r /etc/unbound/forwarding.conf
+
+# HPCloud has selinux enabled by default, Rackspace apparently not.
+# Regardless, apply the correct context.
+if [ -x /sbin/restorecon ] ; then
+ chcon system_u:object_r:named_conf_t:s0 /etc/unbound/forwarding.conf
+fi
+
+# Overwrite /etc/resolv.conf at boot
+dd of=/etc/rc.local <<EOF
+
+#!/bin/bash
+set -o xtrace
+
+# Some providers inject dynamic network config statically. Work around this
+# for DNS nameservers. This is expected to fail on some nodes so remove -e.
+set +e
+sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-*
+set -e
+
+echo 'nameserver 127.0.0.1' > /etc/resolv.conf
+
+exit 0
+EOF
+
+echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf
diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details
new file mode 100755
index 0000000000..534399c29d
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/05-record-details
@@ -0,0 +1,18 @@
+#!/bin/bash -xe
+# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo $DIB_IMAGE_NAME > /etc/image-hostname.txt
diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory
new file mode 100755
index 0000000000..5dec858a53
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/50-restrict-memory
@@ -0,0 +1,31 @@
+#!/bin/bash -xe
+# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is done in 50 because the vm element has, at 51, an bunch of code
+# for properly handling grub. All we need to do here is get our values
+# in to the files.
+
+set -e
+
+# Limit all test slaves to 8GB of memory so that larger flavors with more
+# cpu resources can be used without the risk of becoming dependent on more
+# memory.
+if [ -f /etc/default/grub ] ; then
+ sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' -e 's/#\?GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="mem=8G"/g' /etc/default/grub
+elif [ -f /boot/grub/grub.conf ] ; then
+ sed -i -e 's/^timeout=[0-9]\+/timeout=0/' -e 's/\(^\s\+kernel.*\)/\1 mem=8G/' /boot/grub/grub.conf
+fi
diff --git a/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul
new file mode 100755
index 0000000000..5b94945fbf
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/nodepool-base/install.d/99-install-zuul
@@ -0,0 +1,22 @@
+#!/bin/bash -xe
+# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# Install Zuul into a virtualenv
+# This is in /usr instead of /usr/local due to this bug on precise:
+# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
+
+git clone /opt/git/openstack-infra/zuul /tmp/zuul
+sudo virtualenv /usr/zuul-env
+sudo /usr/zuul-env/bin/pip install /tmp/zuul
+sudo rm -fr /tmp/zuul
diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst b/modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst
new file mode 100644
index 0000000000..82bcfde8cf
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/README.rst
@@ -0,0 +1 @@
+Download all repos and packages that a devstack run might need.
diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps b/modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps
new file mode 100644
index 0000000000..b56002654e
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/element-deps
@@ -0,0 +1,3 @@
+cache-url
+puppet
+source-repositories
diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list b/modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list
new file mode 100755
index 0000000000..ef84ed6174
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import urllib2
+import yaml
+
+URL = ('https://git.openstack.org/cgit/openstack-infra/config/plain/'
+ 'modules/openstack_project/files/review.projects.yaml')
+
+TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH']
+PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml')
+GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org')
+
+
+def main():
+ projects = [f['project'] for f in yaml.load(urllib2.urlopen(URL))]
+ with open(PROJECTS_REPOS, 'w') as projects_list:
+ for project in projects:
+ args = dict(
+ name=os.path.basename(project),
+ location=os.path.join('/opt/git', project),
+ url='%s/%s.git' % (GIT_BASE, project))
+
+ projects_list.write("%(name)s git %(location)s %(url)s\n" % args)
+ # Clone openstack-infra/config again so that we can use it to build
+ # the image without interferring with the slave repo cache.
+ project = 'openstack-infra/config'
+ args = dict(
+ name='config_tmp',
+ location=os.path.join('/opt/build_git', project),
+ url=os.environ.get('CONFIG_SOURCE',
+ '%s/%s.git' % (GIT_BASE, project)),
+ ref=os.environ.get('CONFIG_REF', 'master'))
+ projects_list.write(
+ "%(name)s git %(location)s %(url)s %(ref)s\n" % args)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins b/modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins
new file mode 100755
index 0000000000..5037b591a2
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/openstack-repos/install.d/95-chown-jenkins
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+chown -R jenkins:jenkins /home/jenkins
diff --git a/modules/openstack_project/files/nodepool/elements/puppet/README.rst b/modules/openstack_project/files/nodepool/elements/puppet/README.rst
new file mode 100644
index 0000000000..071da1712b
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/puppet/README.rst
@@ -0,0 +1 @@
+Bootstrap puppet on a node
diff --git a/modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node b/modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node
new file mode 100644
index 0000000000..3ad72c35e9
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/puppet/bin/prepare-node
@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SUDO=${SUDO:-true}
+THIN=${THIN:-true}
+PYTHON3=${PYTHON3:-false}
+PYPY=${PYPY:-false}
+ALL_MYSQL_PRIVS=${ALL_MYSQL_PRIVS:-false}
+
+export FACTER_in_chroot=true
+cat >/tmp/local.pp <<EOF
+Service {
+ start => '/bin/true',
+ stop => '/bin/true',
+ status => '/bin/true',
+ restart => '/bin/true'
+}
+class {'openstack_project::single_use_slave':
+ sudo => $SUDO,
+ thin => $THIN,
+ python3 => $PYTHON3,
+ include_pypy => $PYPY,
+ all_mysql_privs => $ALL_MYSQL_PRIVS,
+ install_resolv_conf => false
+}
+EOF
+# Puppet doesn't return nonzero if some things fail by default.
+# Use detailed exit codes to get that info and determine whether
+# the return code indicates failure.
+set +e
+puppet apply --detailed-exitcodes --modulepath=/opt/build_git/openstack-infra/config/modules:/etc/puppet/modules /tmp/local.pp
+PUPPET_RETURN=$?
+if [ "$PUPPET_RETURN" -eq 4 ] || [ "$PUPPET_RETURN" -eq 6 ] ; then
+ exit $PUPPET_RETURN
+fi
+set -e
+
+# Make sure resolv.conf settings don't break dib
+echo 'nameserver 8.8.8.8'> /etc/resolv.conf
diff --git a/modules/openstack_project/files/nodepool/elements/puppet/element-deps b/modules/openstack_project/files/nodepool/elements/puppet/element-deps
new file mode 100644
index 0000000000..95746ba5ab
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/puppet/element-deps
@@ -0,0 +1,3 @@
+source-repositories
+cache-url
+openstack-repos
diff --git a/modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet b/modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet
new file mode 100755
index 0000000000..faa80370e6
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/puppet/install.d/05-puppet
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Unset the download cache for this invocation to prevent bleed from build host
+unset PIP_DOWNLOAD_CACHE
+export PUPPET_VERSION=${PUPPET_VERSION:-'2'}
+
+/bin/bash /opt/git/openstack-infra/config/install_puppet.sh
+/bin/bash /opt/git/openstack-infra/config/install_modules.sh
+
+install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin
diff --git a/modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos b/modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos
new file mode 100755
index 0000000000..613e5d1866
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/puppet/install.d/95-clean-repos
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remove additional sources used to install puppet or special version of pypi.
+# We do this because leaving these sources in place causes every test that
+# does an apt-get update to hit those servers which may not have the uptime
+# of our local mirrors.
+
+set -e
+
+OS_FAMILY=$(facter osfamily)
+if [ "$OS_FAMILY" == "Debian" ] ; then
+ rm -f /etc/apt/sources.list.d/*
+ apt-get update
+elif [ "$OS_FAMILY" == "RedHat" ] ; then
+ # Can't delete * in yum.repos.d since all of the repos are listed there.
+ # Be specific instead.
+ if [ -f /etc/yum.repos.d/puppetlabs.repo ] ; then
+ sudo rm -f /etc/yum.repos.d/puppetlabs.repo
+ fi
+fi
+
+rm /usr/local/bin/prepare-node
diff --git a/modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron b/modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron
new file mode 100755
index 0000000000..4c8e54f721
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/puppet/install.d/96-clean-cron
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remove cron jobs
+# We create fresh servers for these hosts, and they are used once. They don't
+# need to do things like update the locatedb or the mandb or rotate logs
+# or really any of those things. We only want code running here that we want
+# here.
+rm -f /etc/cron.{monthly,weekly,daily,hourly,d}/*
diff --git a/modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed b/modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed
new file mode 100755
index 0000000000..4c5dbef515
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/puppet/pre-install.d/10-preseed
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# workaround a broken maintainer script in iptables-persistent that
+# assumes it can execute things during postinst. Since we're installing
+# in a chroot, we're actually not running the right kernel for its assumptions
+# to work.
+
+set -e
+
+# If lsb_release is missing, just do nothing.
+DISTRO=`lsb_release -si` || true
+
+case $DISTRO in
+ 'Ubuntu'|'Debian')
+ PRESEED=`mktemp`
+ cat > $PRESEED <<EOF
+d-i iptables-persistent/autosave_done boolean true
+d-i iptables-persistent/autosave_v4 boolean false
+d-i iptables-persistent/autosave_v6 boolean false
+EOF
+ debconf-set-selections $PRESEED
+ rm $PRESEED
+ ;;
+esac
diff --git a/modules/openstack_project/files/nodepool/elements/slave-db/README.rst b/modules/openstack_project/files/nodepool/elements/slave-db/README.rst
new file mode 100644
index 0000000000..98c248b4f3
--- /dev/null
+++ b/modules/openstack_project/files/nodepool/elements/slave-db/README.rst
@@ -0,0 +1 @@
+Install databases on slave nodes
diff --git a/modules/openstack_project/manifests/nodepool_prod.pp b/modules/openstack_project/manifests/nodepool_prod.pp
index cd5cee8df5..e4ec68eee5 100644
--- a/modules/openstack_project/manifests/nodepool_prod.pp
+++ b/modules/openstack_project/manifests/nodepool_prod.pp
@@ -60,4 +60,16 @@ class openstack_project::nodepool_prod(
source => 'puppet:///modules/openstack_project/nodepool/scripts',
}
+ file { '/etc/nodepool/elements':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ recurse => true,
+ purge => true,
+ force => true,
+ require => File['/etc/nodepool'],
+ source => 'puppet:///modules/openstack_project/nodepool/elements',
+ }
+
}
diff --git a/modules/openstack_project/manifests/slave_common.pp b/modules/openstack_project/manifests/slave_common.pp
index 88f37bbb68..bf662bbc25 100644
--- a/modules/openstack_project/manifests/slave_common.pp
+++ b/modules/openstack_project/manifests/slave_common.pp
@@ -92,4 +92,27 @@ class openstack_project::slave_common(
}
}
}
+
+ # install linux-headers depending on OS version
+ case $::osfamily {
+ 'RedHat': {
+ $header_packages = ['kernel-devel', 'kernel-headers']
+ }
+ 'Debian': {
+ if ($::operatingsystem == 'Debian') {
+ # install depending on kernel release
+ $header_packages = [ "linux-headers-${::kernelrelease}", ]
+ }
+ else {
+ $header_packages = ['linux-headers-virtual', 'linux-headers-generic']
+ }
+ }
+ default: {
+ fail("Unsupported osfamily: ${::osfamily}.")
+ }
+ }
+
+ package { $header_packages:
+ ensure => present
+ }
}
diff --git a/modules/openstack_project/templates/nodepool/nodepool.yaml.erb b/modules/openstack_project/templates/nodepool/nodepool.yaml.erb
index 7f8fcac794..113eb5bebf 100644
--- a/modules/openstack_project/templates/nodepool/nodepool.yaml.erb
+++ b/modules/openstack_project/templates/nodepool/nodepool.yaml.erb
@@ -1,4 +1,6 @@
script-dir: /etc/nodepool/scripts
+elements-dir: /etc/nodepool/elements
+images-dir: /opt/nodepool_dib
dburi: 'mysql://nodepool:<%= mysql_password %>@localhost/nodepool'
cron:
diff --git a/tools/build-dib-in-docker.sh b/tools/build-dib-in-docker.sh
new file mode 100644
index 0000000000..7cfd0a2b81
--- /dev/null
+++ b/tools/build-dib-in-docker.sh
@@ -0,0 +1,16 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker.io run --privileged=true -v ~/.cache:/.cache -v $(git rev-parse --show-toplevel):/opt/config -w /opt/config -i -t openstack_infra/ubuntu /bin/bash tools/build-image.sh
diff --git a/tools/build-image.sh b/tools/build-image.sh
new file mode 100755
index 0000000000..5f67944e9d
--- /dev/null
+++ b/tools/build-image.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export ELEMENTS_PATH=${ELEMENTS_PATH:-modules/openstack_project/files/nodepool/elements}
+export DISTRO=${DISTRO:-ubuntu}
+export DIB_RELEASE=${DIB_RELEASE:-trusty}
+export DIB_IMAGE_NAME=${DIB_IMAGE_NAME:-${DISTRO}_${DIB_RELEASE}}
+export DIB_IMAGE_FILENAME=${DIB_IMAGE_FILENAME:-${DIB_IMAGE_NAME}.qcow}
+export NODEPOOL_SCRIPTDIR=${NODEPOOL_SCRIPTDIR:-modules/openstack_project/files/nodepool/scripts}
+export CONFIG_SOURCE=${CONFIG_SOURCE:-file://$(pwd)}
+export CONFIG_REF=${CONFIG_REF:-$(git rev-parse HEAD)}
+
+disk-image-create -x --no-tmpfs -o devstack-gate-$DIB_RELEASE $DISTRO vm openstack-repos puppet nodepool-base node-devstack
diff --git a/tools/mount-image.sh b/tools/mount-image.sh
new file mode 100644
index 0000000000..4e9e51f9a7
--- /dev/null
+++ b/tools/mount-image.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+IMAGE=$1
+
+rm -rf /tmp/newimage
+mkdir -p /tmp/newimage
+
+qemu-nbd -c /dev/nbd1 $1
+mount /dev/nbd1 /tmp/newimage
diff --git a/tools/umount-image.sh b/tools/umount-image.sh
new file mode 100644
index 0000000000..16ce2769cc
--- /dev/null
+++ b/tools/umount-image.sh
@@ -0,0 +1,19 @@
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+umount /tmp/newimage
+qemu-nbd -d /dev/nbd1