Adding of deployment scripts for sahara-ci

Added deployemnt scripts for sahara-ci. Compatible with
Ubuntu Xenial.

Change-Id: Ic218243efeb6c6aae17faa665755c5e722e10e2b

config/jjb/credentials.conf

@@ -0,0 +1,4 @@
+[default]
+OS_USERNAME=ci-user
+OS_TENANT_NAME=ci
+OS_PASSWORD=nova

config/jjb/jenkins_jobs.ini

@@ -0,0 +1,6 @@
+[jenkins]
+user=USER
+password=PASSWORD
+url=http://127.0.0.1/jenkins
+[job_builder]
+ignore_cache=True

config/nodepool/config/logging.conf

@@ -0,0 +1,56 @@
+[loggers]
+keys=root,nodepool,requests,image
+
+[handlers]
+keys=console,debug,normal,image
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_requests]
+level=WARNING
+handlers=debug,normal
+qualname=requests
+
+[logger_nodepool]
+level=DEBUG
+handlers=debug,normal
+qualname=nodepool
+
+[logger_image]
+level=INFO
+handlers=image
+qualname=nodepool.image.build
+propagate=0
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.TimedRotatingFileHandler
+formatter=simple
+args=('/var/log/nodepool/debug.log', 'midnight', 1, 30,)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.TimedRotatingFileHandler
+formatter=simple
+args=('/var/log/nodepool/nodepool.log', 'midnight', 1, 30,)
+
+[handler_image]
+level=INFO
+class=logging.handlers.TimedRotatingFileHandler
+formatter=simple
+args=('/var/log/nodepool/image.log', 'midnight', 1, 30,)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=

config/nodepool/config/sahara.yaml

@@ -1,4 +1,5 @@
 script-dir: /opt/ci/files/nodepool-scripts
+images-dir: /opt/ci/files/nodepool_dib
 
 cron:
   check: '*/1 * * * *'
@@ -58,4 +59,4 @@ providers:
         setup: prepare_node.sh
 
 targets:
-  - name: sahara-gate
+  - name: sahara-ci

config/nodepool/config/secure.conf

@@ -1,7 +1,7 @@
 [database]
-dburi=mysql://nodepool@localhost/nodepool
+dburi=mysql+pymysql://nodepool@localhost/nodepool
 
-[jenkins "sahara-gate"]
+[jenkins "sahara-ci"]
 url=http://127.0.0.1/jenkins
 user=admin
 apikey=JENKINS_API_KEY

config/zuul/zuul.conf

@@ -23,7 +23,7 @@ status_url=https://sahara.mirantis.com/zuul/
 
 [merger]
 git_dir=/var/lib/zuul/git
-zuul_url=http://172.18.87.221/p
+zuul_url=http://172.18.87.222/p
 log_config=/etc/zuul/logging.conf
 pidfile=/var/run/zuul-merger/zuul-merger.pid
 

slave-scripts/update_config.sh

@@ -7,14 +7,14 @@ sudo su - zuul -c "cat $WORKSPACE/config/zuul/gearman-logging.conf > /etc/zuul/g
 sudo su - zuul -c "cat $WORKSPACE/config/zuul/layout.yaml > /etc/zuul/layout.yaml"
 sudo su - zuul -c "cat $WORKSPACE/config/zuul/logging.conf > /etc/zuul/logging.conf"
 sudo su - zuul -c "cat $WORKSPACE/config/zuul/openstack_functions.py > /etc/zuul/openstack_functions.py"
-sudo service zuul reload
+sudo systemctl reload zuul.service
 
-sed "s%- net-id: 'LAB_42_PRIVATE_NETWORK_ID'%- net-id: '$LAB_42_PRIVATE_NETWORK_ID'%g" -i $WORKSPACE/config/nodepool/sahara.yaml
+sed "s%- net-id: 'LAB_42_PRIVATE_NETWORK_ID'%- net-id: '$LAB_42_PRIVATE_NETWORK_ID'%g" -i $WORKSPACE/config/nodepool/config/sahara.yaml
-sed "s%- net-id: 'LAB_43_PRIVATE_NETWORK_ID'%- net-id: '$LAB_43_PRIVATE_NETWORK_ID'%g" -i $WORKSPACE/config/nodepool/sahara.yaml
+sed "s%- net-id: 'LAB_43_PRIVATE_NETWORK_ID'%- net-id: '$LAB_43_PRIVATE_NETWORK_ID'%g" -i $WORKSPACE/config/nodepool/config/sahara.yaml
-sed "s%apikey=JENKINS_API_KEY%apikey=$JENKINS_API_KEY%g" -i $WORKSPACE/config/nodepool/secure.conf
+sed "s%apikey=JENKINS_API_KEY%apikey=$JENKINS_API_KEY%g" -i $WORKSPACE/config/nodepool/config/secure.conf
-sed "s%credentials=CREDENTIALS_ID%credentials=$CREDENTIALS_ID%g" -i $WORKSPACE/config/nodepool/secure.conf
+sed "s%credentials=CREDENTIALS_ID%credentials=$CREDENTIALS_ID%g" -i $WORKSPACE/config/nodepool/config/secure.conf
-sudo su - nodepool -c "cat $WORKSPACE/config/nodepool/sahara.yaml > /etc/nodepool/nodepool.yaml"
+sudo su - nodepool -c "cat $WORKSPACE/config/nodepool/config/sahara.yaml > /etc/nodepool/nodepool.yaml"
-sudo su - nodepool -c "cat $WORKSPACE/config/nodepool/secure.conf > /etc/nodepool/secure.conf"
+sudo su - nodepool -c "cat $WORKSPACE/config/nodepool/config/secure.conf > /etc/nodepool/secure.conf"
 
 sed "s%MYSQL_PASS=MYSQL_ROOT_PASSWORD%MYSQL_PASS=$MYSQL_ROOT_PASSWORD%g" -i $WORKSPACE/config/nodepool/scripts/prepare_node.sh
 sed "s%JENKINS_PUBLIC_KEY%$JENKINS_PUBLIC_KEY%g" -i $WORKSPACE/config/nodepool/scripts/prepare_node.sh

system-configs/README.rst

@@ -0,0 +1,11 @@
+After script running
+1. Setup Jenkins
+2. Run jenkins-jobs
+3. Add credentials to JENKINS_HOME DIR
+4. Trigger "update config" job
+5. Add key for zuul to /etc/zuul/gerrit
+6. Start zuul
+7. Add key for nodepool to /etc/nodepool/id_dsa
+8. Add full permissions on nodepool key
+9. Start nodepool
+10. Enable apache2 sites (start from jenkins.conf file)

system-configs/functions.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -xe
+
+clone() {
+  local project_name=$1
+  local project_dir=$2
+
+  git clone https://review.openstack.org/"$project_name" "$project_dir"
+}
+
+install_to_venv() {
+  local project_dir=$1
+  local venv_name=${2:-"venv"}
+  local venv_path=$project_dir/$venv_name
+
+  virtualenv $venv_path
+  $venv_path/bin/pip install $project_dir
+}

system-configs/install_projects.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set +x
+
+. /home/ubuntu/sahara-ci-config/system-configs/functions.sh
+
+default_path=/opt/ci
+projects=( "openstack-infra/nodepool" "openstack-infra/zuul" "openstack-infra/jenkins-job-builder" )
+
+mkdir -p $default_path
+
+for project_repo in "${projects[@]}"
+do
+    project=$(basename $project_repo)
+    project_dir=$default_path/$project
+    clone $project_repo $project_dir
+    install_to_venv $project_dir
+    sudo chown -R $project:$project $project_dir
+done
+
+# prepare apache
+
+sudo cp /home/ubuntu/sahara-ci-config/system-configs/sites-available/* /etc/apache2/sites-available/
+
+sudo a2enmod proxy
+sudo a2enmod proxy_http
+sudo a2enmod proxy_connect
+sudo a2enmod rewrite
+sudo a2enmod dav
+sudo a2enmod dav_fs
+sudo a2enmod cgi
+
+sudo systemctl restart apache2
+
+for host in $(/etc/apache2/sites-available/*)
+do
+    sudo a2ensite $(basename $host)
+done
+
+sudo systemctl reload apache2
+
+# prepare jenkins
+echo "jenkins ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/jenkins
+mkdir -p /opt/ci/files/nodepool-scripts /opt/ci/files/ext-files /opt/ci/logs
+sudo chown -R jenkins:jenkins /opt/ci/files/ /opt/ci/logs
+
+#prepare jenkins-jobs-builder
+mkdir /etc/jenkins_jobs
+cp /home/ubuntu/sahara-ci-config/config/jjb/* /etc/jenkins_jobs
+sed "s%user=USER%user=$JJB_USER%g" -i /etc/jenkins_jobs/jenkins_jobs.ini
+sed "s%password=PASSWORD%password=$JJB_PASSWORD%g" -i /etc/jenkins_jobs/jenkins_jobs.ini
+sudo chown -R jenkins:jenkins /etc/jenkins_jobs/ /opt/ci/jenkins-job-builder
+sudo ln -s /opt/ci/jenkins-job-builder/venv/bin/jenkins-jobs /usr/local/bin/jenkins-jobs
+
+# prepare zuul
+
+# need trigger update-config job
+sudo mkdir -p /var/www/zuul /etc/zuul /var/lib/zuul/ /var/log/zuul
+sudo touch /etc/zuul/layout.yaml /var/lib/zuul/times /var/log/zuul/gearman-server
+sudo bash /opt/ci/zuul/etc/status/fetch-dependencies.sh
+sudo cp -r /opt/ci/zuul/etc/status/public_html/* /var/www/zuul
+sudo chown -R zuul:zuul /var/www/zuul /etc/zuul /var/lib/zuul /var/log/zuul
+
+sudo ln -s /opt/ci/zuul/venv/bin/zuul /usr/sbin/zuul-client
+
+#prepare nodepool
+mkdir -p /var/log/nodepool /var/run/nodepool /etc/nodepool /opt/ci/files/nodepool_dib
+sudo cp /home/ubuntu/sahara-ci-config/config/nodepool/scripts/* /opt/ci/files/nodepool-scripts
+sudo cp /home/ubuntu/sahara-ci-config/config/nodepool/config/secure.conf /etc/nodepool/
+sudo cp /home/ubuntu/sahara-ci-config/config/nodepool/config/logging.conf /etc/nodepool/
+sudo cp /home/ubuntu/sahara-ci-config/slave-scripts/update_pool.sh /opt/ci/files
+sudo chown jenkins:jenkins /opt/ci/files/update_pool.sh
+sudo chown -R nodepool:nodepool /var/run/nodepool/ /etc/nodepool/ /var/log/nodepool /opt/ci/files/nodepool_dib
+
+sudo mysql -uroot -Bse "CREATE USER 'nodepool'@'localhost'"
+sudo mysql -uroot -Bse "GRANT ALL PRIVILEGES ON *.* TO 'nodepool'@'localhost' WITH GRANT OPTION"
+sudo mysql -uroot -Bse "FLUSH PRIVILEGES"
+sudo mysql -unodepool -Bse "create database nodepool"
+
+sudo ln -s /opt/ci/nodepool/venv/bin/nodepool /usr/sbin/nodepool-client
+
+cp /home/ubuntu/sahara-ci-config/system-configs/systemd/* /lib/systemd/system/

system-configs/jenkins-plugins

@@ -0,0 +1,58 @@
+ansicolor.hpi
+antisamy-markup-formatter.hpi
+ant.hpi
+bazaar.hpi
+build-blocker-plugin.hpi
+build-timeout.hpi
+conditional-buildstep.hpi
+copyartifact.hpi
+credentials.hpi
+cvs.hpi
+dashboard-view.hpi
+envinject.hpi
+extended-read-permission.hpi
+external-monitor-job.hpi
+gearman-plugin.hpi
+git-client.hpi
+github-api.hpi
+github.hpi
+github-oauth.hpi
+git.jpi
+global-build-stats.hpi
+greenballs.hpi
+htmlpublisher.hpi
+javadoc.hpi
+jenkins-multijob-plugin.hpi
+jobConfigHistory.hpi
+junit.hpi
+ldap.hpi
+mailer.hpi
+mapdb-api.hpi
+matrix-auth.hpi
+matrix-project.hpi
+maven-plugin.hpi
+openid4java.hpi
+openid.hpi
+pam-auth.hpi
+parameterized-trigger.hpi
+pathignore.hpi
+postbuild-task.hpi
+project-stats-plugin.hpi
+publish-over-ftp.hpi
+publish-over-ssh.hpi
+rebuild.hpi
+ruby-runtime.hpi
+run-condition.hpi
+scm-api.hpi
+scp.hpi
+simple-theme-plugin.hpi
+ssh-agent.hpi
+ssh-credentials.hpi
+ssh-slaves.hpi
+subversion.hpi
+timestamper.hpi
+token-macro.hpi
+translation.hpi
+windows-slaves.hpi
+ws-cleanup.hpi
+zmq-event-publisher.hpi

system-configs/prepare_host.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set +x
+
+wget -q -O - https://jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add -
+sudo sh -c 'echo deb http://pkg.jenkins-ci.org/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list'
+
+APT_PACKAGES="git python-dev gcc make jenkins python-pip apache2 unzip mysql-server libssl-dev"
+PIP_PACKAGES+=" virtualenv"
+
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password '
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password '
+
+sudo apt update
+sudo apt install -y $APT_PACKAGES
+
+sudo pip install -U $PIP_PACKAGES
+
+mkdir -p /opt/ci/files
+
+# create users
+projects=( "nodepool" "zuul" )
+for project in "${projects[@]}"
+do
+    sudo useradd -d /home/$project -G sudo -s /bin/bash -m $project
+    sudo mkdir /home/$project/.ssh
+    sudo chown -R $project:$project /home/$project
+done
+
+bash /home/ubuntu/sahara-ci-config/system-configs/install_projects.sh
+
+# install jenkins plugins
+while read plugin
+do
+    wget http://updates.jenkins-ci.org/1.651/latest/$plugin -P /var/lib/jenkins/plugins
+done < /home/ubuntu/sahara-ci-config/system-configs/jenkins-plugins
+
+sudo systemctl restart jenkins

system-configs/sites-available/000-default.conf

@@ -0,0 +1,45 @@
+ProxyErrorOverride On
+ErrorDocument      404 /
+
+<VirtualHost *:80>
+    ServerAdmin webmaster@localhost
+
+    DocumentRoot /var/www
+    <Directory />
+        Options FollowSymLinks
+        AllowOverride None
+    </Directory>
+    <Directory /var/www/>
+        Options FollowSymLinks
+        AllowOverride None
+        Order allow,deny
+        allow from all
+    </Directory>
+
+    RewriteEngine on
+    RewriteRule ^/zuul/status.json$ http://127.0.0.1:8001/status.json [P]
+    SetEnv GIT_PROJECT_ROOT /var/lib/zuul/git/
+    SetEnv GIT_HTTP_EXPORT_ALL
+
+    AliasMatch ^/p/(.*/objects/[0-9a-f]{2}/[0-9a-f]{38})$ /var/lib/zuul/git/$1
+    AliasMatch ^/p/(.*/objects/pack/pack-[0-9a-f]{40}.(pack|idx))$ /var/lib/zuul/git/$1
+
+    ScriptAliasMatch /p/ /usr/lib/git-core/git-http-backend/
+
+    Alias /p /var/lib/zuul/git/
+    <Directory "/var/lib/zuul/git/">
+        DAV on
+        Options Indexes FollowSymLinks
+        AllowOverride None
+        Require all granted
+    </Directory>
+
+    ErrorLog ${APACHE_LOG_DIR}/error.log
+
+    # Possible values include: debug, info, notice, warn, error, crit,
+    # alert, emerg.
+    LogLevel warn
+
+    CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+</VirtualHost>

system-configs/sites-available/files.conf

@@ -0,0 +1,4 @@
+Alias /files /opt/ci/files/ext-files
+<Directory /opt/ci/files/ext-files>
+  Require all granted
+</Directory>

system-configs/sites-available/jenkins.conf

@@ -0,0 +1,9 @@
+ProxyPass         /jenkins  http://localhost:8080/jenkins
+ProxyPassReverse  /jenkins  http://localhost:8080/jenkins
+ProxyRequests     Off
+
+# Local reverse proxy authorization override
+# Most unix distribution deny proxy by default (ie /etc/apache2/mods-enabled/proxy.conf in Ubuntu)
+<Proxy http://localhost:8080/jenkins*>
+  Require all granted
+</Proxy>

system-configs/sites-available/logs.conf

@@ -0,0 +1,5 @@
+Alias /logs /opt/ci/logs
+<Directory /opt/ci/logs>
+  Require all granted
+  Options +Indexes
+</Directory>

system-configs/systemd/nodepool.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Nodepool
+
+[Service]
+ExecStart=/opt/ci/nodepool/venv/bin/nodepoold -c /etc/nodepool/nodepool.yaml -l /etc/nodepool/logging.conf
+Restart=on-failure
+RestartPreventExitStatus=255
+Type=idle
+User=nodepool
+Group=nodepool
+
+[Install]
+WantedBy=multi-user.target
+Alias=nodepool.service

system-configs/systemd/zuul-merger.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Zuul merger
+
+[Service]
+ExecStart=/opt/ci/zuul/venv/bin/zuul-merger -d
+Restart=on-failure
+RestartPreventExitStatus=255
+Type=idle
+User=zuul
+Group=zuul
+
+[Install]
+WantedBy=multi-user.target
+Alias=zuul-merger.service

system-configs/systemd/zuul.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Zuul server
+
+[Service]
+ExecStart=/opt/ci/zuul/venv/bin/zuul-server -d
+Restart=on-failure
+RestartPreventExitStatus=255
+Type=idle
+User=zuul
+Group=zuul
+
+[Install]
+WantedBy=multi-user.target
+Alias=zuul.service