From 39495ffdd5f2ed226e4d97ae2f709ee747d6d9dc Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Mon, 27 Apr 2020 08:28:38 -0500
Subject: [PATCH] Test zuul-executor on focal
We want to replace the current executors with focal executors.
Make sure zuul-executor can run there.
Kubic is apparently the new source for libcontainers stuff:
https://podman.io/getting-started/installation.html
Use only timesyncd on focal
ntp and timesyncd have a hard conflict with each other. Our test
images install ntp. Remove it and just stay with timesyncd.
Change-Id: I0126f7c77d92deb91711f38a19384a9319955cf5
---
.zuul.yaml | 5 ++
inventory/groups.yaml | 2 +
.../group_vars/zuul-executor-opendev.yaml | 3 +
playbooks/group_vars/zuul-executor.yaml | 22 ++++++++
.../files/sources.list.focal.x86_64 | 7 +++
playbooks/roles/base-server/tasks/Debian.yaml | 17 ++++--
playbooks/roles/zuul-executor/tasks/main.yaml | 56 +++++++++++++++----
.../zuul-executor/templates/sources.list.j2 | 1 +
.../templates/zuul-executor.defaults.j2 | 4 ++
.../zuul-executor/vars/Ubuntu.focal.yaml | 5 ++
.../roles/zuul-executor/vars/default.yaml | 12 ++++
testinfra/test_base.py | 13 ++++-
12 files changed, 128 insertions(+), 19 deletions(-)
create mode 100644 playbooks/group_vars/zuul-executor-opendev.yaml
create mode 100644 playbooks/roles/base-repos/files/sources.list.focal.x86_64
create mode 100644 playbooks/roles/zuul-executor/templates/sources.list.j2
create mode 100644 playbooks/roles/zuul-executor/templates/zuul-executor.defaults.j2
create mode 100644 playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml
create mode 100644 playbooks/roles/zuul-executor/vars/default.yaml
diff --git a/.zuul.yaml b/.zuul.yaml
index 484bda51c8..e290a92e7f 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1588,6 +1588,8 @@
label: ubuntu-bionic
- name: zm01.openstack.org
label: ubuntu-xenial
+ - name: ze01.opendev.org
+ label: ubuntu-focal
- name: ze01.openstack.org
label: ubuntu-xenial
- name: zuul01.openstack.org
@@ -1604,6 +1606,9 @@
zm01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
+ ze01.opendev.org:
+ host_copy_output:
+ '/etc/zuul/zuul.conf': logs
ze01.openstack.org:
host_copy_output:
'/etc/zuul/zuul.conf': logs
diff --git a/inventory/groups.yaml b/inventory/groups.yaml
index 5828fda144..c9871aab2e 100644
--- a/inventory/groups.yaml
+++ b/inventory/groups.yaml
@@ -257,6 +257,8 @@ groups:
- zuul[0-9]*.open*.org
zuul-executor:
- ze[0-9]*.open*.org
+ zuul-executor-opendev:
+ - ze[0-9]*.opendev.org
zuul-merger:
- zm[0-9]*.open*.org
zuul-preview:
diff --git a/playbooks/group_vars/zuul-executor-opendev.yaml b/playbooks/group_vars/zuul-executor-opendev.yaml
new file mode 100644
index 0000000000..a775b0272c
--- /dev/null
+++ b/playbooks/group_vars/zuul-executor-opendev.yaml
@@ -0,0 +1,3 @@
+# NOTE: ubuntu focal has new enough docker, and upstream docker
+# does not have focal support yet.
+use_upstream_docker: false
diff --git a/playbooks/group_vars/zuul-executor.yaml b/playbooks/group_vars/zuul-executor.yaml
index 7699535fc0..b2f2a4877b 100644
--- a/playbooks/group_vars/zuul-executor.yaml
+++ b/playbooks/group_vars/zuul-executor.yaml
@@ -20,3 +20,25 @@ zuul_connections:
user: 'git-infra-root.openstack.org'
stream_events: 'false'
auth_type: 'basic'
+libcontainers_apt_key: |
+ -----BEGIN PGP PUBLIC KEY BLOCK-----
+ Version: GnuPG v2.0.15 (GNU/Linux)
+
+ mQENBFtkV0cBCADStSTCG5qgYtzmWfymHZqxxhfwfS6fdHJcbGUeXsI5dxjeCWhs
+ XarZm6rWZOd5WfSmpXhbKOyM6Ll+6bpSl5ICHLa6fcpizYWEPa8fpg9EGl0cF12G
+ GgVLnnOZ6NIbsoW0LHt2YN0jn8xKVwyPp7KLHB2paZh+KuURERG406GXY/DgCxUx
+ Ffgdelym/gfmt3DSq6GAQRRGHyucMvPYm53r+jVcKsf2Bp6E1XAfqBrD5r0maaCU
+ Wvd7bi0B2Q0hIX0rfDCBpl4rFqvyaMPgn+Bkl6IW37zCkWIXqf1E5eDm/XzP881s
+ +yAvi+JfDwt7AE+Hd2dSf273o3WUdYJGRwyZABEBAAG0OGRldmVsOmt1YmljIE9C
+ UyBQcm9qZWN0IDxkZXZlbDprdWJpY0BidWlsZC5vcGVuc3VzZS5vcmc+iQE+BBMB
+ CAAoBQJbZFdHAhsDBQkEHrAABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBN
+ ZDkDdQYKpL0BCACEuegX4bxPkdnML70BbvBmDazyuWYZtSAOLoTosVwXBlTeAIoY
+ xAw2Sdc895808blU964fuu8IDcR5KuvkTGSGIDBJ6fHZSFD8r68WSS0k1gOY7bUW
+ 8kkeThlvuMATgULJIYGQ3vYrEhn5e+Pe8fb3AJmQpQWzcjtPX+pk66sVUT6iNaSt
+ QtBVGNTQvVDHekF1j/mYmVbIjIWL6whYkdtlu7KYzCx6KhY9u7i3THxY5ICsq64b
+ rYcy5FHaVv5SHfTvtu1WtiSlACVfOgPGntpI+Xi4P2F4n4c5I5VKBxpHteW4LGz8
+ YOISzS7ZZlbZ7qus353OU0v6UbA660ju5/gGiEYEExECAAYFAltkV0cACgkQOzAR
+ t2udZSOoswCdF44NTN09DwhPFbNYhEMb9juP5ykAn0bcELvuKmgDwEwZMrPQkG8t
+ Pu9n
+ =CNmf
+ -----END PGP PUBLIC KEY BLOCK-----
diff --git a/playbooks/roles/base-repos/files/sources.list.focal.x86_64 b/playbooks/roles/base-repos/files/sources.list.focal.x86_64
new file mode 100644
index 0000000000..7ab0f00049
--- /dev/null
+++ b/playbooks/roles/base-repos/files/sources.list.focal.x86_64
@@ -0,0 +1,7 @@
+# This file is kept updated by ansible, adapted from
+# https://help.ubuntu.com/lts/serverguide/configuration.html
+
+deb http://us.archive.ubuntu.com/ubuntu focal main universe
+deb http://us.archive.ubuntu.com/ubuntu focal-updates main universe
+deb http://us.archive.ubuntu.com/ubuntu focal-backports main universe
+deb http://security.ubuntu.com/ubuntu focal-security main universe
diff --git a/playbooks/roles/base-server/tasks/Debian.yaml b/playbooks/roles/base-server/tasks/Debian.yaml
index 342c3577d4..376b22009a 100644
--- a/playbooks/roles/base-server/tasks/Debian.yaml
+++ b/playbooks/roles/base-server/tasks/Debian.yaml
@@ -26,12 +26,19 @@
enabled: yes
state: started
-- name: Ensure systemd-timesyncd is running
- service:
- name: systemd-timesyncd
- enabled: yes
- state: started
+- name: Remove ntp and run timesyncd
when: ansible_distribution_version is version('18.04', '>=')
+ block:
+ - name: Remove ntp
+ package:
+ name: ntp
+ state: absent
+
+ - name: Ensure systemd-timesyncd is running
+ service:
+ name: systemd-timesyncd
+ enabled: yes
+ state: started
- name: Remove packages that make no sense for our servers
package:
diff --git a/playbooks/roles/zuul-executor/tasks/main.yaml b/playbooks/roles/zuul-executor/tasks/main.yaml
index f9447b1584..1149acb04f 100644
--- a/playbooks/roles/zuul-executor/tasks/main.yaml
+++ b/playbooks/roles/zuul-executor/tasks/main.yaml
@@ -1,17 +1,41 @@
+- name: Include OS-specific variables
+ include_vars: "{{ lookup('first_found', params) }}"
+ vars:
+ params:
+ files: "{{ distro_lookup_path }}"
+ paths:
+ - 'vars'
+
- name: Install PPAs
apt_repository:
repo: '{{ item }}'
become: yes
- loop:
- # For bubblewrap
- - ppa:openstack-ci-core/bubblewrap
- # Temporary PPA needed for bpo-27945 while waiting for SRU to be published
- - ppa:openstack-ci-core/python-bpo-27945-backport
- # We use later HWE kernels for better memory managment, requiring an
- # updated AFS version which we install from our custom ppa.
- - ppa:openstack-ci-core/openafs-amd64-hwe
- # For skopeo
- - ppa:projectatomic/ppa
+ loop: '{{ zuul_executor_ppas }}'
+
+- name: Atomic for focal
+ when: ansible_distribution_version is version('20.04', '>=')
+ block:
+
+ - name: Add Kubic libcontainers OBS repo key
+ become: yes
+ apt_key:
+ data: "{{ libcontainers_apt_key }}"
+ keyring: /etc/apt/trusted.gpg.d/projectatomic.gpg
+
+ - name: Add kubic project libcontainers apt repo
+ become: yes
+ template:
+ dest: /etc/apt/sources.list.d/projectatomic.list
+ group: root
+ mode: 0644
+ owner: root
+ src: sources.list.j2
+ register: projectatomic_repo
+
+ - name: Run the equivalent of "apt-get update" as a separate step
+ apt:
+ update_cache: yes
+ when: projectatomic_repo is changed
- name: Install bindep
pip:
@@ -25,11 +49,16 @@
name: '{{ item }}'
state: present
loop:
- - libjemalloc1
- bubblewrap
- skopeo
- socat
+- name: Install distro-specific executor extra packages
+ package:
+ name: '{{ item }}'
+ state: present
+ loop: '{{ zuul_executor_extra_packages }}'
+
- name: Clone zuul repo
git:
repo: https://opendev.org/zuul/zuul
@@ -117,6 +146,11 @@
vars:
logrotate_file_name: /var/log/zuul/executor-debug.log
+- name: Install defaults file
+ template:
+ src: zuul-executor.defaults.j2
+ dest: /etc/default/zuul-executor
+
- name: Install init script
copy:
src: zuul-executor.init
diff --git a/playbooks/roles/zuul-executor/templates/sources.list.j2 b/playbooks/roles/zuul-executor/templates/sources.list.j2
new file mode 100644
index 0000000000..6f1084c931
--- /dev/null
+++ b/playbooks/roles/zuul-executor/templates/sources.list.j2
@@ -0,0 +1 @@
+deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_lsb.release }}/ /
diff --git a/playbooks/roles/zuul-executor/templates/zuul-executor.defaults.j2 b/playbooks/roles/zuul-executor/templates/zuul-executor.defaults.j2
new file mode 100644
index 0000000000..9a4656b241
--- /dev/null
+++ b/playbooks/roles/zuul-executor/templates/zuul-executor.defaults.j2
@@ -0,0 +1,4 @@
+PIDFILE=/var/run/zuul/executor.pid
+{% if ansible_distribution_version is version('20.04', '<') %}
+export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+{% endif %}
diff --git a/playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml b/playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml
new file mode 100644
index 0000000000..57596f3440
--- /dev/null
+++ b/playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml
@@ -0,0 +1,5 @@
+zuul_executor_ppas:
+ # We use later HWE kernels for better memory managment, requiring an
+ # updated AFS version which we install from our custom ppa.
+ - ppa:openstack-ci-core/openafs
+zuul_executor_extra_packages: []
diff --git a/playbooks/roles/zuul-executor/vars/default.yaml b/playbooks/roles/zuul-executor/vars/default.yaml
new file mode 100644
index 0000000000..f8e44c2964
--- /dev/null
+++ b/playbooks/roles/zuul-executor/vars/default.yaml
@@ -0,0 +1,12 @@
+zuul_executor_ppas:
+ # For bubblewrap
+ - ppa:openstack-ci-core/bubblewrap
+ # Temporary PPA needed for bpo-27945 while waiting for SRU to be published
+ - ppa:openstack-ci-core/python-bpo-27945-backport
+ # We use later HWE kernels for better memory managment, requiring an
+ # updated AFS version which we install from our custom ppa.
+ - ppa:openstack-ci-core/openafs
+ # For skopeo
+ - ppa:projectatomic/ppa
+zuul_executor_extra_packages:
+ - libjemalloc1
diff --git a/testinfra/test_base.py b/testinfra/test_base.py
index 3a86f0d826..c6f0e4f909 100644
--- a/testinfra/test_base.py
+++ b/testinfra/test_base.py
@@ -80,9 +80,9 @@ def test_iptables(host):
def test_ntp(host):
package = host.package("ntp")
- assert package.is_installed
+ if host.system_info.codename in ('xenial'):
+ assert package.is_installed
- if host.system_info.codename != 'bionic':
if host.system_info.distribution in ['ubuntu', 'debian']:
service = host.service("ntp")
else:
@@ -91,11 +91,18 @@ def test_ntp(host):
assert service.is_enabled
else:
+ assert not package.is_installed
+
service = host.service('systemd-timesyncd')
assert service.is_running
+ # Focal updates the status string to just say NTP
+ if host.system_info.codename == 'bionic':
+ stdout_string = 'systemd-timesyncd.service active'
+ else:
+ stdout_string = 'NTP service: active'
cmd = host.run("timedatectl status")
- assert 'systemd-timesyncd.service active: yes' in cmd.stdout
+ assert stdout_string in cmd.stdout
def test_snmp(host):