From d0467bfc98bf433cef50c21ffd2b9d1eb43a2462 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 21 Oct 2021 14:43:48 +1100 Subject: [PATCH] Refactor infra-prod jobs for parallel running Refactor the infra-prod jobs to specify dependencies so they can run in parallel. Change-Id: I8f6150ec2f696933c93560c11fed0fd16b11bf65 --- zuul.d/infra-prod.yaml | 66 +------- zuul.d/project.yaml | 374 ++++++++++++++++++++++++++++++----------- 2 files changed, 277 insertions(+), 163 deletions(-) diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index ca1a52e854..6d1a505ddb 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -1,3 +1,7 @@ +# NOTE: job dependencies keep this running in parallel and are defined +# in projects.yaml because it's easier to keep an overall view of +# what's happening in there. + # Make sure only one run of a system-config playbook happens at a time - semaphore: name: infra-prod-playbook @@ -48,9 +52,6 @@ name: infra-prod-base parent: infra-prod-playbook description: Run the base playbook everywhere. - dependencies: - - name: infra-prod-install-ansible - soft: true vars: playbook_name: base.yaml infra_prod_ansible_forks: 50 @@ -67,9 +68,6 @@ description: Run letsencrypt.yaml playbook. vars: playbook_name: letsencrypt.yaml - dependencies: - - name: infra-prod-install-ansible - soft: true files: - inventory/ - playbooks/letsencrypt.yaml @@ -102,11 +100,6 @@ parent: infra-prod-playbook description: Base job for most service playbooks. abstract: true - dependencies: - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true - job: name: infra-prod-service-bridge @@ -203,11 +196,6 @@ name: infra-prod-service-meetpad parent: infra-prod-service-base description: Run service-meetpad.yaml playbook. - dependencies: - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true vars: playbook_name: service-meetpad.yaml files: @@ -358,13 +346,6 @@ is changed. vars: playbook_name: service-zuul.yaml - dependencies: - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true - - name: infra-prod-manage-projects - soft: true files: - inventory/base - playbooks/install-ansible.yaml @@ -386,13 +367,6 @@ description: Run service-review.yaml playbook. vars: playbook_name: service-review.yaml - dependencies: &infra_prod_service_review_deps - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true - - name: system-config-promote-image-gerrit-3.3 - soft: true files: - inventory/base - playbooks/service-review.yaml @@ -409,13 +383,6 @@ description: Run service-refstack.yaml playbook. vars: playbook_name: service-refstack.yaml - dependencies: - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true - - name: system-config-promote-image-refstack - soft: true files: - inventory/base - playbooks/service-refstack.yaml @@ -435,15 +402,6 @@ description: Run service-gitea.yaml playbook. vars: playbook_name: service-gitea.yaml - dependencies: - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true - - name: system-config-promote-image-gitea-init - soft: true - - name: system-config-promote-image-gitea - soft: true files: - inventory/base - playbooks/service-gitea.yaml @@ -466,17 +424,6 @@ required-projects: - opendev/system-config - openstack/project-config - dependencies: - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true - - name: system-config-promote-image-accessbot - soft: true - - name: system-config-promote-image-ircbot - soft: true - - name: system-config-promote-image-matrix-eavesdrop - soft: true vars: playbook_name: service-eavesdrop.yaml files: &infra_prod_eavesdrop_files @@ -505,8 +452,6 @@ required-projects: - opendev/system-config - openstack/project-config - dependencies: - - infra-prod-service-eavesdrop vars: playbook_name: run-accessbot.yaml files: @@ -661,9 +606,6 @@ required-projects: - opendev/ansible-role-cloud-launcher - opendev/system-config - dependencies: - - name: infra-prod-service-bridge - soft: true files: - playbooks/run_cloud_launcher.yaml - inventory/service/host_vars/bridge.openstack.org.yaml diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 39d507e3b1..c36042eef3 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -347,10 +347,239 @@ - system-config-promote-image-uwsgi-base-3.7-bullseye - system-config-promote-image-uwsgi-base-3.8-bullseye - system-config-promote-image-uwsgi-base-3.9-bullseye + + # NOTE: infra-prod-* jobs have a hierarchy below that ensure + # they can run in parallel. We are deliberately keeping their + # dependencies here rather than job definitions to help keep + # these relationships clear. + + # This installs the ansible on bridge that all the infra-prod + # jobs will run with. Note the jobs use this ansible to then + # run against zuul's checkout of system-config. - infra-prod-install-ansible - - infra-prod-base - - infra-prod-letsencrypt - - infra-prod-manage-projects: + + # From now on, all jobs should depend on base + - infra-prod-base: &infra-prod-base + dependencies: + - name: infra-prod-install-ansible + soft: true + + # Legacy puppet hosts + - infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else + dependencies: + - name: infra-prod-base + soft: true + + # + # Only depends on base, or amongst themselves. + # + + - infra-prod-service-bridge: &infra-prod-service-bridge + dependencies: + - name: infra-prod-base + soft: true + - infra-prod-run-cloud-launcher: &infra-prod-run-cloud-launcher + dependencies: + # depends on the cloud config written out by + # service-bridge + - name: infra-prod-service-bridge + soft: true + + - infra-prod-service-kerberos: &infra-prod-service-kerberos + dependencies: + - name: infra-prod-base + soft: true + - infra-prod-service-afs: &infra-prod-service-afs + dependencies: + - name: infra-prod-base + soft: true + # NOTE(ianw) in theory we'd want auth changes before + # updating services like openafs using them. Not sure + # in practice this matters much; we very rarely change + # things here anyway. + - name: infra-prod-service-kerberos + soft: true + + - infra-prod-service-nameserver: &infra-prod-service-nameserver + dependencies: + - name: infra-prod-base + soft: true + + - infra-prod-service-mirror-update: &infra-prod-service-mirror-update + dependencies: + - name: infra-prod-base + soft: true + + # + # Hosts using certificates and backups + # + + # Hosts that backup should depend on this as this will create + # the users and deploy the keys required for the borg-backup + # role to work. + - infra-prod-service-borg-backup: &infra-prod-service-borg-backup + dependencies: + - name: infra-prod-base + soft: true + + # Hosts that have letsencrypt certs should depend on this, as + # it will write out the key material before they try to start + # services that depend on it. For simplicity, we parent to + # this job. + - infra-prod-letsencrypt: &infra-prod-letsencrypt + dependencies: + - name: infra-prod-base + soft: true + - name: infra-prod-service-nameserver + soft: true + + # letsencrypt depdencies. keep in alphabetical order + - infra-prod-service-codesearch: &infra-prod-service-codesearch + dependencies: + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-hound + soft: true + - infra-prod-service-eavesdrop: &infra-prod-service-eavesdrop + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-ircbot + soft: true + - name: system-config-promote-image-matrix-eavesdrop + soft: true + - infra-prod-service-etherpad: &infra-prod-service-etherpad + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-etherpad + soft: true + - infra-prod-service-gitea: &infra-prod-service-gitea + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-gitea + soft: true + - infra-prod-service-gitea-lb: &infra-prod-service-gitea-lb + dependencies: + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-haproxy-lb + soft: true + - infra-prod-service-grafana: &infra-prod-service-grafana + dependencies: + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-grafana + soft: true + - infra-prod-service-graphite: &infra-prod-service-graphite + dependencies: + - name: infra-prod-letsencrypt + soft: true + - infra-prod-service-meetpad: &infra-prod-service-meetpad + dependencies: + - name: infra-prod-letsencrypt + soft: true + - infra-prod-service-lists: &infra-prod-service-lists + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-grafana + soft: true + - infra-prod-service-mirror: &infra-prod-service-mirror + dependencies: + - name: infra-prod-letsencrypt + soft: true + - infra-prod-service-nodepool: &infra-prod-service-nodepool + dependencies: + - name: infra-prod-letsencrypt + soft: true + - infra-prod-service-static: &infra-prod-service-static + dependencies: + - name: infra-prod-letsencrypt + soft: true + - infra-prod-service-paste: &infra-prod-service-paste + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + - infra-prod-service-registry: &infra-prod-service-registry + dependencies: + - name: infra-prod-letsencrypt + soft: true + - infra-prod-service-refstack: &infra-prod-service-refstack + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-refstack + soft: true + - infra-prod-service-review: &infra-prod-service-review + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-gerrit-3.3 + soft: true + - infra-prod-service-zookeeper: &infra-prod-service-zookeeper + dependencies: + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-zookeeper-statsd + - infra-prod-service-zuul: &infra-prod-service-zuul + dependencies: + - name: infra-prod-service-borg-backup + soft: true + - name: infra-prod-letsencrypt + soft: true + # should reconfigure after any project updates + - name: infra-prod-manage-projects + soft: true + - infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview + dependencies: + - name: infra-prod-letsencrypt + soft: true + + # + # Jobs that run as secondary steps + # + + # accessbot should run on a setup eavesdrop host + - infra-prod-run-accessbot: &infra-prod-run-accessbot + dependencies: + - name: infra-prod-base + soft: true + - name: infra-prod-service-eavesdrop + soft: true + - name: system-config-promote-image-accessbot + soft: true + + # manage-projects runs jeepyb etc. and should run on + # a setup review host. also sets up gitea + - infra-prod-manage-projects: &infra-prod-manage-projects + dependencies: + - name: infra-prod-base + soft: true + - name: infra-prod-service-review + soft: true + - name: infra-prod-service-gitea + soft: true + - name: system-config-promote-image-gerrit-3.3 + soft: true + # Note that this job also runs from project-config, so we + # match system-config specific files here rather than the + # job definition. files: - inventory/.* - playbooks/manage-projects.yaml @@ -361,60 +590,7 @@ - playbooks/roles/gitea-git-repos/ - playbooks/roles/gerrit/defaults/main.yaml - playbooks/roles/gerrit/tasks/manage-projects.yaml - dependencies: - - name: system-config-promote-image-gerrit-3.3 - soft: true - - name: infra-prod-install-ansible - soft: true - - infra-prod-service-bridge - - infra-prod-service-gitea-lb - - infra-prod-service-kerberos - - infra-prod-service-nameserver - - infra-prod-service-lists - - infra-prod-service-nodepool - - infra-prod-service-codesearch: - dependencies: - - name: infra-prod-letsencrypt - soft: true - - name: system-config-promote-image-hound - soft: true - - infra-prod-service-etherpad: - dependencies: - - name: infra-prod-install-ansible - soft: true - - name: infra-prod-letsencrypt - soft: true - - name: system-config-promote-image-etherpad - soft: true - - infra-prod-service-grafana: - dependencies: - - name: infra-prod-letsencrypt - soft: true - - name: system-config-promote-image-grafana - soft: true - - infra-prod-service-graphite - - infra-prod-service-meetpad - - infra-prod-service-mirror-update - - infra-prod-service-mirror - - infra-prod-service-static - - infra-prod-service-borg-backup - - infra-prod-service-paste - - infra-prod-service-registry - - infra-prod-service-refstack - - infra-prod-service-zookeeper - - infra-prod-service-zuul - - infra-prod-service-zuul-preview - - infra-prod-service-review - - infra-prod-service-gitea - - infra-prod-service-eavesdrop - - infra-prod-run-accessbot: - dependencies: - - infra-prod-service-eavesdrop - - name: system-config-promote-image-accessbot - soft: true - - infra-prod-service-afs - - infra-prod-remote-puppet-else - - infra-prod-run-cloud-launcher + periodic: jobs: - developer-openstack-goaccess-report @@ -428,51 +604,47 @@ - tarballs-opendev-goaccess-report - zuul-ci-goaccess-report # Nightly runs of ansible things for catchup + # Keep in order from above - infra-prod-install-ansible - - infra-prod-base - - infra-prod-letsencrypt - - infra-prod-service-bridge - - infra-prod-service-gitea-lb - - infra-prod-service-nameserver - - infra-prod-service-lists - - infra-prod-service-etherpad - - infra-prod-service-meetpad - - infra-prod-service-kerberos - - infra-prod-service-mirror-update - - infra-prod-service-mirror - - infra-prod-service-paste - - infra-prod-service-static - - infra-prod-service-borg-backup - - infra-prod-service-zookeeper - - infra-prod-service-review - - infra-prod-service-gitea - - infra-prod-service-codesearch - - infra-prod-service-eavesdrop - - infra-prod-run-accessbot - - infra-prod-service-afs - - infra-prod-service-zuul-preview - - infra-prod-remote-puppet-else - - infra-prod-run-cloud-launcher + - infra-prod-base: *infra-prod-base + - infra-prod-remote-puppet-else: *infra-prod-remote-puppet-else + - infra-prod-letsencrypt: *infra-prod-letsencrypt + - infra-prod-service-bridge: *infra-prod-service-bridge + - infra-prod-run-cloud-launcher: *infra-prod-run-cloud-launcher + - infra-prod-service-kerberos: *infra-prod-service-kerberos + - infra-prod-service-afs: *infra-prod-service-afs + - infra-prod-service-nameserver: *infra-prod-service-nameserver + - infra-prod-service-mirror-update: *infra-prod-service-mirror-update + - infra-prod-service-borg-backup: *infra-prod-service-borg-backup + - infra-prod-letsencrypt: *infra-prod-letsencrypt + - infra-prod-service-codesearch: *infra-prod-service-codesearch + - infra-prod-service-eavesdrop: *infra-prod-service-eavesdrop + - infra-prod-service-etherpad: *infra-prod-service-etherpad + - infra-prod-service-gitea: *infra-prod-service-gitea + - infra-prod-service-gitea-lb: *infra-prod-service-gitea-lb + - infra-prod-service-grafana: *infra-prod-service-grafana + - infra-prod-service-graphite: *infra-prod-service-graphite + - infra-prod-service-meetpad: *infra-prod-service-meetpad + - infra-prod-service-lists: *infra-prod-service-lists + - infra-prod-service-mirror: *infra-prod-service-mirror + - infra-prod-service-nodepool: *infra-prod-service-nodepool + - infra-prod-service-static: *infra-prod-service-static + - infra-prod-service-paste: *infra-prod-service-paste + - infra-prod-service-registry: *infra-prod-service-registry + - infra-prod-service-refstack: *infra-prod-service-refstack + - infra-prod-service-review: *infra-prod-service-review + - infra-prod-service-zookeeper: *infra-prod-service-zookeeper + - infra-prod-service-zuul: *infra-prod-service-zuul + - infra-prod-service-zuul-preview: *infra-prod-service-zuul-preview + - infra-prod-run-accessbot: *infra-prod-run-accessbot + - infra-prod-manage-projects: *infra-prod-manage-projects + + opendev-prod-hourly: jobs: - infra-prod-install-ansible - - infra-prod-service-bridge: - dependencies: - - name: infra-prod-install-ansible - soft: true - - infra-prod-service-nodepool: - dependencies: - - name: infra-prod-install-ansible - soft: true - - infra-prod-service-registry: - dependencies: - - name: infra-prod-install-ansible - soft: true - - infra-prod-service-zuul: - dependencies: - - name: infra-prod-install-ansible - soft: true - - infra-prod-service-eavesdrop: - dependencies: - - name: infra-prod-install-ansible - soft: true + - infra-prod-service-bridge: *infra-prod-service-bridge + - infra-prod-service-nodepool: *infra-prod-service-nodepool + - infra-prod-service-registry: *infra-prod-service-registry + - infra-prod-service-zuul: *infra-prod-service-zuul + - infra-prod-service-eavesdrop: *infra-prod-service-eavesdrop