Merge "Refactor infra-prod jobs for parallel running"

zuul.d/infra-prod.yaml

@@ -1,3 +1,7 @@
+# NOTE: job dependencies keep this running in parallel and are defined
+# in projects.yaml because it's easier to keep an overall view of
+# what's happening in there.
+
 # Make sure only one run of a system-config playbook happens at a time
 - semaphore:
     name: infra-prod-playbook
@@ -48,9 +52,6 @@
     name: infra-prod-base
     parent: infra-prod-playbook
     description: Run the base playbook everywhere.
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
     vars:
       playbook_name: base.yaml
       infra_prod_ansible_forks: 50
@@ -67,9 +68,6 @@
     description: Run letsencrypt.yaml playbook.
     vars:
       playbook_name: letsencrypt.yaml
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
     files:
       - inventory/
       - playbooks/letsencrypt.yaml
@@ -102,11 +100,6 @@
     parent: infra-prod-playbook
     description: Base job for most service playbooks.
     abstract: true
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
-      - name: infra-prod-letsencrypt
-        soft: true
 
 - job:
     name: infra-prod-service-bridge
@@ -203,11 +196,6 @@
     name: infra-prod-service-meetpad
     parent: infra-prod-service-base
     description: Run service-meetpad.yaml playbook.
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
-      - name: infra-prod-letsencrypt
-        soft: true
     vars:
       playbook_name: service-meetpad.yaml
     files:
@@ -358,13 +346,6 @@
       is changed.
     vars:
       playbook_name: service-zuul.yaml
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
-      - name: infra-prod-letsencrypt
-        soft: true
-      - name: infra-prod-manage-projects
-        soft: true
     files:
       - inventory/base
       - playbooks/install-ansible.yaml
@@ -386,13 +367,6 @@
     description: Run service-review.yaml playbook.
     vars:
       playbook_name: service-review.yaml
-    dependencies: &infra_prod_service_review_deps
-      - name: infra-prod-install-ansible
-        soft: true
-      - name: infra-prod-letsencrypt
-        soft: true
-      - name: system-config-promote-image-gerrit-3.3
-        soft: true
     files:
       - inventory/base
       - playbooks/service-review.yaml
@@ -409,13 +383,6 @@
     description: Run service-refstack.yaml playbook.
     vars:
       playbook_name: service-refstack.yaml
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
-      - name: infra-prod-letsencrypt
-        soft: true
-      - name: system-config-promote-image-refstack
-        soft: true
     files:
       - inventory/base
       - playbooks/service-refstack.yaml
@@ -435,15 +402,6 @@
     description: Run service-gitea.yaml playbook.
     vars:
       playbook_name: service-gitea.yaml
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
-      - name: infra-prod-letsencrypt
-        soft: true
-      - name: system-config-promote-image-gitea-init
-        soft: true
-      - name: system-config-promote-image-gitea
-        soft: true
     files:
       - inventory/base
       - playbooks/service-gitea.yaml
@@ -466,17 +424,6 @@
     required-projects:
       - opendev/system-config
       - openstack/project-config
-    dependencies:
-      - name: infra-prod-install-ansible
-        soft: true
-      - name: infra-prod-letsencrypt
-        soft: true
-      - name: system-config-promote-image-accessbot
-        soft: true
-      - name: system-config-promote-image-ircbot
-        soft: true
-      - name: system-config-promote-image-matrix-eavesdrop
-        soft: true
     vars:
       playbook_name: service-eavesdrop.yaml
     files: &infra_prod_eavesdrop_files
@@ -505,8 +452,6 @@
     required-projects:
       - opendev/system-config
       - openstack/project-config
-    dependencies:
-      - infra-prod-service-eavesdrop
     vars:
       playbook_name: run-accessbot.yaml
     files:
@@ -661,9 +606,6 @@
     required-projects:
       - opendev/ansible-role-cloud-launcher
       - opendev/system-config
-    dependencies:
-      - name: infra-prod-service-bridge
-        soft: true
     files:
       - playbooks/run_cloud_launcher.yaml
       - inventory/service/host_vars/bridge.openstack.org.yaml

zuul.d/project.yaml

@@ -347,10 +347,239 @@
         - system-config-promote-image-uwsgi-base-3.7-bullseye
         - system-config-promote-image-uwsgi-base-3.8-bullseye
         - system-config-promote-image-uwsgi-base-3.9-bullseye
+
+        # NOTE: infra-prod-* jobs have a hierarchy below that ensure
+        # they can run in parallel.  We are deliberately keeping their
+        # dependencies here rather than job definitions to help keep
+        # these relationships clear.
+
+        # This installs the ansible on bridge that all the infra-prod
+        # jobs will run with.  Note the jobs use this ansible to then
+        # run against zuul's checkout of system-config.
         - infra-prod-install-ansible
-        - infra-prod-base
+
-        - infra-prod-letsencrypt
+        # From now on, all jobs should depend on base
-        - infra-prod-manage-projects:
+        - infra-prod-base: &infra-prod-base
+            dependencies:
+              - name: infra-prod-install-ansible
+                soft: true
+
+        # Legacy puppet hosts
+        - infra-prod-remote-puppet-else: &infra-prod-remote-puppet-else
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+
+        #
+        # Only depends on base, or amongst themselves.
+        #
+
+        - infra-prod-service-bridge: &infra-prod-service-bridge
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+        - infra-prod-run-cloud-launcher: &infra-prod-run-cloud-launcher
+            dependencies:
+              # depends on the cloud config written out by
+              # service-bridge
+              - name: infra-prod-service-bridge
+                soft: true
+
+        - infra-prod-service-kerberos: &infra-prod-service-kerberos
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+        - infra-prod-service-afs: &infra-prod-service-afs
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+              # NOTE(ianw) in theory we'd want auth changes before
+              # updating services like openafs using them.  Not sure
+              # in practice this matters much; we very rarely change
+              # things here anyway.
+              - name: infra-prod-service-kerberos
+                soft: true
+
+        - infra-prod-service-nameserver: &infra-prod-service-nameserver
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+
+        - infra-prod-service-mirror-update: &infra-prod-service-mirror-update
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+
+        #
+        # Hosts using certificates and backups
+        #
+
+        # Hosts that backup should depend on this as this will create
+        # the users and deploy the keys required for the borg-backup
+        # role to work.
+        - infra-prod-service-borg-backup: &infra-prod-service-borg-backup
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+
+        # Hosts that have letsencrypt certs should depend on this, as
+        # it will write out the key material before they try to start
+        # services that depend on it.  For simplicity, we parent to
+        # this job.
+        - infra-prod-letsencrypt: &infra-prod-letsencrypt
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+              - name: infra-prod-service-nameserver
+                soft: true
+
+        # letsencrypt depdencies.  keep in alphabetical order
+        - infra-prod-service-codesearch: &infra-prod-service-codesearch
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-hound
+                soft: true
+        - infra-prod-service-eavesdrop: &infra-prod-service-eavesdrop
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-ircbot
+                soft: true
+              - name: system-config-promote-image-matrix-eavesdrop
+                soft: true
+        - infra-prod-service-etherpad: &infra-prod-service-etherpad
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-etherpad
+                soft: true
+        - infra-prod-service-gitea: &infra-prod-service-gitea
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-gitea
+                soft: true
+        - infra-prod-service-gitea-lb: &infra-prod-service-gitea-lb
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-haproxy-lb
+                soft: true
+        - infra-prod-service-grafana: &infra-prod-service-grafana
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-grafana
+                soft: true
+        - infra-prod-service-graphite: &infra-prod-service-graphite
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+        - infra-prod-service-meetpad: &infra-prod-service-meetpad
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+        - infra-prod-service-lists: &infra-prod-service-lists
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-grafana
+                soft: true
+        - infra-prod-service-mirror: &infra-prod-service-mirror
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+        - infra-prod-service-nodepool: &infra-prod-service-nodepool
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+        - infra-prod-service-static: &infra-prod-service-static
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+        - infra-prod-service-paste: &infra-prod-service-paste
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+        - infra-prod-service-registry: &infra-prod-service-registry
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+        - infra-prod-service-refstack: &infra-prod-service-refstack
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-refstack
+                soft: true
+        - infra-prod-service-review: &infra-prod-service-review
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-gerrit-3.3
+                soft: true
+        - infra-prod-service-zookeeper: &infra-prod-service-zookeeper
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+              - name: system-config-promote-image-zookeeper-statsd
+        - infra-prod-service-zuul: &infra-prod-service-zuul
+            dependencies:
+              - name: infra-prod-service-borg-backup
+                soft: true
+              - name: infra-prod-letsencrypt
+                soft: true
+              # should reconfigure after any project updates
+              - name: infra-prod-manage-projects
+                soft: true
+        - infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview
+            dependencies:
+              - name: infra-prod-letsencrypt
+                soft: true
+
+        #
+        # Jobs that run as secondary steps
+        #
+
+        # accessbot should run on a setup eavesdrop host
+        - infra-prod-run-accessbot: &infra-prod-run-accessbot
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+              - name: infra-prod-service-eavesdrop
+                soft: true
+              - name: system-config-promote-image-accessbot
+                soft: true
+
+        # manage-projects runs jeepyb etc. and should run on
+        # a setup review host.  also sets up gitea
+        - infra-prod-manage-projects: &infra-prod-manage-projects
+            dependencies:
+              - name: infra-prod-base
+                soft: true
+              - name: infra-prod-service-review
+                soft: true
+              - name: infra-prod-service-gitea
+                soft: true
+              - name: system-config-promote-image-gerrit-3.3
+                soft: true
+            # Note that this job also runs from project-config, so we
+            # match system-config specific files here rather than the
+            # job definition.
             files:
               - inventory/.*
               - playbooks/manage-projects.yaml
@@ -361,60 +590,7 @@
               - playbooks/roles/gitea-git-repos/
               - playbooks/roles/gerrit/defaults/main.yaml
               - playbooks/roles/gerrit/tasks/manage-projects.yaml
-            dependencies:
+
-              - name: system-config-promote-image-gerrit-3.3
-                soft: true
-              - name: infra-prod-install-ansible
-                soft: true
-        - infra-prod-service-bridge
-        - infra-prod-service-gitea-lb
-        - infra-prod-service-kerberos
-        - infra-prod-service-nameserver
-        - infra-prod-service-lists
-        - infra-prod-service-nodepool
-        - infra-prod-service-codesearch:
-            dependencies:
-              - name: infra-prod-letsencrypt
-                soft: true
-              - name: system-config-promote-image-hound
-                soft: true
-        - infra-prod-service-etherpad:
-            dependencies:
-              - name: infra-prod-install-ansible
-                soft: true
-              - name: infra-prod-letsencrypt
-                soft: true
-              - name: system-config-promote-image-etherpad
-                soft: true
-        - infra-prod-service-grafana:
-            dependencies:
-              - name: infra-prod-letsencrypt
-                soft: true
-              - name: system-config-promote-image-grafana
-                soft: true
-        - infra-prod-service-graphite
-        - infra-prod-service-meetpad
-        - infra-prod-service-mirror-update
-        - infra-prod-service-mirror
-        - infra-prod-service-static
-        - infra-prod-service-borg-backup
-        - infra-prod-service-paste
-        - infra-prod-service-registry
-        - infra-prod-service-refstack
-        - infra-prod-service-zookeeper
-        - infra-prod-service-zuul
-        - infra-prod-service-zuul-preview
-        - infra-prod-service-review
-        - infra-prod-service-gitea
-        - infra-prod-service-eavesdrop
-        - infra-prod-run-accessbot:
-            dependencies:
-              - infra-prod-service-eavesdrop
-              - name: system-config-promote-image-accessbot
-                soft: true
-        - infra-prod-service-afs
-        - infra-prod-remote-puppet-else
-        - infra-prod-run-cloud-launcher
     periodic:
       jobs:
         - developer-openstack-goaccess-report
@@ -428,51 +604,47 @@
         - tarballs-opendev-goaccess-report
         - zuul-ci-goaccess-report
         # Nightly runs of ansible things for catchup
+        # Keep in order from above
         - infra-prod-install-ansible
-        - infra-prod-base
+        - infra-prod-base: *infra-prod-base
-        - infra-prod-letsencrypt
+        - infra-prod-remote-puppet-else: *infra-prod-remote-puppet-else
-        - infra-prod-service-bridge
+        - infra-prod-letsencrypt: *infra-prod-letsencrypt
-        - infra-prod-service-gitea-lb
+        - infra-prod-service-bridge: *infra-prod-service-bridge
-        - infra-prod-service-nameserver
+        - infra-prod-run-cloud-launcher: *infra-prod-run-cloud-launcher
-        - infra-prod-service-lists
+        - infra-prod-service-kerberos: *infra-prod-service-kerberos
-        - infra-prod-service-etherpad
+        - infra-prod-service-afs: *infra-prod-service-afs
-        - infra-prod-service-meetpad
+        - infra-prod-service-nameserver: *infra-prod-service-nameserver
-        - infra-prod-service-kerberos
+        - infra-prod-service-mirror-update: *infra-prod-service-mirror-update
-        - infra-prod-service-mirror-update
+        - infra-prod-service-borg-backup: *infra-prod-service-borg-backup
-        - infra-prod-service-mirror
+        - infra-prod-letsencrypt: *infra-prod-letsencrypt
-        - infra-prod-service-paste
+        - infra-prod-service-codesearch: *infra-prod-service-codesearch
-        - infra-prod-service-static
+        - infra-prod-service-eavesdrop: *infra-prod-service-eavesdrop
-        - infra-prod-service-borg-backup
+        - infra-prod-service-etherpad: *infra-prod-service-etherpad
-        - infra-prod-service-zookeeper
+        - infra-prod-service-gitea: *infra-prod-service-gitea
-        - infra-prod-service-review
+        - infra-prod-service-gitea-lb: *infra-prod-service-gitea-lb
-        - infra-prod-service-gitea
+        - infra-prod-service-grafana: *infra-prod-service-grafana
-        - infra-prod-service-codesearch
+        - infra-prod-service-graphite: *infra-prod-service-graphite
-        - infra-prod-service-eavesdrop
+        - infra-prod-service-meetpad: *infra-prod-service-meetpad
-        - infra-prod-run-accessbot
+        - infra-prod-service-lists: *infra-prod-service-lists
-        - infra-prod-service-afs
+        - infra-prod-service-mirror: *infra-prod-service-mirror
-        - infra-prod-service-zuul-preview
+        - infra-prod-service-nodepool: *infra-prod-service-nodepool
-        - infra-prod-remote-puppet-else
+        - infra-prod-service-static: *infra-prod-service-static
-        - infra-prod-run-cloud-launcher
+        - infra-prod-service-paste: *infra-prod-service-paste
+        - infra-prod-service-registry: *infra-prod-service-registry
+        - infra-prod-service-refstack: *infra-prod-service-refstack
+        - infra-prod-service-review: *infra-prod-service-review
+        - infra-prod-service-zookeeper: *infra-prod-service-zookeeper
+        - infra-prod-service-zuul: *infra-prod-service-zuul
+        - infra-prod-service-zuul-preview: *infra-prod-service-zuul-preview
+        - infra-prod-run-accessbot: *infra-prod-run-accessbot
+        - infra-prod-manage-projects: *infra-prod-manage-projects
+
+
     opendev-prod-hourly:
       jobs:
         - infra-prod-install-ansible
-        - infra-prod-service-bridge:
+        - infra-prod-service-bridge: *infra-prod-service-bridge
-            dependencies:
+        - infra-prod-service-nodepool: *infra-prod-service-nodepool
-              - name: infra-prod-install-ansible
+        - infra-prod-service-registry: *infra-prod-service-registry
-                soft: true
+        - infra-prod-service-zuul: *infra-prod-service-zuul
-        - infra-prod-service-nodepool:
+        - infra-prod-service-eavesdrop: *infra-prod-service-eavesdrop
-            dependencies:
-              - name: infra-prod-install-ansible
-                soft: true
-        - infra-prod-service-registry:
-            dependencies:
-              - name: infra-prod-install-ansible
-                soft: true
-        - infra-prod-service-zuul:
-            dependencies:
-              - name: infra-prod-install-ansible
-                soft: true
-        - infra-prod-service-eavesdrop:
-            dependencies:
-              - name: infra-prod-install-ansible
-                soft: true