From 19ea4603f4268ed300790ba09dc3832abf6b6920 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 2 Sep 2020 15:20:07 +1000 Subject: [PATCH] puppet: don't run module install steps multiple times It turns out you can't use "run_once" with the "free" strategy in Ansible. It actually warns you about this, if you're looking in the right place. The existing run-puppet role calls two things with "run_once:", both delegated to localhost -- cloning the ansible-role-puppet repo (so we can include_role: puppet) and installing the puppet modules (via install-ansible-roles role), which are copied from bridge to the remote side and run by ansible-role-puppet. With remote_puppet_else.yaml we are running all the puppet hosts at once with the "free" strategy. This means that these two tasks, both delegated to localhost (bridge) are actually running for every host. install-ansible-roles does a git clone, and thus we often see one of the clones bailing out with a git locking error, because the other host is running similtaneously. I8585a1af2dcc294c0e61fc45d9febb044e42151d tried to stop this with "run_once:" -- but as noted because it's running under the "free" strategy this is silently ignored. To get around this, split out the two copying steps into a new role "puppet-setup". To maintain the namespace, the "run-puppet" module is renamed to "puppet-run". Before each call of (now) "puppet-run", make sure we run "puppet-setup" just on localhost. Remove the run_once and delegation on "install-ansible-roles"; because this is now called from the playbook with localhost context. Change-Id: I3b1cea5a25974f56ea9202e252af7b8420f4adc9 --- inventory/service/group_vars/puppet.yaml | 3 --- playbooks/remote_puppet_afs.yaml | 8 +++++++- playbooks/remote_puppet_else.yaml | 8 +++++++- .../roles/install-ansible-roles/tasks/main.yaml | 2 -- playbooks/roles/puppet-run/README.rst | 13 +++++++++++++ .../{run-puppet => puppet-run}/tasks/main.yaml | 10 ---------- playbooks/roles/puppet-setup-ansible/README.rst | 5 +++++ .../roles/puppet-setup-ansible/tasks/main.yaml | 10 ++++++++++ playbooks/roles/run-puppet/README.rst | 12 ------------ playbooks/service-codesearch.yaml | 8 +++++++- playbooks/service-eavesdrop.yaml | 8 +++++++- playbooks/service-nodepool.yaml | 8 +++++++- 12 files changed, 63 insertions(+), 32 deletions(-) create mode 100644 playbooks/roles/puppet-run/README.rst rename playbooks/roles/{run-puppet => puppet-run}/tasks/main.yaml (54%) create mode 100644 playbooks/roles/puppet-setup-ansible/README.rst create mode 100644 playbooks/roles/puppet-setup-ansible/tasks/main.yaml delete mode 100644 playbooks/roles/run-puppet/README.rst diff --git a/inventory/service/group_vars/puppet.yaml b/inventory/service/group_vars/puppet.yaml index 68409237f7..37eab8292a 100644 --- a/inventory/service/group_vars/puppet.yaml +++ b/inventory/service/group_vars/puppet.yaml @@ -11,6 +11,3 @@ mgmt_hieradata: /etc/ansible/hosts mgmt_puppet_module_dir: /etc/puppet/modules puppet_hieradata_link_dest: /opt/system-config/hieradata puppet_nolog_sync: '{{ silence_synchronize }}' - -ansible_roles: - - puppet diff --git a/playbooks/remote_puppet_afs.yaml b/playbooks/remote_puppet_afs.yaml index eed1ea92d3..6f98aa5014 100644 --- a/playbooks/remote_puppet_afs.yaml +++ b/playbooks/remote_puppet_afs.yaml @@ -1,8 +1,14 @@ +- hosts: 'localhost:!disabled' + name: Install puppet role/modules + strategy: linear + roles: + - puppet-setup-ansible + - hosts: "afs:afsdb:!disabled" name: "AFS: run puppet on the AFS servers" strategy: free roles: - - run-puppet + - puppet-run - hosts: "mirror-update:!disabled" name: "Create key for remote vos release" diff --git a/playbooks/remote_puppet_else.yaml b/playbooks/remote_puppet_else.yaml index 36b2bdc1c0..aaff2c7b20 100644 --- a/playbooks/remote_puppet_else.yaml +++ b/playbooks/remote_puppet_else.yaml @@ -1,5 +1,11 @@ +- hosts: 'localhost:!disabled' + name: Install puppet role/modules + strategy: linear + roles: + - puppet-setup-ansible + - hosts: 'puppet:!review:!afs:!afsdb:!puppetmaster*:!nb*:!codesearch:!eavesdrop:!disabled' name: "Puppet-else: run puppet on all other servers" strategy: free roles: - - run-puppet + - puppet-run diff --git a/playbooks/roles/install-ansible-roles/tasks/main.yaml b/playbooks/roles/install-ansible-roles/tasks/main.yaml index 9a2d5c2e8f..1662a5e394 100644 --- a/playbooks/roles/install-ansible-roles/tasks/main.yaml +++ b/playbooks/roles/install-ansible-roles/tasks/main.yaml @@ -3,8 +3,6 @@ repo: '{{ ansible_role_src_root }}/src/opendev.org/opendev/ansible-role-{{ ansible_role }}' dest: '/etc/ansible/roles/{{ ansible_role }}' force: yes - delegate_to: localhost - run_once: true loop: '{{ ansible_roles }}' loop_control: loop_var: ansible_role diff --git a/playbooks/roles/puppet-run/README.rst b/playbooks/roles/puppet-run/README.rst new file mode 100644 index 0000000000..54753d634e --- /dev/null +++ b/playbooks/roles/puppet-run/README.rst @@ -0,0 +1,13 @@ +Run puppet on remote servers + +Omnibus role that takes care of installing puppet and then running +puppet. Uses include_role so that the installation of the puppet role +can run as the first task, then the puppet role can be used in a +following task. + +This role should run after ``puppet-setup-ansible`` + +.. zuul:rolevar:: manifest + :default: manifests/site.pp + + Puppet manifest file to run. diff --git a/playbooks/roles/run-puppet/tasks/main.yaml b/playbooks/roles/puppet-run/tasks/main.yaml similarity index 54% rename from playbooks/roles/run-puppet/tasks/main.yaml rename to playbooks/roles/puppet-run/tasks/main.yaml index 9573186f6a..9b4b4285ba 100644 --- a/playbooks/roles/run-puppet/tasks/main.yaml +++ b/playbooks/roles/puppet-run/tasks/main.yaml @@ -1,19 +1,9 @@ # Use include_role instead of roles: so that we can late-bind the roles list - include_role: name: iptables -- include_role: - name: install-ansible-roles - include_role: name: puppet-install - include_role: name: disable-puppet-agent - -- name: Run puppet module install - delegate_to: localhost - run_once: true - command: - cmd: bash install_modules.sh - chdir: /etc/puppet - - include_role: name: puppet diff --git a/playbooks/roles/puppet-setup-ansible/README.rst b/playbooks/roles/puppet-setup-ansible/README.rst new file mode 100644 index 0000000000..eb94dff6a5 --- /dev/null +++ b/playbooks/roles/puppet-setup-ansible/README.rst @@ -0,0 +1,5 @@ +Setup Ansible on this host to run puppet on remote hosts. + +Import the ansible-roles-puppet role for running puppet on remote +hosts and bring in the repository of required puppet modules. + diff --git a/playbooks/roles/puppet-setup-ansible/tasks/main.yaml b/playbooks/roles/puppet-setup-ansible/tasks/main.yaml new file mode 100644 index 0000000000..dea701c0cb --- /dev/null +++ b/playbooks/roles/puppet-setup-ansible/tasks/main.yaml @@ -0,0 +1,10 @@ +- include_role: + name: install-ansible-roles + vars: + ansible_roles: + - puppet + +- name: Run puppet module install on bridge + command: + cmd: bash install_modules.sh + chdir: /etc/puppet diff --git a/playbooks/roles/run-puppet/README.rst b/playbooks/roles/run-puppet/README.rst deleted file mode 100644 index b0a8bc5cb8..0000000000 --- a/playbooks/roles/run-puppet/README.rst +++ /dev/null @@ -1,12 +0,0 @@ -Run puppet on remote servers - -Omnibus role that takes care of installing the puppet role, -installing puppet and then running puppet. Uses include_role -so that the installation of the puppet role can run as the -first task, then the puppet role can be used in a following -task. - -.. zuul:rolevar:: manifest - :default: manifests/site.pp - - Puppet manifest file to run. diff --git a/playbooks/service-codesearch.yaml b/playbooks/service-codesearch.yaml index 27b2e4507d..fb6e7d9167 100644 --- a/playbooks/service-codesearch.yaml +++ b/playbooks/service-codesearch.yaml @@ -1,3 +1,9 @@ +- hosts: 'localhost:!disabled' + name: Install puppet role/modules + strategy: linear + roles: + - puppet-setup-ansible + - hosts: 'codesearch:!disabled' name: "codesearch: run puppet on codesearch" strategy: free @@ -5,5 +11,5 @@ - iptables - sync-project-config - pip3 - - name: run-puppet + - name: puppet-run manifest: /opt/system-config/production/manifests/codesearch.pp diff --git a/playbooks/service-eavesdrop.yaml b/playbooks/service-eavesdrop.yaml index 8be03a3139..124a82eae8 100644 --- a/playbooks/service-eavesdrop.yaml +++ b/playbooks/service-eavesdrop.yaml @@ -1,3 +1,9 @@ +- hosts: 'localhost:!disabled' + name: Install puppet role/modules + strategy: linear + roles: + - puppet-setup-ansible + - hosts: 'eavesdrop:!disabled' name: "eavesdrop: run puppet on eavesdrop" strategy: free @@ -8,5 +14,5 @@ - install-docker - accessbot - gerritbot - - name: run-puppet + - name: puppet-run manifest: /opt/system-config/production/manifests/eavesdrop.pp diff --git a/playbooks/service-nodepool.yaml b/playbooks/service-nodepool.yaml index 51f5178fa2..87b6125044 100644 --- a/playbooks/service-nodepool.yaml +++ b/playbooks/service-nodepool.yaml @@ -8,6 +8,12 @@ - configure-openstacksdk - nodepool-builder +- hosts: 'localhost:!disabled' + name: Install puppet role/modules + strategy: linear + roles: + - puppet-setup-ansible + - hosts: 'nb03.openstack.org:!disabled' name: "run puppet on all older servers" strategy: free @@ -16,7 +22,7 @@ - nodepool-base-legacy - configure-openstacksdk - configure-kubectl - - run-puppet + - puppet-run - hosts: nodepool-launcher:!disabled name: "Configure nodepool launchers"