From 7c74729c5325966e4a521602a0b7b69623f7e96f Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 29 Oct 2019 19:25:47 +1100 Subject: [PATCH] AFS: Allow for remote vos release with localauth We constantly have problems with various timeouts on the release of our mirror volumes creating locked volumes or stuck transactions; this then requires significant manual intervention. This has been discussed multiple times, but this short exchange from #openafs probably sums it up best: Sep 11 13:32:35 The timeout problem is due to the fact that UV_ReleaseVolume performs multiple RPCs. vos acquires a token from the cache manager when it starts. it has no method of acquiring a new token if it expired during an RPC. Therefore, if the token did expire the remaining RPCs are performed unauthenticated. Without appropriate permissions the cleanup of the volservers, writing the updating VL entry will fail. Sep 11 13:33:59 A frequent solution is to deploy a remctld service which has access to issue vos commands as -localauth and then use remctld ACLs to restrict the identities of the processes that are permitted to request the volume release. Sep 11 14:37:28 Yeah, the -localauth tokens are pretty key for long-running stuff, at the moment. Indeed remctl [1] has been written to be the kerberos-based remote control AFS wrapper. However, it is complex to setup, uses a lot of Perl and it is unlikely to be familiar to very many people (making the footprint of people who can help us admin it low). Getting it wrong seems to be a pretty good vector for remote exploits. It does not seem to be a good fit. However, we can take a simpler approach. We can use Ansible to setup our afs server to allow a particular key to run a release script that wraps the "vos release -localauth" for us. With this in place, we can update the scripts that run on mirror-update to ssh remotely and call this, rather than call "vos release" directly. This implements this basic support for the remote script. A new key will be generated on mirror-update.opendev.org and it will be allowed to run the vos_release.sh script remotely; which filters the command to just do "vos release -localauth". After we have tested this, we can start using it in scripts. I think time will tell if we need locking or other features; this seems like the KISS place to start. [1] https://www.eyrie.org/~eagle/software/remctl/remctl.html Change-Id: I6c96f89c6f113362e6085febca70d58176f678e7 --- playbooks/remote_puppet_afs.yaml | 15 +++++++++++ playbooks/roles/vos-release/README.rst | 27 +++++++++++++++++++ .../roles/vos-release/files/vos_release.sh | 17 ++++++++++++ playbooks/roles/vos-release/tasks/main.yaml | 19 +++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 playbooks/roles/vos-release/README.rst create mode 100755 playbooks/roles/vos-release/files/vos_release.sh create mode 100644 playbooks/roles/vos-release/tasks/main.yaml diff --git a/playbooks/remote_puppet_afs.yaml b/playbooks/remote_puppet_afs.yaml index 6b57788f26..dc45e87606 100644 --- a/playbooks/remote_puppet_afs.yaml +++ b/playbooks/remote_puppet_afs.yaml @@ -9,3 +9,18 @@ strategy: free roles: - puppet + +- hosts: "mirror-update:!disabled" + name: "Create key for remote vos release" + tasks: + - name: Create vos release keypair + openssh_keypair: + path: /root/.ssh/id_vos_release + type: ed25519 + register: vos_release_keypair + +# Note: relies on vos_release_keypair installed to mirror above +- hosts: "afs:!disabled" + name: "Allow remote vos_release" + roles: + - vos-release \ No newline at end of file diff --git a/playbooks/roles/vos-release/README.rst b/playbooks/roles/vos-release/README.rst new file mode 100644 index 0000000000..27d2b9d1f4 --- /dev/null +++ b/playbooks/roles/vos-release/README.rst @@ -0,0 +1,27 @@ +vos release with localauth + +Install a user and script to do remote ``vos release`` with +``localauth`` authentication. This can avoid kerberos or AFS +timeouts. + +This relies on ``vos_release_keypair`` which is expected to be a +single keypair from the mirror-update host. It will allow that +keypair to run ``/usr/local/bin/vos_release.sh``, which filters the +incoming command. Releases are expected to be triggered on the update +host with:: + + ssh -i /root/.ssh/id_vos_release afs01.dfw.openstack.org vos release . + +Future work, if required + +* Allow multiple hosts to call the release script (i.e. handle + multiple keys). +* Implement locking within ``vos_release.sh`` script to prevent too + many simulatenous releases. + +**Role Variables** + +.. zuul:rolevar:: vos_release_keypair + + The authorized key to allow to run the + ``/usr/local/bin/vos_release.sh`` script diff --git a/playbooks/roles/vos-release/files/vos_release.sh b/playbooks/roles/vos-release/files/vos_release.sh new file mode 100755 index 0000000000..c2f2101b99 --- /dev/null +++ b/playbooks/roles/vos-release/files/vos_release.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [[ -z "${SSH_ORIGINAL_COMMAND}" ]]; then + echo "usage: vos release " + exit 1 +fi + +set -- $SSH_ORIGINAL_COMMAND + +if [[ $# != 3 || $1 != "vos" || $2 != "release" ]]; then + echo "usage: vos release " + exit 1 +fi + +vos release -v -localauth $3 + + diff --git a/playbooks/roles/vos-release/tasks/main.yaml b/playbooks/roles/vos-release/tasks/main.yaml new file mode 100644 index 0000000000..450df2daca --- /dev/null +++ b/playbooks/roles/vos-release/tasks/main.yaml @@ -0,0 +1,19 @@ +- name: Install vos release script + copy: + src: vos_relase.sh + dest: '/usr/local/bin/' + owner: root + group: root + mode: 0755 + +- name: Ensure update key + assert: + that: + - vos_release_keypair is defined + +- name: Install vos release key + authorized_key: + user: 'root' + state: present + key: '{{ vos_release_keypair["public_key"] }}' + key_options: 'command="/usr/local/bin/vos_release.sh",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty' \ No newline at end of file