From e7387a6baa86e96c817c7c6272050e7b91e6be07 Mon Sep 17 00:00:00 2001
From: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
Date: Thu, 30 Aug 2018 21:21:24 +0100
Subject: [PATCH] MNAIO: Make galera startup cleanly when using images

In an ideal state, if the galera containers are shut down
cleanly, they will leave behind a gvwstate.dat file on each
node which provides the cluster member details so that it
can automatically start up again without intervention.

However, when imaging the MNAIO systems we only interact
with the hosts, so the galera containers sometimes do no
shut down cleanly.

To cater for this, we inspect the disk images for the
primary component, then build the gvwstate.dat file for
the other galera containers. With those put back into the
image, when the VM's start, the cluster forms immediately.

References:
http://galeracluster.com/documentation-webpages/pcrecovery.html
http://galeracluster.com/documentation-webpages/restartingcluster.html

Change-Id: Icfe067607baefd661147f3c22ce846f06fff7c60
---
 multi-node-aio/playbooks/deploy-vms.yml       | 19 +++--
 .../playbooks/kvm/prepare-image-galera.sh     | 70 +++++++++++++++++++
 2 files changed, 83 insertions(+), 6 deletions(-)
 create mode 100644 multi-node-aio/playbooks/kvm/prepare-image-galera.sh

diff --git a/multi-node-aio/playbooks/deploy-vms.yml b/multi-node-aio/playbooks/deploy-vms.yml
index d5992d61..c07122e4 100644
--- a/multi-node-aio/playbooks/deploy-vms.yml
+++ b/multi-node-aio/playbooks/deploy-vms.yml
@@ -118,11 +118,16 @@
             - hostvars[item]['server_vm'] | default(false) | bool
           with_items: "{{ groups['pxe_servers'] }}"
 
-        # Note (odyssey4me):
-        # This will only work on a host which has
-        # libguestfs >= 1.35.2 and >= 1.34.1
-        # Ubuntu bionic works, but xenial does not (even with UCA).
-        # ref: https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1615337.
+    # Note (odyssey4me):
+    # This will only work on a host which has
+    # libguestfs >= 1.35.2 and >= 1.34.1
+    # Ubuntu bionic works, but xenial does not (even with UCA).
+    # ref: https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1615337.
+    - name: Prepare file-based disk images
+      when:
+        - default_vm_disk_mode == "file"
+        - vm_use_snapshot | bool
+      block:
         - name: Inject the host ssh key into the VM disk image
           command: >-
             virt-sysprep
@@ -130,10 +135,12 @@
             --ssh-inject root:file:/root/.ssh/id_rsa.pub
             --add {{ _virt_pools.pools.default.path | default('/data/images') }}/{{ hostvars[item]['server_hostname'] }}.img
           when:
-            - vm_use_snapshot | bool
             - hostvars[item]['server_vm'] | default(false) | bool
           with_items: "{{ groups['pxe_servers'] }}"
 
+        - name: Prepare the galera containers for startup
+          script: kvm/prepare-image-galera.sh
+
     - name: Wait for guest capabilities to appear
       command: "virsh capabilities"
       register: virsh_caps
diff --git a/multi-node-aio/playbooks/kvm/prepare-image-galera.sh b/multi-node-aio/playbooks/kvm/prepare-image-galera.sh
new file mode 100644
index 00000000..9ebd37c4
--- /dev/null
+++ b/multi-node-aio/playbooks/kvm/prepare-image-galera.sh
@@ -0,0 +1,70 @@
+#!/bin/bash -ex
+
+# provide default images to inspect
+infra_images="/data/images/infra1.img /data/images/infra2.img /data/images/infra3.img"
+
+# declare an array to map images to container names
+declare -A image_map
+
+# declare an array to map container names to uuid's
+declare -A uuid_map
+
+# at this stage, no galera container is the master
+master_cnt=""
+
+# get the list of galera container names
+for img in ${infra_images}; do
+  image_map[${img}]="$(virt-ls --add ${img} --mount /dev/vmvg00/openstack00 / | grep galera_container)"
+done
+
+# get the gvwstate.dat files from the image and
+# put it into a local folder using the same name
+# as the container
+for img in ${infra_images}; do
+  mkdir -p /tmp/${image_map[$img]}
+  guestfish --ro --add ${img} --mount /dev/vmvg00/openstack00 glob copy-out /${image_map[$img]}/*.dat /tmp/${image_map[$img]}/
+done
+
+# work through the existing gvwstate files
+# there may be more than one, so we need to
+# find the one holding the view_id
+for cnt in $(ls -1 /tmp | grep galera_container); do
+  gvwstate_path="/tmp/${cnt}/gvwstate.dat"
+  if [[ -e ${gvwstate_path} ]]; then
+    my_uuid=$(awk '/^my_uuid:/ { print $2 }' ${gvwstate_path})
+    view_id=$(awk '/^view_id:/ { print $3 }' ${gvwstate_path})
+    if [[ "${my_uuid}" == "${view_id}" ]]; then
+      master_gvwstate_path=${gvwstate_path}
+      master_cnt=${cnt}
+    fi
+  fi
+  if [[ "${cnt}" == "${master_cnt}" ]]; then
+    uuid_map[${cnt}]=${my_uuid}
+  else
+    uuid_map[${cnt}]=$(uuidgen)
+  fi
+done
+
+# prepare a new master in a temporary location
+tmp_gvwstate="/tmp/gvwstate.dat"
+cp ${master_gvwstate_path} ${tmp_gvwstate}
+member_num=$(awk '/^member: '${my_uuid}'/ {print $3}' ${tmp_gvwstate})
+
+# clear the existing members
+sed -i.bak '/^member:/d' ${tmp_gvwstate}
+
+# insert the new set of members
+for cnt_uuid in "${uuid_map[@]}"; do
+sed -i.bak "/^#vwend$/i \\
+member: ${cnt_uuid} ${member_num}" ${tmp_gvwstate}
+done
+
+# copy the new version to each location
+for cnt in "${!uuid_map[@]}"; do
+  sed "s/my_uuid: .*/my_uuid: ${uuid_map[$cnt]}/" ${tmp_gvwstate} > /tmp/${cnt}/gvwstate.dat
+done
+
+# put the gvwstate.dat files back into the image
+for img in ${infra_images}; do
+  guestfish --rw --add ${img} --mount /dev/vmvg00/openstack00 copy-in /tmp/${image_map[$img]}/gvwstate.dat  /${image_map[$img]}/
+done