From 4e9c1c5fd857bff76f85104a13f2c4bd07826a82 Mon Sep 17 00:00:00 2001
From: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
Date: Tue, 4 Sep 2018 19:19:50 +0100
Subject: [PATCH] MNAIO: Make the galera image prep more robust

Unfortunately guestfish may error out silently (no return code
of 1), making hunting down the error a bit obscure. To combat
this we add a bunch of stdout output to the script, and look
for that final step to validate success. To make this work, we
need to copy the script over and execute it with the command
module, because the script module puts everything into stderr.

Change-Id: I8e514ceb2462870721745c9445ec149864a45f4d
---
 multi-node-aio/playbooks/deploy-vms.yml       | 17 ++++++++++++++-
 .../playbooks/kvm/prepare-image-galera.sh     | 21 +++++++++++++------
 2 files changed, 31 insertions(+), 7 deletions(-)
 mode change 100644 => 100755 multi-node-aio/playbooks/kvm/prepare-image-galera.sh

diff --git a/multi-node-aio/playbooks/deploy-vms.yml b/multi-node-aio/playbooks/deploy-vms.yml
index c07122e4..afbe649f 100644
--- a/multi-node-aio/playbooks/deploy-vms.yml
+++ b/multi-node-aio/playbooks/deploy-vms.yml
@@ -138,8 +138,23 @@
             - hostvars[item]['server_vm'] | default(false) | bool
           with_items: "{{ groups['pxe_servers'] }}"
 
+        - name: Copy over prepare-image-galera.sh
+          copy:
+            src: kvm/prepare-image-galera.sh
+            dest: /opt/prepare-image-galera.sh
+            mode: "0755"
+
         - name: Prepare the galera containers for startup
-          script: kvm/prepare-image-galera.sh
+          command: /opt/prepare-image-galera.sh
+          register: _galera_prepare
+
+        # guestfissh does not always give a return code which indicates
+        # failure, so we look for our final stdout output as an indicator
+        - name: Fail if the preparation script did not complete
+          fail:
+            msg: "The galera container preparation failed."
+          when:
+            - "'Image preparation completed.' not in _galera_prepare.stdout_lines"
 
     - name: Wait for guest capabilities to appear
       command: "virsh capabilities"
diff --git a/multi-node-aio/playbooks/kvm/prepare-image-galera.sh b/multi-node-aio/playbooks/kvm/prepare-image-galera.sh
old mode 100644
new mode 100755
index 9ebd37c4..bc8adfcc
--- a/multi-node-aio/playbooks/kvm/prepare-image-galera.sh
+++ b/multi-node-aio/playbooks/kvm/prepare-image-galera.sh
@@ -1,5 +1,8 @@
 #!/bin/bash -ex
 
+# clean up from any previous attempts
+rm -rf /tmp/*galera* /tmp/gvw*
+
 # provide default images to inspect
 infra_images="/data/images/infra1.img /data/images/infra2.img /data/images/infra3.img"
 
@@ -12,7 +15,7 @@ declare -A uuid_map
 # at this stage, no galera container is the master
 master_cnt=""
 
-# get the list of galera container names
+echo "Getting the list of galera container names."
 for img in ${infra_images}; do
   image_map[${img}]="$(virt-ls --add ${img} --mount /dev/vmvg00/openstack00 / | grep galera_container)"
 done
@@ -22,6 +25,7 @@ done
 # as the container
 for img in ${infra_images}; do
   mkdir -p /tmp/${image_map[$img]}
+  echo "Copying *.dat from ${img} into /tmp/${image_map[$img]}/"
   guestfish --ro --add ${img} --mount /dev/vmvg00/openstack00 glob copy-out /${image_map[$img]}/*.dat /tmp/${image_map[$img]}/
 done
 
@@ -31,9 +35,11 @@ done
 for cnt in $(ls -1 /tmp | grep galera_container); do
   gvwstate_path="/tmp/${cnt}/gvwstate.dat"
   if [[ -e ${gvwstate_path} ]]; then
+    echo "Found ${gvwstate_path}, extracting my_uuid/view_id."
     my_uuid=$(awk '/^my_uuid:/ { print $2 }' ${gvwstate_path})
     view_id=$(awk '/^view_id:/ { print $3 }' ${gvwstate_path})
     if [[ "${my_uuid}" == "${view_id}" ]]; then
+      echo "Found galera master in ${gvwstate_path}."
       master_gvwstate_path=${gvwstate_path}
       master_cnt=${cnt}
     fi
@@ -45,26 +51,29 @@ for cnt in $(ls -1 /tmp | grep galera_container); do
   fi
 done
 
-# prepare a new master in a temporary location
+echo "Prepare a new master gvwstate.dat in a temporary location."
 tmp_gvwstate="/tmp/gvwstate.dat"
 cp ${master_gvwstate_path} ${tmp_gvwstate}
 member_num=$(awk '/^member: '${my_uuid}'/ {print $3}' ${tmp_gvwstate})
 
-# clear the existing members
+echo "Clearing the existing members."
 sed -i.bak '/^member:/d' ${tmp_gvwstate}
 
-# insert the new set of members
+echo "Inserting the new set of members."
 for cnt_uuid in "${uuid_map[@]}"; do
 sed -i.bak "/^#vwend$/i \\
 member: ${cnt_uuid} ${member_num}" ${tmp_gvwstate}
 done
 
-# copy the new version to each location
+echo "Copying the new gvwstate.dat version to each working location."
 for cnt in "${!uuid_map[@]}"; do
   sed "s/my_uuid: .*/my_uuid: ${uuid_map[$cnt]}/" ${tmp_gvwstate} > /tmp/${cnt}/gvwstate.dat
 done
 
-# put the gvwstate.dat files back into the image
+echo "Putting the gvwstate.dat files back into the images."
 for img in ${infra_images}; do
+  echo "Copying /tmp/${image_map[$img]}/gvwstate.dat into ${img}."
   guestfish --rw --add ${img} --mount /dev/vmvg00/openstack00 copy-in /tmp/${image_map[$img]}/gvwstate.dat  /${image_map[$img]}/
 done
+
+echo "Image preparation completed."