From 40755451b1d5d62a27f857b2af5b60e87dae591d Mon Sep 17 00:00:00 2001
From: alextricity25 <miguel.cantu@rackspace.com>
Date: Mon, 22 Aug 2016 17:11:48 -0500
Subject: [PATCH] Added successerator function to osa-multi-node-aio

The successerator function retries a failed playbook
that might of failed due to network saturation issues.
i.e. target host ssh unreachable errors.

Change-Id: I2164a22555474749228fcb278d34885cf154a743
---
 multi-node-aio/deploy-osa.sh | 11 +++++++++--
 multi-node-aio/functions.rc  | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/multi-node-aio/deploy-osa.sh b/multi-node-aio/deploy-osa.sh
index a9c161c4..aa562a95 100755
--- a/multi-node-aio/deploy-osa.sh
+++ b/multi-node-aio/deploy-osa.sh
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+MAX_RETRIES=${MAX_RETRIES:-5}
+
 # Load all functions
 source functions.rc
 
@@ -102,10 +104,15 @@ export ANSIBLE_FORKS=${ANSIBLE_FORKS:-15}
 pushd /opt/openstack-ansible/playbooks
 
 # Running the HAP play is done because it "may" be needed. Note: In Master its not.
-openstack-ansible haproxy-install.yml
+install_bits haproxy-install.yml
 
 # Setup everything else
-openstack-ansible setup-everything.yml
+for root_include in $(awk -F'include:' '{print $2}' setup-everything.yml); do
+  for include in $(awk -F'include:' '{print $2}' "${root_include}"); do
+    echo "[Executing \"${include}\" playbook]"
+    install_bits "${include}"
+  done
+done
 
 # This is optional and only being done to give the cloud networks and an image.
 #  The tempest install will work out of the box because the deployment is setup
diff --git a/multi-node-aio/functions.rc b/multi-node-aio/functions.rc
index 519fe185..7f992e0c 100755
--- a/multi-node-aio/functions.rc
+++ b/multi-node-aio/functions.rc
@@ -195,3 +195,36 @@ for node_type in $(get_all_types); do
   done
 done
 }
+
+function install_bits {
+successerator openstack-ansible ${ANSIBLE_PARAMETERS} $@
+}
+
+function successerator {
+set +e
+# Get the time taht the method was started
+OP_START_TIME=$(date +%s)
+#Set the initial return value to failure.
+false
+for ((RETRY=0; $? != 0 && RETRY < MAX_RETRIES; RETRY++)); do
+  if [ ${RETRY} -gt 1 ]; then
+    $@ -vvvv
+  else
+    $@
+  fi
+done
+
+# If max retries were hit, fail.
+if [ $? -ne 0 && [ ${RETRY} -eq ${MAX_RETRIES} ];then
+  echo -e "\n Hit maximum number of retries, giving up..\n"
+  exit
+fi
+
+# Print the time that the method completed.
+OP_TOTAL_SECONDS="$(( $(date +%s) - OP_START_TIME ))"
+REPORT_OUTPUT="${OP_TOTAL_SECONDS} seconds"
+REPORT_DATA+="- Operation: [ $@ ]\t${REPORT_OUTPUT}\tNumber of Attempts [ ${RETRY} ]\n"
+echo -e "Run Time = ${REPORT_OUTPUT}"
+set -e
+}
+