Added successerator function to osa-multi-node-aio

The successerator function retries a failed playbook
that might of failed due to network saturation issues.
i.e. target host ssh unreachable errors.

Change-Id: I2164a22555474749228fcb278d34885cf154a743

multi-node-aio/deploy-osa.sh

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+MAX_RETRIES=${MAX_RETRIES:-5}
+
 # Load all functions
 source functions.rc
 
@@ -102,10 +104,15 @@ export ANSIBLE_FORKS=${ANSIBLE_FORKS:-15}
 pushd /opt/openstack-ansible/playbooks
 
 # Running the HAP play is done because it "may" be needed. Note: In Master its not.
-openstack-ansible haproxy-install.yml
+install_bits haproxy-install.yml
 
 # Setup everything else
-openstack-ansible setup-everything.yml
+for root_include in $(awk -F'include:' '{print $2}' setup-everything.yml); do
+  for include in $(awk -F'include:' '{print $2}' "${root_include}"); do
+    echo "[Executing \"${include}\" playbook]"
+    install_bits "${include}"
+  done
+done
 
 # This is optional and only being done to give the cloud networks and an image.
 #  The tempest install will work out of the box because the deployment is setup

multi-node-aio/functions.rc

@@ -195,3 +195,36 @@ for node_type in $(get_all_types); do
   done
 done
 }
+
+function install_bits {
+successerator openstack-ansible ${ANSIBLE_PARAMETERS} $@
+}
+
+function successerator {
+set +e
+# Get the time taht the method was started
+OP_START_TIME=$(date +%s)
+#Set the initial return value to failure.
+false
+for ((RETRY=0; $? != 0 && RETRY < MAX_RETRIES; RETRY++)); do
+  if [ ${RETRY} -gt 1 ]; then
+    $@ -vvvv
+  else
+    $@
+  fi
+done
+
+# If max retries were hit, fail.
+if [ $? -ne 0 && [ ${RETRY} -eq ${MAX_RETRIES} ];then
+  echo -e "\n Hit maximum number of retries, giving up..\n"
+  exit
+fi
+
+# Print the time that the method completed.
+OP_TOTAL_SECONDS="$(( $(date +%s) - OP_START_TIME ))"
+REPORT_OUTPUT="${OP_TOTAL_SECONDS} seconds"
+REPORT_DATA+="- Operation: [ $@ ]\t${REPORT_OUTPUT}\tNumber of Attempts [ ${RETRY} ]\n"
+echo -e "Run Time = ${REPORT_OUTPUT}"
+set -e
+}
+