From 0c80a415bfbdcb18cc984a6b7945d60091d3abf7 Mon Sep 17 00:00:00 2001
From: "Gupta, Sangeet (sg774j)" <sg774j@att.com>
Date: Tue, 5 Oct 2021 03:01:22 +0000
Subject: [PATCH] nova: Give service time to restore

nova-service-cleaner job deletes the service which are down. If the
database is down, the service will go down as well. When database comes
back up, all the services starts to come back to up status. If the
nova-service-cleaner is run in this interim time, the service that
were down gets deleted. These would have come up if the job had not
run. Adding sleep to this job to give service time to come back up
if recovering. The sleep is set to 2 times the report_interval.

Change-Id: Ia292d19508e9449ccb40d1100b1d56b1283e5d53
---
 nova/Chart.yaml                                 | 2 +-
 nova/templates/bin/_nova-service-cleaner.sh.tpl | 7 +++++++
 nova/values.yaml                                | 1 +
 releasenotes/notes/nova.yaml                    | 1 +
 4 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/nova/Chart.yaml b/nova/Chart.yaml
index 5c95d0c955..9f47cfa930 100644
--- a/nova/Chart.yaml
+++ b/nova/Chart.yaml
@@ -14,7 +14,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Nova
 name: nova
-version: 0.2.17
+version: 0.2.18
 home: https://docs.openstack.org/nova/latest/
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/Nova/OpenStack_Project_Nova_vertical.png
 sources:
diff --git a/nova/templates/bin/_nova-service-cleaner.sh.tpl b/nova/templates/bin/_nova-service-cleaner.sh.tpl
index d3319c6d80..07ab81cde7 100644
--- a/nova/templates/bin/_nova-service-cleaner.sh.tpl
+++ b/nova/templates/bin/_nova-service-cleaner.sh.tpl
@@ -16,6 +16,13 @@ limitations under the License.
 
 set -xe
 
+# If any non-compute service is down, then sleep for 2 times the report_interval
+# to confirm service is still down.
+DISABLED_SVC="$(openstack compute service list -f value | grep -v 'nova-compute' | grep 'down')"
+if [ ! -z "${DISABLED_SVC}" ]; then
+  sleep $((2 * {{ .Values.conf.nova.DEFAULT.report_interval }}))
+fi
+
 NOVA_SERVICES_TO_CLEAN="$(openstack compute service list -f value -c Binary | sort | uniq | grep -v '^nova-compute$')"
 for NOVA_SERVICE in ${NOVA_SERVICES_TO_CLEAN}; do
   DEAD_SERVICE_IDS=$(openstack compute service list --service ${NOVA_SERVICE} -f json | jq -r '.[] | select(.State == "down") | .ID')
diff --git a/nova/values.yaml b/nova/values.yaml
index 7a45586b3e..1826eaa9d4 100644
--- a/nova/values.yaml
+++ b/nova/values.yaml
@@ -1439,6 +1439,7 @@ conf:
       instance_usage_audit_period: hour
       notify_on_state_change: vm_and_task_state
       resume_guests_state_on_host_boot: True
+      report_interval: 10
     vnc:
       novncproxy_host: 0.0.0.0
       vncserver_listen: 0.0.0.0
diff --git a/releasenotes/notes/nova.yaml b/releasenotes/notes/nova.yaml
index d0ed11adbb..77db078954 100644
--- a/releasenotes/notes/nova.yaml
+++ b/releasenotes/notes/nova.yaml
@@ -38,4 +38,5 @@ nova:
   - 0.2.15 Fix archive-deleted-rows for enabling date command as value for before option
   - 0.2.16 Remove the policy document in values file
   - 0.2.17 Fix disablement of helm.sh/hook for Helm v2
+  - 0.2.18 Give service time to restore
 ...