From 1d7f880c4274b0cf4192f1f7ba77e2642273fe03 Mon Sep 17 00:00:00 2001
From: Huy Tran <ht095u@att.com>
Date: Wed, 27 Oct 2021 10:52:39 -0500
Subject: [PATCH] Add check for number of computes in cell-setup-init script

This PS further enhances [1] to handle case where present
computes are up, but the number of present computes is not
equal to total number of expected computes.

[1] https://review.opendev.org/c/openstack/openstack-helm/+/815086

Change-Id: Idb2a7aeb202fe29fc528ba0dde987e7e0ee65a95
---
 nova/Chart.yaml                            |  2 +-
 nova/templates/bin/_cell-setup-init.sh.tpl | 31 +++++++++++++---
 nova/templates/job-cell-setup.yaml         | 42 ++++++++++++++++++++++
 nova/values.yaml                           |  3 ++
 releasenotes/notes/nova.yaml               |  1 +
 5 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/nova/Chart.yaml b/nova/Chart.yaml
index 60cc765741..fba86e1f48 100644
--- a/nova/Chart.yaml
+++ b/nova/Chart.yaml
@@ -14,7 +14,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Nova
 name: nova
-version: 0.2.24
+version: 0.2.25
 home: https://docs.openstack.org/nova/latest/
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/Nova/OpenStack_Project_Nova_vertical.png
 sources:
diff --git a/nova/templates/bin/_cell-setup-init.sh.tpl b/nova/templates/bin/_cell-setup-init.sh.tpl
index daa9048490..953508584d 100644
--- a/nova/templates/bin/_cell-setup-init.sh.tpl
+++ b/nova/templates/bin/_cell-setup-init.sh.tpl
@@ -19,16 +19,37 @@ set -ex
 {{- if .Values.jobs.cell_setup.extended_wait.enabled }}
 iteration={{ .Values.jobs.cell_setup.extended_wait.iteration }}
 duration={{ .Values.jobs.cell_setup.extended_wait.duration }}
-
 extra_wait=true
+# Init for case wait_for_computes is not enabled. It'll have
+# the same effect as the original code that checks for at
+# least one compute is registered
+expected_computes=1
+
+if [[ -f /tmp/compute_nodes.txt ]]
+then
+  expected_computes=$(cat /tmp/compute_nodes.txt | wc -w)
+fi
 
 while [[ "$extra_wait" == true ]]
 do
-  if [[ -z "$(openstack compute service list --service nova-compute -f value -c State | grep '^down$')" ]]
+  nova_computes=$(openstack compute service list --service nova-compute -f value -c State)
+
+  if [[ -z "$(echo $nova_computes | grep down)" ]]
+  then
+    # No more down. Although all present computes are up,
+    # the number of present computes may not be the total
+    # expected number of computes as some of the remaining
+    # computes may take a bit longer to register/join.
+    actual_computes=$(echo $nova_computes | wc -w)
+    if [[ "$actual_computes" -ge "$expected_computes" ]]
+    then
+      # All expected nodes are up
+      extra_wait=false
+    fi
+  fi
+
+  if [[ "$extra_wait" == true ]]
   then
-    # No more down
-    extra_wait=false
-  else
     sleep "$duration"
 
     if [[ "$iteration" -gt 1 ]]
diff --git a/nova/templates/job-cell-setup.yaml b/nova/templates/job-cell-setup.yaml
index 429fed53bb..cdcdf2512f 100644
--- a/nova/templates/job-cell-setup.yaml
+++ b/nova/templates/job-cell-setup.yaml
@@ -40,6 +40,22 @@ spec:
         {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
       initContainers:
 {{ tuple $envAll "cell_setup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+{{- if $envAll.Values.bootstrap.wait_for_computes.enabled }}
+        - name: nova-wait-for-computes-init
+{{ tuple $envAll "nova_wait_for_computes_init" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ dict "envAll" $envAll "application" "bootstrap" "container" "nova_wait_for_computes_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /bin/bash
+            - -c
+            - /tmp/wait-for-computes-init.sh
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: nova-bin
+              mountPath: /tmp/wait-for-computes-init.sh
+              subPath: wait-for-computes-init.sh
+              readOnly: true
+{{- end }}
         - name: nova-cell-setup-init
 {{ tuple $envAll "nova_cell_setup_init" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.cell_setup | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -104,4 +120,30 @@ spec:
             defaultMode: 0555
 {{- dict "enabled" .Values.manifests.certificates "name" .Values.endpoints.oslo_db.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.compute.osapi.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
 {{- end }}
diff --git a/nova/values.yaml b/nova/values.yaml
index 823fb0a9fb..cdb1457520 100644
--- a/nova/values.yaml
+++ b/nova/values.yaml
@@ -2210,6 +2210,9 @@ pod:
       pod:
         runAsUser: 42424
       container:
+        nova_wait_for_computes_init:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
         nova_cell_setup_init:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
diff --git a/releasenotes/notes/nova.yaml b/releasenotes/notes/nova.yaml
index b7ad3c250e..a9696b48f4 100644
--- a/releasenotes/notes/nova.yaml
+++ b/releasenotes/notes/nova.yaml
@@ -45,4 +45,5 @@ nova:
   - 0.2.22 Update htk requirements repo
   - 0.2.23 Add option to enable extra wait for cell-setup-init
   - 0.2.24 Fix nova-bootstrap job labels
+  - 0.2.25 Add check for compute nodes
 ...