Configuration Management Overrides for Cinder

This commit is based on how Nova got its configuration overrides
implemented.

An important thing here is support for setting multiple Cinder backends
(e.g. Ceph cluster + NFS) in the values.yaml. This was required as Cinder
accepts backend configurations only in [<backend_id>] sections in the
cinder.conf.

Please note that autogeneration of ceph.conf and
ceph.client.<rbd_user>.keyring works only for a backend named "rbd1".
In case you want to add another RBD backend, you need to mount those
files by yourself. Commit ehancing this is planned to follow shortly.

Change-Id: Ifb58a85300bbfbb9e63d6b3bfc2ad19a99d2c9d4

cinder/templates/_funcs.tpl

@@ -0,0 +1,23 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- define "cinder.is_ceph_configured" -}}
+{{- range $section, $values := .Values.conf.backends -}}
+{{- if kindIs "map" $values -}}
+{{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}}
+true
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}

cinder/templates/configmap-etc.yaml

@@ -12,20 +12,102 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- include "cinder.conf.cinder_values_skeleton" .Values.conf.cinder | trunc 0 -}}
+
+{{- if empty .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.auth_uri -}}
+{{- tuple "identity" "internal" "api" . | include "helm-toolkit.keystone_endpoint_uri_lookup"| set .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token "auth_uri" | quote | trunc 0 -}}
+{{- end -}}
+
+# FIXME(alanmeadows) fix for broken keystonemiddleware oslo config gen in newton - will remove in future
+{{- if empty .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.auth_url -}}
+{{- tuple "identity" "internal" "api" . | include "helm-toolkit.keystone_endpoint_uri_lookup"| set .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token "auth_url" | quote | trunc 0 -}}
+{{- end -}}
+
+{{- if empty .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.memcached_servers -}}
+{{- tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.hostname_endpoint_uri_lookup" | set .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token "memcached_servers" | quote | trunc 0 -}}
+{{- end -}}
+
+{{- if empty .Values.conf.cinder.database.oslo.db.connection -}}
+{{- tuple "oslo_db" "internal" "user" "mysql" . | include "helm-toolkit.authenticated_endpoint_uri_lookup"| set .Values.conf.cinder.database.oslo.db "connection" | quote | trunc 0 -}}
+{{- end -}}
+
+{{- if empty .Values.conf.cinder.default.oslo.messaging.transport_url -}}
+{{- tuple "oslo_messaging" "internal" "user" "amqp" . | include "helm-toolkit.authenticated_endpoint_uri_lookup" | set .Values.conf.cinder.default.oslo.messaging "transport_url" | quote | trunc 0 -}}
+{{- end -}}
+
+{{- if empty .Values.conf.cinder.default.cinder.glance_api_servers -}}
+{{- tuple "image" "internal" "api" . | include "helm-toolkit.keystone_endpoint_uri_lookup"| set .Values.conf.cinder.default.cinder "glance_api_servers" | quote | trunc 0 -}}
+{{- end -}}
+
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: cinder-etc
 data:
   cinder.conf: |+
+{{ if .Values.conf.cinder.override -}}
+{{ .Values.conf.cinder.override | indent 4 }}
+{{- else -}}
+{{- if .Values.conf.cinder.prefix -}}
+{{ .Values.conf.cinder.prefix | indent 4 }}
+{{- end }}
 {{ tuple "etc/_cinder.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- if .Values.conf.cinder.append -}}
+{{ .Values.conf.cinder.append | indent 4 }}
+{{- end }}
+  backends.conf: |+
+{{ if .Values.conf.backends.override -}}
+{{ .Values.conf.backends.override | indent 4 }}
+{{- else -}}
+{{- if .Values.conf.backends.prefix -}}
+{{ .Values.conf.backends.prefix | indent 4 }}
+{{- end }}
+{{ include "helm-toolkit.utils.to_ini" .Values.conf.backends | indent 4 }}
+{{- end }}
+{{- if .Values.conf.backends.append -}}
+{{ .Values.conf.backends.append | indent 4 }}
+{{- end }}
   rootwrap.conf: |+
+{{ if .Values.conf.rootwrap.override -}}
+{{ .Values.conf.rootwrap.override | indent 4 }}
+{{- else -}}
+{{- if .Values.conf.rootwrap.prefix -}}
+{{ .Values.conf.rootwrap.prefix | indent 4 }}
+{{- end }}
 {{ tuple "etc/_rootwrap.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- if .Values.conf.rootwrap.append -}}
+{{ .Values.conf.rootwrap.append | indent 4 }}
+{{- end }}
   api-paste.ini: |+
-{{ tuple "etc/_cinder-api-paste.ini.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ if .Values.conf.paste.override -}}
+{{ .Values.conf.paste.override | indent 4 }}
+{{- else -}}
+{{- if .Values.conf.paste.prefix -}}
+{{ .Values.conf.paste.prefix | indent 4 }}
+{{- end }}
+{{ tuple "etc/_api-paste.ini.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- if .Values.conf.paste.append -}}
+{{ .Values.conf.paste.append | indent 4 }}
+{{- end }}
   policy.json: |+
+{{ if .Values.conf.policy.override -}}
+{{ .Values.conf.policy.override | indent 4 }}
+{{- else -}}
 {{ tuple "etc/_policy.json.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- if include "cinder.is_ceph_configured" . -}}
   ceph.conf: |+
+{{ if .Values.conf.ceph.override -}}
+{{ .Values.conf.ceph.override | indent 4 }}
+{{- else -}}
+{{- end }}
 {{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  ceph.client.{{ .Values.ceph.cinder_user }}.keyring: |+
+{{- if .Values.conf.ceph.append -}}
+{{ .Values.conf.ceph.append | indent 4 }}
+{{- end }}
+  ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring: |+
 {{ tuple "etc/_ceph-cinder.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

cinder/templates/deployment-api.yaml

@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 {{- $envAll := . }}
 {{- $dependencies := .Values.dependencies.api }}
+{{- $mounts_cinder_api := .Values.mounts.cinder_api.cinder_api }}
+{{- $mounts_cinder_api_init := .Values.mounts.cinder_api.init_container }}
+
 apiVersion: apps/v1beta1
 kind: Deployment
 metadata:
@@ -36,7 +38,7 @@ spec:
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
         pod.beta.kubernetes.io/init-containers: '[
-{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
+{{ tuple $envAll $dependencies $mounts_cinder_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
         ]'
     spec:
       nodeSelector:
@@ -59,13 +61,11 @@ spec:
             - /tmp/cinder-api.sh
           ports:
             - name: c-api
-              containerPort: {{ .Values.network.api.port }}
+              containerPort: {{ .Values.conf.cinder.default.cinder.osapi_volume_listen_port }}
           readinessProbe:
             tcpSocket:
-              port: {{ .Values.network.api.port }}
+              port: {{ .Values.conf.cinder.default.cinder.osapi_volume_listen_port }}
           volumeMounts:
-            - name: pod-etc-cinder
-              mountPath: /etc/cinder
             - name: pod-var-cache-cinder
               mountPath: /var/cache/cinder
             - name: cinder-bin
@@ -84,9 +84,8 @@ spec:
               mountPath: /etc/cinder/policy.json
               subPath: policy.json
               readOnly: true
+{{ if $mounts_cinder_api.volumeMounts }}{{ toYaml $mounts_cinder_api.volumeMounts | indent 12 }}{{ end }}
       volumes:
-        - name: pod-etc-cinder
-          emptyDir: {}
         - name: pod-var-cache-cinder
           emptyDir: {}
         - name: cinder-bin
@@ -95,3 +94,4 @@ spec:
         - name: cinder-etc
           configMap:
             name: cinder-etc
+{{ if $mounts_cinder_api.volumes }}{{ toYaml $mounts_cinder_api.volumes | indent 8 }}{{ end }}

cinder/templates/deployment-scheduler.yaml

@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 {{- $envAll := . }}
 {{- $dependencies := .Values.dependencies.scheduler }}
+{{- $mounts_cinder_scheduler := .Values.mounts.cinder_scheduler.cinder_scheduler }}
+{{- $mounts_cinder_scheduler_init := .Values.mounts.cinder_scheduler.init_container }}
+
 apiVersion: apps/v1beta1
 kind: Deployment
 metadata:
@@ -36,7 +38,7 @@ spec:
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
         pod.beta.kubernetes.io/init-containers: '[
-{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
+{{ tuple $envAll $dependencies $mounts_cinder_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
         ]'
     spec:
       nodeSelector:
@@ -58,8 +60,6 @@ spec:
             - bash
             - /tmp/cinder-scheduler.sh
           volumeMounts:
-            - name: pod-etc-cinder
-              mountPath: /etc/cinder
             - name: pod-var-cache-cinder
               mountPath: /var/cache/cinder
             - name: cinder-bin
@@ -78,9 +78,8 @@ spec:
               mountPath: /etc/cinder/policy.json
               subPath: policy.json
               readOnly: true
+{{ if $mounts_cinder_scheduler.volumeMounts }}{{ toYaml $mounts_cinder_scheduler.volumeMounts | indent 12 }}{{ end }}
       volumes:
-        - name: pod-etc-cinder
-          emptyDir: {}
         - name: pod-var-cache-cinder
           emptyDir: {}
         - name: cinder-bin
@@ -89,3 +88,4 @@ spec:
         - name: cinder-etc
           configMap:
             name: cinder-etc
+{{ if $mounts_cinder_scheduler.volumes }}{{ toYaml $mounts_cinder_scheduler.volumes | indent 8 }}{{ end }}

cinder/templates/deployment-volume.yaml

@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 {{- $envAll := . }}
 {{- $dependencies := .Values.dependencies.volume }}
+{{- $mounts_cinder_volume := .Values.mounts.cinder_volume.cinder_volume }}
+{{- $mounts_cinder_volume_init := .Values.mounts.cinder_volume.init_container }}
+
 apiVersion: apps/v1beta1
 kind: Deployment
 metadata:
@@ -36,7 +38,7 @@ spec:
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
         configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
         pod.beta.kubernetes.io/init-containers: '[
-{{ tuple $envAll $dependencies "[]" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
+{{ tuple $envAll $dependencies $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
         ]'
     spec:
       nodeSelector:
@@ -58,8 +60,6 @@ spec:
             - bash
             - /tmp/cinder-volume.sh
           volumeMounts:
-            - name: pod-etc-cinder
-              mountPath: /etc/cinder
             - name: pod-var-cache-cinder
               mountPath: /var/cache/cinder
             - name: cinder-bin
@@ -70,17 +70,21 @@ spec:
               mountPath: /etc/cinder/cinder.conf
               subPath: cinder.conf
               readOnly: true
+            - name: cinder-etc
+              mountPath: /etc/cinder/conf/backends.conf
+              subPath: backends.conf
+              readOnly: true
+            {{- if include "cinder.is_ceph_configured" . }}
             - name: cinder-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            - name: cinder-etc
+              mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
-              mountPath: /etc/ceph/ceph.client.{{ .Values.ceph.cinder_user }}.keyring
+              subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
-              subPath: ceph.client.{{ .Values.ceph.cinder_user }}.keyring
               readOnly: true
+            {{- end }}
+{{ if $mounts_cinder_volume.volumeMounts }}{{ toYaml $mounts_cinder_volume.volumeMounts | indent 12 }}{{ end }}
       volumes:
-        - name: pod-etc-cinder
-          emptyDir: {}
         - name: pod-var-cache-cinder
           emptyDir: {}
         - name: cinder-bin
@@ -89,3 +93,4 @@ spec:
         - name: cinder-etc
           configMap:
             name: cinder-etc
+{{ if $mounts_cinder_volume.volumes }}{{ toYaml $mounts_cinder_volume.volumes | indent 8 }}{{ end }}

cinder/templates/etc/_ceph-cinder.keyring.tpl

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-[client.{{ .Values.ceph.cinder_user }}]
+[client.{{ .Values.conf.backends.rbd1.rbd_user }}]
-{{- if .Values.ceph.cinder_keyring }}
+{{- if .Values.conf.ceph.cinder_keyring }}
-    key = {{ .Values.ceph.cinder_keyring }}
+    key = {{ .Values.conf.ceph.cinder_keyring }}
 {{- else }}
     key = {{- include "secrets/ceph-client-key" . -}}
 {{- end }}

cinder/templates/etc/_ceph.conf.tpl

@@ -15,9 +15,9 @@
 [global]
 rgw_thread_pool_size = 1024
 rgw_num_rados_handles = 100
-{{- if .Values.ceph.monitors }}
+{{- if .Values.conf.ceph.monitors }}
 [mon]
-{{ range .Values.ceph.monitors }}
+{{ range .Values.conf.ceph.monitors }}
     [mon.{{ . }}]
       host = {{ . }}
       mon_addr = {{ . }}

cinder/templates/secret-keystone-user.env.yaml

@@ -21,14 +21,14 @@ data:
   OS_AUTH_URL: |
 {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc | indent 4 }}
   OS_REGION_NAME: |
-{{ .Values.keystone.cinder_region_name  | b64enc | indent 4 }}
+{{ .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.region_name | b64enc | indent 4 }}
   OS_PROJECT_DOMAIN_NAME: |
-{{ .Values.keystone.cinder_project_domain  | b64enc | indent 4 }}
+{{ .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.project_domain_name | b64enc | indent 4 }}
   OS_PROJECT_NAME: |
-{{ .Values.keystone.cinder_project_name  | b64enc | indent 4 }}
+{{ .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.project_name | b64enc | indent 4 }}
   OS_USER_DOMAIN_NAME: |
-{{ .Values.keystone.cinder_user_domain  | b64enc | indent 4 }}
+{{ .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.user_domain_name | b64enc | indent 4 }}
   OS_USERNAME: |
-{{ .Values.keystone.cinder_user  | b64enc | indent 4 }}
+{{ .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.username | b64enc | indent 4 }}
   OS_PASSWORD: |
-{{ .Values.keystone.cinder_password  | b64enc | indent 4 }}
+{{ .Values.conf.cinder.keystone_authtoken.keystonemiddleware.auth_token.password | b64enc | indent 4 }}

cinder/templates/service-api.yaml

@@ -18,7 +18,7 @@ metadata:
   name: cinder-api
 spec:
   ports:
-    - port: {{ .Values.network.api.port }}
+    - port: {{ .Values.conf.cinder.default.cinder.osapi_volume_listen_port }}
     {{ if .Values.network.api.node_port.enabled }}
       nodePort: {{ .Values.network.api.node_port.port }}
     {{ end }}

cinder/values.yaml

@@ -56,54 +56,82 @@ keystone:
   admin_project_name: "admin"
   admin_project_domain: "default"
   admin_region_name: "RegionOne"
-
-  cinder_user: "cinder"
-  cinder_user_domain: "default"
   cinder_user_role: "admin"
-  cinder_password: "password"
-  cinder_project_name: "service"
-  cinder_project_domain: "default"
-  cinder_region_name: "RegionOne"
 
 network:
   api:
     name: "cinder-api"
-    port: 8776
     node_port:
       enabled: false
       port: 30877
 
+
+conf:
+  paste:
+    override:
+    append:
+  policy:
+    override:
+    append:
+  rootwrap:
+    override:
+    append:
   ceph:
-  enabled: true
+    override:
+    append:
     monitors: []
-  cinder_user: "admin"
-  # a null value for the keyring will
-  # attempt to use the key from
-  # common/secrets/ceph-client-key
     cinder_keyring: null
-
+  cinder:
-backends:
+    override:
-  enabled:
+    append:
-  - rbd1
+    database:
-  rbd1:
+      oslo:
-    secret: null
+        db:
-    user: "admin"
+          max_retries: -1
-    pool: "volumes"
+    default:
-
+      oslo:
-glance:
+        log:
-  version: 2
-
-messaging:
-  hosts: rabbitmq
-  user: rabbitmq
-  password: password
-
-
-api:
-  workers: 8
-
-misc:
           debug: false
+          use_syslog: false
+          use_stderr: true
+      cinder:
+        enable_v1_api: false
+        volume_name_template: "%s"
+        osapi_volume_workers: 8
+        glance_api_version: 2
+        os_region_name: RegionOne
+        host: cinder-volume-worker
+        osapi_volume_listen_port: 8776
+        enabled_backends: "rbd1"
+    keystone_authtoken:
+      keystonemiddleware:
+        auth_token:
+          auth_version: v3
+          auth_type: password
+          region_name: RegionOne
+          project_domain_name: default
+          project_name: service
+          user_domain_name: default
+          username: cinder
+          password: password
+    oslo_concurrency:
+      oslo:
+        concurrency:
+          lock_path: "/var/lib/cinder/tmp"
+  backends:
+    override:
+    append:
+    # Those options will be written to backends.conf as-is.
+    rbd1:
+      volume_driver: cinder.volume.drivers.rbd.RBDDriver
+      volume_backend_name: rbd1
+      rbd_pool: volumes
+      rbd_ceph_conf: "/etc/ceph/ceph.conf"
+      rbd_flatten_volume_from_snapshot: false
+      rbd_max_clone_depth: 5
+      rbd_store_chunk_size: 4
+      rados_connect_timeout: -1
+      rbd_user: "admin"
 
 dependencies:
   db_init:
@@ -210,6 +238,25 @@ endpoints:
     scheme: mysql+pymysql
     port:
       mysql: 3306
+  oslo_messaging:
+    auth:
+      admin:
+        username: admin
+        password: password
+      user:
+        username: rabbitmq
+        password: password
+    hosts:
+      default: rabbitmq
+    path: /
+    scheme: rabbit
+    port:
+      amqp: 5672
+  oslo_cache:
+    hosts:
+      default: memcache
+    port:
+      memcache: 11211
 
 resources:
   enabled: false
@@ -269,3 +316,14 @@ resources:
     limits:
       memory: "1024Mi"
       cpu: "2000m"
+
+mounts:
+  cinder_api:
+    init_container: null
+    cinder_api:
+  cinder_scheduler:
+    init_container: null
+    cinder_scheduler:
+  cinder_volume:
+    init_container: null
+    cinder_volume:

helm-toolkit/templates/utils/_to_ini.tpl

@@ -0,0 +1,28 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- define "helm-toolkit.to_ini" -}}
+{{- range $section, $values := . -}}
+{{- if kindIs "map" $values -}}
+[{{ $section }}]
+{{range $key, $value := $values -}}
+{{- if kindIs "slice" $value -}}
+{{ $key }} = {{ include "helm-toolkit.joinListWithComma" $value }}
+{{else -}}
+{{ $key }} = {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}