From a5e5556e46ee143363557dc18c89e6c811c2e2c0 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Fri, 6 Jul 2018 12:33:12 -0500 Subject: [PATCH] Ceph: Split out ceph provsioners from client chart This PS splits out the ceph provisioners from the client chart allowing OSH to be used with external (and multiple) cephs more simply. Change-Id: I0c5d516c7a9a10bc8435e2423eb1dc05460d2c92 Signed-off-by: Pete Birley --- ceph-client/templates/configmap-bin.yaml | 10 +- .../templates/configmap-etc-client.yaml | 2 +- ceph-client/templates/deployment-mds.yaml | 6 +- ceph-client/templates/deployment-mgr.yaml | 6 +- ceph-client/templates/deployment-rgw.yaml | 8 +- ceph-client/templates/job-bootstrap.yaml | 6 +- ceph-client/templates/job-rbd-pool.yaml | 6 +- ceph-client/values.yaml | 54 ---- ceph-provisioners/Chart.yaml | 18 ++ ceph-provisioners/requirements.yaml | 18 ++ .../templates/bin/_bootstrap.sh.tpl | 20 ++ .../cephfs/_client-key-manager.sh.tpl | 0 .../bin/provisioner/cephfs/_start.sh.tpl | 0 .../rbd/_namespace-client-key-cleaner.sh.tpl | 0 .../rbd/_namespace-client-key-manager.sh.tpl | 0 .../bin/provisioner/rbd/_start.sh.tpl | 0 .../templates/configmap-bin-provisioner.yaml | 2 +- .../templates/configmap-bin.yaml | 43 ++++ .../templates/configmap-etc-client.yaml | 51 ++++ .../deployment-cephfs-provisioner.yaml | 6 +- .../templates/deployment-rbd-provisioner.yaml | 6 +- .../templates/job-bootstrap.yaml | 70 ++++++ .../templates/job-cephfs-client-key.yaml | 6 +- .../job-namespace-client-key-cleaner.yaml | 6 +- .../templates/job-namespace-client-key.yaml | 6 +- .../templates/storageclass-cephfs.yaml | 0 .../templates/storageclass-rbd.yaml | 0 ceph-provisioners/values.yaml | 234 ++++++++++++++++++ .../armada/multinode/armada-ceph.yaml | 74 ++++++ .../armada/multinode/armada-lma.yaml | 2 +- .../armada/multinode/armada-osh.yaml | 2 +- tools/deployment/developer/ceph/040-ceph.sh | 4 +- .../developer/ceph/045-ceph-ns-activate.sh | 4 +- tools/deployment/multinode/030-ceph.sh | 2 +- .../multinode/040-ceph-ns-activate.sh | 2 +- 35 files changed, 570 insertions(+), 104 deletions(-) create mode 100644 ceph-provisioners/Chart.yaml create mode 100644 ceph-provisioners/requirements.yaml create mode 100644 ceph-provisioners/templates/bin/_bootstrap.sh.tpl rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/cephfs/_start.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/rbd/_start.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/configmap-bin-provisioner.yaml (96%) create mode 100644 ceph-provisioners/templates/configmap-bin.yaml create mode 100644 ceph-provisioners/templates/configmap-etc-client.yaml rename {ceph-client => ceph-provisioners}/templates/deployment-cephfs-provisioner.yaml (97%) rename {ceph-client => ceph-provisioners}/templates/deployment-rbd-provisioner.yaml (97%) create mode 100644 ceph-provisioners/templates/job-bootstrap.yaml rename {ceph-client => ceph-provisioners}/templates/job-cephfs-client-key.yaml (97%) rename {ceph-client => ceph-provisioners}/templates/job-namespace-client-key-cleaner.yaml (95%) rename {ceph-client => ceph-provisioners}/templates/job-namespace-client-key.yaml (96%) rename {ceph-client => ceph-provisioners}/templates/storageclass-cephfs.yaml (100%) rename {ceph-client => ceph-provisioners}/templates/storageclass-rbd.yaml (100%) create mode 100644 ceph-provisioners/values.yaml diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml index 5b8e5bc7a6..0edfe1f1b2 100644 --- a/ceph-client/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }} +{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }} {{- $envAll := . }} --- apiVersion: v1 @@ -48,14 +48,6 @@ data: mgr-check.sh: | {{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - provisioner-cephfs-start.sh: | -{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - provisioner-cephfs-client-key-manager.sh: | -{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - - provisioner-rbd-start.sh: | -{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - rgw-start.sh: | {{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} rgw-init-keystone.sh: | diff --git a/ceph-client/templates/configmap-etc-client.yaml b/ceph-client/templates/configmap-etc-client.yaml index 88d05cf967..0e1610b927 100644 --- a/ceph-client/templates/configmap-etc-client.yaml +++ b/ceph-client/templates/configmap-etc-client.yaml @@ -51,5 +51,5 @@ data: {{- end }} {{- end }} {{- if .Values.manifests.configmap_etc }} -{{- list "ceph-etc" . | include "ceph.configmap.etc" }} +{{- list "ceph-client-etc" . | include "ceph.configmap.etc" }} {{- end }} diff --git a/ceph-client/templates/deployment-mds.yaml b/ceph-client/templates/deployment-mds.yaml index a680b28253..2118048e34 100644 --- a/ceph-client/templates/deployment-mds.yaml +++ b/ceph-client/templates/deployment-mds.yaml @@ -89,7 +89,7 @@ spec: mountPath: /tmp/mds-start.sh subPath: mds-start.sh readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -108,9 +108,9 @@ spec: mountPath: /run readOnly: false volumes: - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: ceph-client-bin configMap: diff --git a/ceph-client/templates/deployment-mgr.yaml b/ceph-client/templates/deployment-mgr.yaml index e36df7252c..d81f7fda8c 100644 --- a/ceph-client/templates/deployment-mgr.yaml +++ b/ceph-client/templates/deployment-mgr.yaml @@ -123,7 +123,7 @@ spec: readOnly: true - name: pod-etc-ceph mountPath: /etc/ceph - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -148,9 +148,9 @@ spec: configMap: name: ceph-client-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: pod-var-lib-ceph emptyDir: {} diff --git a/ceph-client/templates/deployment-rgw.yaml b/ceph-client/templates/deployment-rgw.yaml index 22c48af626..0f0d1db8d7 100644 --- a/ceph-client/templates/deployment-rgw.yaml +++ b/ceph-client/templates/deployment-rgw.yaml @@ -89,7 +89,7 @@ spec: mountPath: /tmp/rgw-init-keystone.sh subPath: rgw-init-keystone.sh readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /tmp/ceph.conf subPath: ceph.conf readOnly: true @@ -126,7 +126,7 @@ spec: - name: pod-etc-ceph mountPath: /etc/ceph {{- if not .Values.conf.rgw_ks.enabled }} - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -152,9 +152,9 @@ spec: configMap: name: ceph-client-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: pod-var-lib-ceph emptyDir: {} diff --git a/ceph-client/templates/job-bootstrap.yaml b/ceph-client/templates/job-bootstrap.yaml index 6e558e2bd9..72a935973b 100644 --- a/ceph-client/templates/job-bootstrap.yaml +++ b/ceph-client/templates/job-bootstrap.yaml @@ -47,7 +47,7 @@ spec: mountPath: /tmp/bootstrap.sh subPath: bootstrap.sh readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -60,9 +60,9 @@ spec: configMap: name: ceph-client-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: ceph-client-admin-keyring secret: diff --git a/ceph-client/templates/job-rbd-pool.yaml b/ceph-client/templates/job-rbd-pool.yaml index c978519b57..961321259b 100644 --- a/ceph-client/templates/job-rbd-pool.yaml +++ b/ceph-client/templates/job-rbd-pool.yaml @@ -57,7 +57,7 @@ spec: mountPath: /tmp/pool-calc.py subPath: pool-calc.py readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -72,9 +72,9 @@ spec: mountPath: /run readOnly: false volumes: - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: ceph-client-bin configMap: diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 144388e9c8..3f6a0d628c 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -19,9 +19,6 @@ deployment: ceph: true - client_secrets: false - rbd_provisioner: true - cephfs_provisioner: true rgw_keystone_user_and_endpoints: false release_group: null @@ -30,12 +27,10 @@ images: pull_policy: IfNotPresent tags: ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' - ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1' ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3' ceph_mds: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_mgr: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_rbd_pool: 'docker.io/port/ceph-config-helper:v1.10.3' - ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1' ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1' ks_endpoints: 'docker.io/openstackhelm/heat:newton' @@ -68,8 +63,6 @@ labels: pod: dns_policy: "ClusterFirstWithHostNet" replicas: - cephfs_provisioner: 2 - rbd_provisioner: 2 mds: 2 mgr: 2 rgw: 2 @@ -95,20 +88,6 @@ pod: limits: memory: "50Mi" cpu: "500m" - rbd_provisioner: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" - cephfs_provisioner: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" mgr: requests: memory: "5Mi" @@ -433,32 +412,6 @@ ceph_mgr_enabled_modules: # num_rep: "3" # min_size: "2" -# if you change provision_storage_class to false -# it is presumed you manage your own storage -# class definition externally -storageclass: - rbd: - provision_storage_class: true - provisioner: ceph.com/rbd - name: general - monitors: null - pool: rbd - admin_id: admin - admin_secret_name: pvc-ceph-conf-combined-storageclass - admin_secret_namespace: ceph - user_id: admin - user_secret_name: pvc-ceph-client-key - image_format: "2" - image_features: layering - cephfs: - provision_storage_class: true - provisioner: ceph.com/cephfs - name: cephfs - admin_id: admin - user_secret_name: pvc-ceph-cephfs-client-key - admin_secret_name: pvc-ceph-conf-combined-storageclass - admin_secret_namespace: ceph - endpoints: cluster_domain_suffix: cluster.local local_image_registry: @@ -563,11 +516,8 @@ monitoring: manifests: configmap_bin: true configmap_bin_ks: true - configmap_bin_common: true configmap_etc: true deployment_mds: true - deployment_rbd_provisioner: true - deployment_cephfs_provisioner: true deployment_rgw: true deployment_mgr: true ingress_rgw: true @@ -577,8 +527,6 @@ manifests: job_ks_endpoints: true job_ks_service: true job_ks_user: true - job_namespace_client_key_cleaner: true - job_namespace_client_key: true job_rbd_pool: true secret_ingress_tls: true secret_keystone_rgw: true @@ -586,5 +534,3 @@ manifests: service_ingress_rgw: true service_mgr: true service_rgw: true - storageclass_cephfs: true - storageclass_rbd: true diff --git a/ceph-provisioners/Chart.yaml b/ceph-provisioners/Chart.yaml new file mode 100644 index 0000000000..a0d25ad40f --- /dev/null +++ b/ceph-provisioners/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Ceph Client +name: ceph-provisioners +version: 0.1.0 diff --git a/ceph-provisioners/requirements.yaml b/ceph-provisioners/requirements.yaml new file mode 100644 index 0000000000..53782e69b2 --- /dev/null +++ b/ceph-provisioners/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/ceph-provisioners/templates/bin/_bootstrap.sh.tpl b/ceph-provisioners/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..533c0a5a3f --- /dev/null +++ b/ceph-provisioners/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,20 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph-client/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/cephfs/_start.sh.tpl b/ceph-provisioners/templates/bin/provisioner/cephfs/_start.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/cephfs/_start.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/cephfs/_start.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/rbd/_start.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_start.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/rbd/_start.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/rbd/_start.sh.tpl diff --git a/ceph-client/templates/configmap-bin-provisioner.yaml b/ceph-provisioners/templates/configmap-bin-provisioner.yaml similarity index 96% rename from ceph-client/templates/configmap-bin-provisioner.yaml rename to ceph-provisioners/templates/configmap-bin-provisioner.yaml index 8a52855eaa..d34870fba3 100644 --- a/ceph-client/templates/configmap-bin-provisioner.yaml +++ b/ceph-provisioners/templates/configmap-bin-provisioner.yaml @@ -20,7 +20,7 @@ limitations under the License. apiVersion: v1 kind: ConfigMap metadata: - name: ceph-client-bin-provisioner + name: ceph-provisioners-bin-clients data: provisioner-rbd-namespace-client-key-manager.sh: | {{ tuple "bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph-provisioners/templates/configmap-bin.yaml b/ceph-provisioners/templates/configmap-bin.yaml new file mode 100644 index 0000000000..ed735d790d --- /dev/null +++ b/ceph-provisioners/templates/configmap-bin.yaml @@ -0,0 +1,43 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-provisioners-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + + provisioner-cephfs-start.sh: | +{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + provisioner-cephfs-client-key-manager.sh: | +{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + provisioner-rbd-start.sh: | +{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + +{{- end }} diff --git a/ceph-provisioners/templates/configmap-etc-client.yaml b/ceph-provisioners/templates/configmap-etc-client.yaml new file mode 100644 index 0000000000..3592990580 --- /dev/null +++ b/ceph-provisioners/templates/configmap-etc-client.yaml @@ -0,0 +1,51 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list "ceph-etc" . | include "ceph.configmap.etc" }} +{{- end }} diff --git a/ceph-client/templates/deployment-cephfs-provisioner.yaml b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml similarity index 97% rename from ceph-client/templates/deployment-cephfs-provisioner.yaml rename to ceph-provisioners/templates/deployment-cephfs-provisioner.yaml index a3cbe1af36..6548a31738 100644 --- a/ceph-client/templates/deployment-cephfs-provisioner.yaml +++ b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml @@ -166,13 +166,13 @@ spec: command: - /tmp/provisioner-cephfs-start.sh volumeMounts: - - name: ceph-client-bin + - name: ceph-provisioners-bin mountPath: /tmp/provisioner-cephfs-start.sh subPath: provisioner-cephfs-start.sh readOnly: true volumes: - - name: ceph-client-bin + - name: ceph-provisioners-bin configMap: - name: ceph-client-bin + name: ceph-provisioners-bin defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/deployment-rbd-provisioner.yaml b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml similarity index 97% rename from ceph-client/templates/deployment-rbd-provisioner.yaml rename to ceph-provisioners/templates/deployment-rbd-provisioner.yaml index fc29d5924a..ad21d36af0 100644 --- a/ceph-client/templates/deployment-rbd-provisioner.yaml +++ b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml @@ -156,13 +156,13 @@ spec: command: - /tmp/provisioner-rbd-start.sh volumeMounts: - - name: ceph-client-bin + - name: ceph-provisioners-bin mountPath: /tmp/provisioner-rbd-start.sh subPath: provisioner-rbd-start.sh readOnly: true volumes: - - name: ceph-client-bin + - name: ceph-provisioners-bin configMap: - name: ceph-client-bin + name: ceph-provisioners-bin defaultMode: 0555 {{- end }} diff --git a/ceph-provisioners/templates/job-bootstrap.yaml b/ceph-provisioners/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..00c4737368 --- /dev/null +++ b/ceph-provisioners/templates/job-bootstrap.yaml @@ -0,0 +1,70 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-client-bootstrap"}} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-client-bootstrap +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-client-bootstrap +{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: ceph-provisioners-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: ceph-provisioners-bin + configMap: + name: ceph-provisioners-bin + defaultMode: 0555 + - name: ceph-etc + configMap: + name: ceph-etc + defaultMode: 0444 + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} diff --git a/ceph-client/templates/job-cephfs-client-key.yaml b/ceph-provisioners/templates/job-cephfs-client-key.yaml similarity index 97% rename from ceph-client/templates/job-cephfs-client-key.yaml rename to ceph-provisioners/templates/job-cephfs-client-key.yaml index d1df477cae..2b0bee5a5b 100644 --- a/ceph-client/templates/job-cephfs-client-key.yaml +++ b/ceph-provisioners/templates/job-cephfs-client-key.yaml @@ -112,13 +112,13 @@ spec: command: - /tmp/provisioner-cephfs-client-key-manager.sh volumeMounts: - - name: ceph-client-bin + - name: ceph-provisioners-bin mountPath: /tmp/provisioner-cephfs-client-key-manager.sh subPath: provisioner-cephfs-client-key-manager.sh readOnly: true volumes: - - name: ceph-client-bin + - name: ceph-provisioners-bin configMap: - name: ceph-client-bin + name: ceph-provisioners-bin defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/job-namespace-client-key-cleaner.yaml b/ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml similarity index 95% rename from ceph-client/templates/job-namespace-client-key-cleaner.yaml rename to ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml index 56c7498112..f0691fc5c4 100644 --- a/ceph-client/templates/job-namespace-client-key-cleaner.yaml +++ b/ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml @@ -81,13 +81,13 @@ spec: command: - /tmp/provisioner-rbd-namespace-client-key-cleaner.sh volumeMounts: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients mountPath: /tmp/provisioner-rbd-namespace-client-key-cleaner.sh subPath: provisioner-rbd-namespace-client-key-cleaner.sh readOnly: true volumes: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients configMap: - name: ceph-client-bin-provisioner + name: ceph-provisioners-bin-clients defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/job-namespace-client-key.yaml b/ceph-provisioners/templates/job-namespace-client-key.yaml similarity index 96% rename from ceph-client/templates/job-namespace-client-key.yaml rename to ceph-provisioners/templates/job-namespace-client-key.yaml index cba3ebb4b3..75fd06872e 100644 --- a/ceph-client/templates/job-namespace-client-key.yaml +++ b/ceph-provisioners/templates/job-namespace-client-key.yaml @@ -112,13 +112,13 @@ spec: command: - /tmp/provisioner-rbd-namespace-client-key-manager.sh volumeMounts: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh subPath: provisioner-rbd-namespace-client-key-manager.sh readOnly: true volumes: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients configMap: - name: ceph-client-bin-provisioner + name: ceph-provisioners-bin-clients defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/storageclass-cephfs.yaml b/ceph-provisioners/templates/storageclass-cephfs.yaml similarity index 100% rename from ceph-client/templates/storageclass-cephfs.yaml rename to ceph-provisioners/templates/storageclass-cephfs.yaml diff --git a/ceph-client/templates/storageclass-rbd.yaml b/ceph-provisioners/templates/storageclass-rbd.yaml similarity index 100% rename from ceph-client/templates/storageclass-rbd.yaml rename to ceph-provisioners/templates/storageclass-rbd.yaml diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml new file mode 100644 index 0000000000..f72b18a346 --- /dev/null +++ b/ceph-provisioners/values.yaml @@ -0,0 +1,234 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for ceph-client. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +deployment: + ceph: true + client_secrets: false + rbd_provisioner: true + cephfs_provisioner: true + +release_group: null + +images: + pull_policy: IfNotPresent + tags: + ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' + ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1' + ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3' + ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1' + dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1' + image_repo_sync: docker.io/docker:17.07.0 + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + provisioner: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + dns_policy: "ClusterFirstWithHostNet" + replicas: + cephfs_provisioner: 2 + rbd_provisioner: 2 + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + resources: + enabled: false + rbd_provisioner: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + cephfs_provisioner: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + jobs: + bootstrap: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +secrets: + keyrings: + admin: ceph-client-admin-keyring + +network: + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + port: + mon: 6789 + mgr: 7000 + +conf: + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - ceph-client-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + bootstrap: + jobs: null + services: + - endpoint: internal + service: ceph_mon + cephfs_client_key_generator: + jobs: null + cephfs_provisioner: + jobs: + - ceph-rbd-pool + services: + - endpoint: internal + service: ceph_mon + namespace_client_key_cleaner: + jobs: null + namespace_client_key_generator: + jobs: null + rbd_provisioner: + jobs: + - ceph-rbd-pool + services: + - endpoint: internal + service: ceph_mon + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +bootstrap: + enabled: false + script: | + ceph -s + function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + local test_luminous=$(ceph tell osd.* version | egrep -c "12.2|luminous") + if [[ ${test_luminous} -gt 0 ]]; then + ceph osd pool application enable $1 $3 + fi + } + #ensure_pool volumes 8 cinder + +# if you change provision_storage_class to false +# it is presumed you manage your own storage +# class definition externally +storageclass: + rbd: + provision_storage_class: true + provisioner: ceph.com/rbd + name: general + monitors: null + pool: rbd + admin_id: admin + admin_secret_name: pvc-ceph-conf-combined-storageclass + admin_secret_namespace: ceph + user_id: admin + user_secret_name: pvc-ceph-client-key + image_format: "2" + image_features: layering + cephfs: + provision_storage_class: true + provisioner: ceph.com/cephfs + name: cephfs + admin_id: admin + user_secret_name: pvc-ceph-cephfs-client-key + admin_secret_name: pvc-ceph-conf-combined-storageclass + admin_secret_namespace: ceph + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + +manifests: + configmap_bin: true + configmap_bin_common: true + configmap_etc: true + deployment_rbd_provisioner: true + deployment_cephfs_provisioner: true + job_bootstrap: false + job_cephfs_client_key: true + job_image_repo_sync: true + job_namespace_client_key_cleaner: true + job_namespace_client_key: true + storageclass_cephfs: true + storageclass_rbd: true diff --git a/tools/deployment/armada/multinode/armada-ceph.yaml b/tools/deployment/armada/multinode/armada-ceph.yaml index 7d9869456b..d3d7d17ca6 100644 --- a/tools/deployment/armada/multinode/armada-ceph.yaml +++ b/tools/deployment/armada/multinode/armada-ceph.yaml @@ -273,6 +273,79 @@ data: dependencies: - helm-toolkit --- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: ceph-provisioners +data: + chart_name: ceph-provisioners + release: ceph-provisioners + namespace: ceph + wait: + timeout: 1800 + labels: + release_group: osh-ceph-provisioners + install: + no_hooks: False + upgrade: + no_hooks: False + pre: + delete: + - type: job + labels: + release_group: osh-ceph-provisioners + values: + endpoints: + identity: + namespace: openstack + auth: + swift: + password: ${SWIFT_USER_PASSWORD} + object_store: + namespace: ceph + ceph_mon: + namespace: ceph + network: + public: ${CEPH_NETWORK} + cluster: ${CEPH_NETWORK} + deployment: + storage_secrets: true + ceph: true + rbd_provisioner: true + cephfs_provisioner: true + client_secrets: false + rgw_keystone_user_and_endpoints: false + bootstrap: + enabled: true + conf: + ceph: + global: + fsid: ${CEPH_FS_ID} + rgw_ks: + enabled: true + pool: + crush: + tunables: ${CRUSH_TUNABLES} + target: + # NOTE(portdirect): 5 nodes, with one osd per node + osd: 5 + pg_per_osd: 100 + storage: + osd: + - data: + type: directory + location: /var/lib/openstack-helm/ceph/osd/osd-one + journal: + type: directory + location: /var/lib/openstack-helm/ceph/osd/journal-one + source: + type: local + location: ${OSH_PATH} + subpath: ceph-provisioners + reference: master + dependencies: + - helm-toolkit +--- schema: armada/ChartGroup/v1 metadata: schema: metadata/Document/v1 @@ -285,6 +358,7 @@ data: - ceph-mon - ceph-osd - ceph-client + - ceph-provisioners --- schema: armada/Manifest/v1 metadata: diff --git a/tools/deployment/armada/multinode/armada-lma.yaml b/tools/deployment/armada/multinode/armada-lma.yaml index b319bf2070..a40ae0444c 100644 --- a/tools/deployment/armada/multinode/armada-lma.yaml +++ b/tools/deployment/armada/multinode/armada-lma.yaml @@ -103,7 +103,7 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph-client + subpath: ceph-provisioners reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/armada/multinode/armada-osh.yaml b/tools/deployment/armada/multinode/armada-osh.yaml index 1f11efc529..8c5b560e60 100644 --- a/tools/deployment/armada/multinode/armada-osh.yaml +++ b/tools/deployment/armada/multinode/armada-osh.yaml @@ -107,7 +107,7 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph-client + subpath: ceph-provisioners reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/developer/ceph/040-ceph.sh b/tools/deployment/developer/ceph/040-ceph.sh index cccc7ec94b..f389d06c62 100755 --- a/tools/deployment/developer/ceph/040-ceph.sh +++ b/tools/deployment/developer/ceph/040-ceph.sh @@ -17,7 +17,7 @@ set -xe #NOTE: Lint and package chart -for CHART in ceph-mon ceph-osd ceph-client; do +for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do make "${CHART}" done @@ -163,7 +163,7 @@ pod: rgw: 1 EOF -for CHART in ceph-mon ceph-osd ceph-client; do +for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do helm upgrade --install ${CHART} ./${CHART} \ --namespace=ceph \ --values=/tmp/ceph.yaml \ diff --git a/tools/deployment/developer/ceph/045-ceph-ns-activate.sh b/tools/deployment/developer/ceph/045-ceph-ns-activate.sh index 9b96d3809c..edd5b69d34 100755 --- a/tools/deployment/developer/ceph/045-ceph-ns-activate.sh +++ b/tools/deployment/developer/ceph/045-ceph-ns-activate.sh @@ -17,7 +17,7 @@ set -xe #NOTE: Lint and package chart -make ceph-client +make ceph-provisioners #NOTE: Deploy command : ${OSH_EXTRA_HELM_ARGS:=""} @@ -45,7 +45,7 @@ conf: rgw_ks: enabled: true EOF -helm upgrade --install ceph-openstack-config ./ceph-client \ +helm upgrade --install ceph-openstack-config ./ceph-provisioners \ --namespace=openstack \ --values=/tmp/ceph-openstack-config.yaml \ ${OSH_EXTRA_HELM_ARGS} \ diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh index 7057316274..043c7c795b 100755 --- a/tools/deployment/multinode/030-ceph.sh +++ b/tools/deployment/multinode/030-ceph.sh @@ -76,7 +76,7 @@ conf: location: /var/lib/openstack-helm/ceph/osd/journal-one EOF -for CHART in ceph-mon ceph-osd ceph-client; do +for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do helm upgrade --install ${CHART} ./${CHART} \ --namespace=ceph \ --values=/tmp/ceph.yaml \ diff --git a/tools/deployment/multinode/040-ceph-ns-activate.sh b/tools/deployment/multinode/040-ceph-ns-activate.sh index c4fb8ff576..fc4fb87fdb 100755 --- a/tools/deployment/multinode/040-ceph-ns-activate.sh +++ b/tools/deployment/multinode/040-ceph-ns-activate.sh @@ -43,7 +43,7 @@ conf: rgw_ks: enabled: true EOF -helm upgrade --install ceph-openstack-config ./ceph-client \ +helm upgrade --install ceph-openstack-config ./ceph-provisioners \ --namespace=openstack \ --values=/tmp/ceph-openstack-config.yaml \ ${OSH_EXTRA_HELM_ARGS} \