openstack/openstack-helm
Code Issues Proposed changes

Merge "Enable Ceph CSI Provisioner to Stand Alone"

Browse Source
This commit is contained in:
Zuul 2021-06-21 04:17:52 +00:00 committed by Gerrit Code Review
commit 196592ded6
32 changed files with 126 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ceph-mon/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Ceph Mon
name: ceph-mon
version: 0.1.8
version: 0.1.9
home: https://github.com/ceph/ceph
...

13
ceph-mon/values.yaml
View File

@ -305,21 +305,8 @@ bootstrap:
# and derive the manifest.
storageclass:
rbd:
provision_storage_class: true
provisioner: ceph.com/rbd
ceph_configmap_name: ceph-etc
metadata:
default_storage_class: true
name: general
parameters:
pool: rbd
adminId: admin
adminSecretName: pvc-ceph-conf-combined-storageclass
adminSecretNamespace: ceph
userId: admin
userSecretName: pvc-ceph-client-key
imageFormat: "2"
imageFeatures: layering
cephfs:
provision_storage_class: true
provisioner: ceph.com/cephfs

2
ceph-provisioners/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Ceph Provisioner
name: ceph-provisioners
version: 0.1.7
version: 0.1.8
home: https://github.com/ceph/ceph
...

4
ceph-provisioners/templates/configmap-etc-client.yaml
View File

@ -46,5 +46,9 @@ data:
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
{{- list .Values.storageclass.csi_rbd.ceph_configmap_name . | include "ceph.configmap.etc" }}
{{- else }}
{{- list .Values.storageclass.rbd.ceph_configmap_name . | include "ceph.configmap.etc" }}
{{- end }}
{{- end }}

4
ceph-provisioners/templates/configmap-etc-csi.yaml
View File

@ -17,7 +17,7 @@ limitations under the License.
{{- $envAll := index . 1 }}
{{- with $envAll }}
{{- if and (.Values.deployment.ceph) (.Values.deployment.csi) }}
{{- if and (.Values.deployment.ceph) (.Values.deployment.csi_rbd_provisioner) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
@ -44,5 +44,5 @@ metadata:
{{- end }}
{{- if .Values.manifests.configmap_etc }}
{{- list .Values.storageclass.rbd.ceph_configmap_name . | include "ceph.configmap.etc.csi" }}
{{- list .Values.storageclass.csi_rbd.ceph_configmap_name . | include "ceph.configmap.etc.csi" }}
{{- end }}

2
ceph-provisioners/templates/daemonset-csi-rbd-plugin.yaml
View File

@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.csi_rbd_provisioner }}
{{- $envAll := . }}
{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-nodeplugin" }}

8
ceph-provisioners/templates/deployment-csi-rbd-provisioner.yaml
View File

@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.rbd_provisioner }}
{{- if and .Values.manifests.deployment_csi_rbd_provisioner .Values.deployment.csi_rbd_provisioner }}
{{- $envAll := . }}
{{- $serviceAccountName := printf "%s-%s" .Release.Name "ceph-rbd-csi-provisioner" }}
@ -112,7 +112,7 @@ metadata:
labels:
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
replicas: {{ .Values.pod.replicas.rbd_provisioner }}
replicas: {{ .Values.pod.replicas.csi_rbd_provisioner }}
selector:
matchLabels:
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
@ -129,7 +129,7 @@ spec:
serviceAccountName: {{ $serviceAccountName }}
affinity:
{{ tuple $envAll "rbd" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
{{ tuple $envAll "rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
{{ tuple $envAll "csi_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }}
nodeSelector:
{{ .Values.labels.provisioner.node_selector_key }}: {{ .Values.labels.provisioner.node_selector_value }}
initContainers:
@ -137,7 +137,7 @@ spec:
containers:
- name: ceph-rbd-provisioner
{{ tuple $envAll "csi_provisioner" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.csi_rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "provisioner" "container" "ceph_rbd_provisioner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: DEPLOYMENT_NAMESPACE

4
ceph-provisioners/templates/job-bootstrap.yaml
View File

@ -72,7 +72,11 @@ spec:
defaultMode: 0555
- name: ceph-etc
configMap:
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
name: {{ .Values.storageclass.csi_rbd.ceph_configmap_name }}
{{- else }}
name: {{ .Values.storageclass.rbd.ceph_configmap_name }}
{{- end }}
defaultMode: 0444
- name: ceph-client-admin-keyring
secret:

4
ceph-provisioners/templates/job-cephfs-client-key.yaml
View File

@ -52,7 +52,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
namespace: {{ .Values.storageclass.cephfs.parameters.adminSecretNamespace }}
rules:
- apiGroups:
- ""
@ -66,7 +66,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
namespace: {{ .Values.storageclass.cephfs.parameters.adminSecretNamespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role

16
ceph-provisioners/templates/job-namespace-client-ceph-config.yaml
View File

@ -52,7 +52,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
rules:
- apiGroups:
- ""
@ -66,7 +70,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
@ -102,13 +110,21 @@ spec:
{{ dict "envAll" $envAll "application" "client_ceph_config_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: CEPH_CONF_ETC
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
value: {{ .Values.storageclass.csi_rbd.ceph_configmap_name }}
{{- else }}
value: {{ .Values.storageclass.rbd.ceph_configmap_name }}
{{- end }}
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
value: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
value: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_PORT_V2

4
ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml
View File

@ -77,7 +77,11 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
value: {{ .Values.storageclass.csi_rbd.parameters.userSecretName }}
{{- else }}
value: {{ .Values.storageclass.rbd.parameters.userSecretName }}
{{- end }}
command:
- /tmp/provisioner-rbd-namespace-client-key-cleaner.sh
volumeMounts:

17
ceph-provisioners/templates/job-namespace-client-key.yaml
View File

@ -52,7 +52,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
rules:
- apiGroups:
- ""
@ -66,7 +70,11 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
namespace: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
namespace: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
@ -105,12 +113,21 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if eq .Values.storageclass.csi_rbd.provision_storage_class true }}
- name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
value: {{ .Values.storageclass.csi_rbd.parameters.userSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.storageclass.csi_rbd.parameters.adminSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
value: {{ .Values.storageclass.csi_rbd.parameters.adminSecretNamespace }}
{{- else }}
- name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
value: {{ .Values.storageclass.rbd.parameters.userSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.storageclass.rbd.parameters.adminSecretName }}
- name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
value: {{ .Values.storageclass.rbd.parameters.adminSecretNamespace }}
{{- end }}
command:
- /tmp/provisioner-rbd-namespace-client-key-manager.sh
volumeMounts:

36
ceph-provisioners/values.yaml
View File

@ -19,8 +19,10 @@
deployment:
ceph: true
client_secrets: false
# Original rbd_provisioner is now DEPRECATED. It will be removed in the
# next release; CSI RBD provisioner should be used instead.
rbd_provisioner: true
csi: true
csi_rbd_provisioner: true
cephfs_provisioner: true
release_group: null
@ -144,6 +146,7 @@ pod:
replicas:
cephfs_provisioner: 2
rbd_provisioner: 2
csi_rbd_provisioner: 2
lifecycle:
upgrades:
deployments:
@ -171,6 +174,13 @@ pod:
limits:
memory: "50Mi"
cpu: "500m"
csi_rbd_provisioner:
requests:
memory: "5Mi"
cpu: "250m"
limits:
memory: "50Mi"
cpu: "500m"
cephfs_provisioner:
requests:
memory: "5Mi"
@ -239,6 +249,16 @@ pod:
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 60
csi_rbd_provisioner:
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 60
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 60
cephfs_provisioner:
tolerations:
- effect: NoExecute
@ -312,6 +332,12 @@ dependencies:
services:
- endpoint: internal
service: ceph_mon
csi_rbd_provisioner:
jobs:
- ceph-rbd-pool
services:
- endpoint: internal
service: ceph_mon
image_repo_sync:
services:
- endpoint: internal
@ -353,6 +379,7 @@ storageclass:
csi_rbd:
provision_storage_class: true
provisioner: ceph.rbd.csi.ceph.com
ceph_configmap_name: ceph-etc
metadata:
default_storage_class: true
name: general
@ -368,6 +395,11 @@ storageclass:
imageFeatures: layering
imageFormat: "2"
pool: rbd
adminId: admin
adminSecretName: pvc-ceph-conf-combined-storageclass
adminSecretNamespace: ceph
userId: admin
userSecretName: pvc-ceph-client-key
cephfs:
provision_storage_class: true
provisioner: ceph.com/cephfs
@ -411,6 +443,8 @@ manifests:
configmap_bin_common: true
configmap_etc: true
deployment_rbd_provisioner: true
# Original rbd_provisioner is now DEPRECATED. It will be removed in the
# next release; CSI RBD provisioner should be used instead.
deployment_csi_rbd_provisioner: true
deployment_cephfs_provisioner: true
job_bootstrap: false

1
doc/source/testing/ceph-resiliency/failure-domain.rst
View File

@ -696,6 +696,7 @@ An example of a lab enviroment had the following paramters set for the ceph yaml
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: true
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
releasenotes/notes/ceph-mon.yaml
View File

@ -9,4 +9,5 @@ ceph-mon:
- 0.1.6 Fix python3 issue for util scripts
- 0.1.7 remove deprecated svc annotation tolerate-unready-endpoints
- 0.1.8 Use full image ref for docker official images
- 0.1.9 Remove unnecessary parameters for ceph-mon
...

1
releasenotes/notes/ceph-provisioners.yaml
View File

@ -8,4 +8,5 @@ ceph-provisioners:
- 0.1.5 Fix Helm tests for the Ceph provisioners
- 0.1.6 Update ceph_mon config as per new ceph clients
- 0.1.7 Use full image ref for docker official images
- 0.1.8 Enable Ceph CSI Provisioner to Stand Alone
...

1
tools/deployment/armada/manifests/armada-ceph.yaml
View File

@ -293,6 +293,7 @@ data:
deployment:
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: false
client_secrets: false
storageclass:

2
tools/deployment/armada/manifests/armada-lma.yaml
View File

@ -93,6 +93,7 @@ data:
deployment:
ceph: False
rbd_provisioner: False
csi_rbd_provisioner: False
cephfs_provisioner: False
client_secrets: True
storageclass:
@ -166,6 +167,7 @@ data:
deployment:
ceph: True
rbd_provisioner: False
csi_rbd_provisioner: False
cephfs_provisioner: False
client_secrets: False
rgw_keystone_user_and_endpoints: False

1
tools/deployment/multinode/030-ceph.sh
View File

@ -56,6 +56,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/multinode/035-ceph-ns-activate.sh
View File

@ -28,6 +28,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/multinode/115-radosgw-osh-infra.sh
View File

@ -33,6 +33,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/openstack-support/025-ceph-ns-activate.sh
View File

@ -30,6 +30,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging-tls/020-ceph.sh
View File

@ -62,6 +62,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: true
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging-tls/025-ceph-ns-activate.sh
View File

@ -30,6 +30,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging-tls/030-radosgw-osh-infra.sh
View File

@ -34,6 +34,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging/020-ceph.sh
View File

@ -62,6 +62,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
csi_rbd_provisioner: true
cephfs_provisioner: true
client_secrets: false
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
View File

@ -30,6 +30,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false

1
tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
View File

@ -31,6 +31,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

7
tools/deployment/tenant-ceph/030-ceph.sh
View File

@ -56,7 +56,8 @@ network:
deployment:
storage_secrets: true
ceph: true
rbd_provisioner: true
rbd_provisioner: false
csi_rbd_provisioner: true
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false
@ -107,8 +108,10 @@ conf:
location: ${CEPH_OSD_DB_WAL_DEVICE}
size: "2GB"
storageclass:
rbd:
csi_rbd:
ceph_configmap_name: ceph-etc
rbd:
provision_storage_class: false
cephfs:
provision_storage_class: false
ceph_mgr_modules_config:

1
tools/deployment/tenant-ceph/040-tenant-ceph.sh
View File

@ -65,6 +65,7 @@ deployment:
storage_secrets: true
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false

15
tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh
View File

@ -35,6 +35,7 @@ deployment:
storage_secrets: false
ceph: false
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: true
rgw_keystone_user_and_endpoints: false
@ -45,16 +46,24 @@ conf:
enabled: true
storageclass:
rbd:
ceph_configmap_name: tenant-ceph-etc
provision_storage_class: false
provision_storage_class: true
metadata:
name: tenant-rbd
parameters:
adminSecretName: pvc-tenant-ceph-conf-combined-storageclass
adminSecretNamespace: tenant-ceph
userSecretName: pvc-tenant-ceph-client-key
csi_rbd:
ceph_configmap_name: tenant-ceph-etc
provision_storage_class: true
metadata:
name: tenant-csi-rbd
parameters:
adminSecretName: pvc-tenant-ceph-conf-combined-storageclass
adminSecretNamespace: tenant-ceph
userSecretName: pvc-tenant-ceph-client-key
cephfs:
provision_storage_class: false
provision_storage_class: true
metadata:
name: cephfs
parameters:

1
tools/deployment/tenant-ceph/060-radosgw-openstack.sh
View File

@ -38,6 +38,7 @@ deployment:
storage_secrets: false
ceph: true
rbd_provisioner: false
csi_rbd_provisioner: false
cephfs_provisioner: false
client_secrets: false
rgw_keystone_user_and_endpoints: false