Refactor ceph to allow monitors to leverage a stateful set.
The IPs of ceph monitors cannot (or should not) change. This refactor allows the ceph monitors to act as statefulsets. It also persists their on disk data to nodeDirs, to allow cluster wide restarts (in parallel), where previously this would lose data. This is accompanied by some docker images changes that ensure that auth for the OSDs is restored/reinserted as auth data is somehow not persisted to disk at this time.
This commit is contained in:
Alan Meadows
parent
bcf23c7399
commit
b366dc252a
@ -78,8 +78,8 @@ spec:
|
|||||||
timeoutSeconds: 5
|
timeoutSeconds: 5
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
memory: "512Mi"
|
memory: {{ .Values.resources.osd.requests.memory | quote }}
|
||||||
cpu: "1000m"
|
cpu: {{ .Values.resources.osd.requests.cpu | quote }}
|
||||||
limits:
|
limits:
|
||||||
memory: "1024Mi"
|
memory: {{ .Values.resources.osd.limits.memory | quote }}
|
||||||
cpu: "2000m"
|
cpu: {{ .Values.resources.osd.limits.cpu | quote }}
|
||||||
72
ceph/templates/deployment-mds.yaml
Normal file
72
ceph/templates/deployment-mds.yaml
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
---
|
||||||
|
kind: Deployment
|
||||||
|
apiVersion: extensions/v1beta1
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: mds
|
||||||
|
name: ceph-mds
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
name: ceph-mds
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: mds
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||||
|
serviceAccount: default
|
||||||
|
volumes:
|
||||||
|
- name: ceph-conf
|
||||||
|
secret:
|
||||||
|
secretName: ceph-conf-combined
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-osd-keyring
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-mds-keyring
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-rgw-keyring
|
||||||
|
containers:
|
||||||
|
- name: ceph-mon
|
||||||
|
image: {{ .Values.images.daemon }}
|
||||||
|
ports:
|
||||||
|
- containerPort: 6800
|
||||||
|
env:
|
||||||
|
- name: CEPH_DAEMON
|
||||||
|
value: MDS
|
||||||
|
- name: CEPHFS_CREATE
|
||||||
|
value: "1"
|
||||||
|
- name: KV_TYPE
|
||||||
|
value: k8s
|
||||||
|
- name: CLUSTER
|
||||||
|
value: ceph
|
||||||
|
volumeMounts:
|
||||||
|
- name: ceph-conf
|
||||||
|
mountPath: /etc/ceph
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-osd
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-mds
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||||
|
livenessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: 6800
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
timeoutSeconds: 5
|
||||||
|
readinessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: 6800
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: {{ .Values.resources.mds.requests.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.mds.requests.cpu | quote }}
|
||||||
|
limits:
|
||||||
|
memory: {{ .Values.resources.mds.limits.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.mds.limits.cpu | quote }}
|
||||||
64
ceph/templates/deployment-moncheck.yaml
Normal file
64
ceph/templates/deployment-moncheck.yaml
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
---
|
||||||
|
kind: Deployment
|
||||||
|
apiVersion: extensions/v1beta1
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: moncheck
|
||||||
|
name: ceph-mon-check
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.replicas.mon_check }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
name: ceph-mon
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: moncheck
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||||
|
serviceAccount: default
|
||||||
|
volumes:
|
||||||
|
- name: ceph-conf
|
||||||
|
secret:
|
||||||
|
secretName: ceph-conf-combined
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-osd-keyring
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-mds-keyring
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-rgw-keyring
|
||||||
|
containers:
|
||||||
|
- name: ceph-mon
|
||||||
|
image: {{ .Values.images.daemon }}
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- containerPort: 6789
|
||||||
|
env:
|
||||||
|
- name: CEPH_DAEMON
|
||||||
|
value: MON_HEALTH
|
||||||
|
- name: KV_TYPE
|
||||||
|
value: k8s
|
||||||
|
- name: MON_IP_AUTO_DETECT
|
||||||
|
value: "1"
|
||||||
|
- name: CLUSTER
|
||||||
|
value: ceph
|
||||||
|
volumeMounts:
|
||||||
|
- name: ceph-conf
|
||||||
|
mountPath: /etc/ceph
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-osd
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-mds
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: {{ .Values.resources.mon_check.requests.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.mon_check.requests.cpu | quote }}
|
||||||
|
limits:
|
||||||
|
memory: {{ .Values.resources.mon_check.limits.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.mon_check.limits.cpu | quote }}
|
||||||
77
ceph/templates/deployment-rgw.yaml
Normal file
77
ceph/templates/deployment-rgw.yaml
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
{{- if .Values.rgw.enabled }}
|
||||||
|
---
|
||||||
|
kind: Deployment
|
||||||
|
apiVersion: extensions/v1beta1
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: rgw
|
||||||
|
name: ceph-rgw
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.replicas.rgw }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
name: ceph-rgw
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: rgw
|
||||||
|
spec:
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||||
|
serviceAccount: default
|
||||||
|
volumes:
|
||||||
|
- name: ceph-conf
|
||||||
|
secret:
|
||||||
|
secretName: ceph-conf-combined
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-osd-keyring
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-mds-keyring
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-rgw-keyring
|
||||||
|
containers:
|
||||||
|
- name: ceph-rgw
|
||||||
|
image: {{ .Values.images.daemon }}
|
||||||
|
ports:
|
||||||
|
- containerPort: {{ .Values.network.port.rgw_target }}
|
||||||
|
env:
|
||||||
|
- name: RGW_CIVETWEB_PORT
|
||||||
|
value: "{{ .Values.network.port.rgw_target }}"
|
||||||
|
- name: CEPH_DAEMON
|
||||||
|
value: RGW
|
||||||
|
- name: KV_TYPE
|
||||||
|
value: k8s
|
||||||
|
- name: CLUSTER
|
||||||
|
value: ceph
|
||||||
|
volumeMounts:
|
||||||
|
- name: ceph-conf
|
||||||
|
mountPath: /etc/ceph
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-osd
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-mds
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: {{ .Values.network.port.rgw_target }}
|
||||||
|
initialDelaySeconds: 120
|
||||||
|
timeoutSeconds: 5
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: {{ .Values.network.port.rgw_target }}
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: {{ .Values.resources.rgw.requests.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.rgwrequests.cpu | quote }}
|
||||||
|
limits:
|
||||||
|
memory: {{ .Values.resources.rgw.limits.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.rgw.limits.cpu | quote }}
|
||||||
|
{{- end }}
|
||||||
@ -1,310 +0,0 @@
|
|||||||
---
|
|
||||||
kind: Deployment
|
|
||||||
apiVersion: extensions/v1beta1
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: mds
|
|
||||||
name: ceph-mds
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
name: ceph-mds
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: mds
|
|
||||||
spec:
|
|
||||||
nodeSelector:
|
|
||||||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
|
||||||
serviceAccount: default
|
|
||||||
volumes:
|
|
||||||
- name: ceph-conf
|
|
||||||
secret:
|
|
||||||
secretName: ceph-conf-combined
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-osd-keyring
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-mds-keyring
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-rgw-keyring
|
|
||||||
containers:
|
|
||||||
- name: ceph-mon
|
|
||||||
image: {{ .Values.images.daemon }}
|
|
||||||
ports:
|
|
||||||
- containerPort: 6800
|
|
||||||
env:
|
|
||||||
- name: CEPH_DAEMON
|
|
||||||
value: MDS
|
|
||||||
- name: CEPHFS_CREATE
|
|
||||||
value: "1"
|
|
||||||
- name: KV_TYPE
|
|
||||||
value: k8s
|
|
||||||
- name: CLUSTER
|
|
||||||
value: ceph
|
|
||||||
volumeMounts:
|
|
||||||
- name: ceph-conf
|
|
||||||
mountPath: /etc/ceph
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-osd
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-mds
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
|
||||||
livenessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 6800
|
|
||||||
initialDelaySeconds: 60
|
|
||||||
timeoutSeconds: 5
|
|
||||||
readinessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 6800
|
|
||||||
timeoutSeconds: 5
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "10Mi"
|
|
||||||
cpu: "250m"
|
|
||||||
limits:
|
|
||||||
memory: "50Mi"
|
|
||||||
cpu: "500m"
|
|
||||||
---
|
|
||||||
kind: Deployment
|
|
||||||
apiVersion: extensions/v1beta1
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: moncheck
|
|
||||||
name: ceph-mon-check
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
name: ceph-mon
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: moncheck
|
|
||||||
spec:
|
|
||||||
serviceAccount: default
|
|
||||||
volumes:
|
|
||||||
- name: ceph-conf
|
|
||||||
secret:
|
|
||||||
secretName: ceph-conf-combined
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-osd-keyring
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-mds-keyring
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-rgw-keyring
|
|
||||||
containers:
|
|
||||||
- name: ceph-mon
|
|
||||||
image: {{ .Values.images.daemon }}
|
|
||||||
imagePullPolicy: Always
|
|
||||||
ports:
|
|
||||||
- containerPort: 6789
|
|
||||||
env:
|
|
||||||
- name: CEPH_DAEMON
|
|
||||||
value: MON_HEALTH
|
|
||||||
- name: KV_TYPE
|
|
||||||
value: k8s
|
|
||||||
- name: MON_IP_AUTO_DETECT
|
|
||||||
value: "1"
|
|
||||||
- name: CLUSTER
|
|
||||||
value: ceph
|
|
||||||
volumeMounts:
|
|
||||||
- name: ceph-conf
|
|
||||||
mountPath: /etc/ceph
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-osd
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-mds
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "5Mi"
|
|
||||||
cpu: "250m"
|
|
||||||
limits:
|
|
||||||
memory: "50Mi"
|
|
||||||
cpu: "500m"
|
|
||||||
---
|
|
||||||
kind: Deployment
|
|
||||||
apiVersion: extensions/v1beta1
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: mon
|
|
||||||
name: ceph-mon
|
|
||||||
spec:
|
|
||||||
replicas: 3
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
name: ceph-mon
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: mon
|
|
||||||
annotations:
|
|
||||||
# alanmeadows: this soft requirement allows single
|
|
||||||
# host deployments to spawn several ceph-mon
|
|
||||||
# containers
|
|
||||||
scheduler.alpha.kubernetes.io/affinity: >
|
|
||||||
{
|
|
||||||
"podAntiAffinity": {
|
|
||||||
"preferredDuringSchedulingIgnoredDuringExecution": [{
|
|
||||||
"labelSelector": {
|
|
||||||
"matchExpressions": [{
|
|
||||||
"key": "daemon",
|
|
||||||
"operator": "In",
|
|
||||||
"values":["mon"]
|
|
||||||
}]
|
|
||||||
},
|
|
||||||
"topologyKey": "kubernetes.io/hostname",
|
|
||||||
"weight": 10
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
spec:
|
|
||||||
serviceAccount: default
|
|
||||||
volumes:
|
|
||||||
- name: ceph-conf
|
|
||||||
secret:
|
|
||||||
secretName: ceph-conf-combined
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-osd-keyring
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-mds-keyring
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-rgw-keyring
|
|
||||||
containers:
|
|
||||||
- name: ceph-mon
|
|
||||||
image: {{ .Values.images.daemon }}
|
|
||||||
# imagePullPolicy: Always
|
|
||||||
lifecycle:
|
|
||||||
preStop:
|
|
||||||
exec:
|
|
||||||
# remove the mon on Pod stop.
|
|
||||||
command:
|
|
||||||
- "/remove-mon.sh"
|
|
||||||
ports:
|
|
||||||
- containerPort: 6789
|
|
||||||
env:
|
|
||||||
- name: CEPH_DAEMON
|
|
||||||
value: MON
|
|
||||||
- name: KV_TYPE
|
|
||||||
value: k8s
|
|
||||||
- name: NETWORK_AUTO_DETECT
|
|
||||||
value: "1"
|
|
||||||
- name: CLUSTER
|
|
||||||
value: ceph
|
|
||||||
volumeMounts:
|
|
||||||
- name: ceph-conf
|
|
||||||
mountPath: /etc/ceph
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-osd
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-mds
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
|
||||||
livenessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 6789
|
|
||||||
initialDelaySeconds: 60
|
|
||||||
timeoutSeconds: 5
|
|
||||||
readinessProbe:
|
|
||||||
tcpSocket:
|
|
||||||
port: 6789
|
|
||||||
timeoutSeconds: 5
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "50Mi"
|
|
||||||
cpu: "1000m"
|
|
||||||
limits:
|
|
||||||
memory: "100Mi"
|
|
||||||
cpu: "2000m"
|
|
||||||
---
|
|
||||||
# rgw not required: using if statement for deployment
|
|
||||||
{{- if .Values.rgw.enabled }}
|
|
||||||
kind: Deployment
|
|
||||||
apiVersion: extensions/v1beta1
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: rgw
|
|
||||||
name: ceph-rgw
|
|
||||||
spec:
|
|
||||||
replicas: 3
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
name: ceph-rgw
|
|
||||||
labels:
|
|
||||||
app: ceph
|
|
||||||
daemon: rgw
|
|
||||||
spec:
|
|
||||||
hostNetwork: true
|
|
||||||
nodeSelector:
|
|
||||||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
|
||||||
serviceAccount: default
|
|
||||||
volumes:
|
|
||||||
- name: ceph-conf
|
|
||||||
secret:
|
|
||||||
secretName: ceph-conf-combined
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-osd-keyring
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-mds-keyring
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
secret:
|
|
||||||
secretName: ceph-bootstrap-rgw-keyring
|
|
||||||
containers:
|
|
||||||
- name: ceph-rgw
|
|
||||||
image: {{ .Values.images.daemon }}
|
|
||||||
ports:
|
|
||||||
- containerPort: {{ .Values.network.port.rgw_target }}
|
|
||||||
env:
|
|
||||||
- name: RGW_CIVETWEB_PORT
|
|
||||||
value: "{{ .Values.network.port.rgw_target }}"
|
|
||||||
- name: CEPH_DAEMON
|
|
||||||
value: RGW
|
|
||||||
- name: KV_TYPE
|
|
||||||
value: k8s
|
|
||||||
- name: CLUSTER
|
|
||||||
value: ceph
|
|
||||||
volumeMounts:
|
|
||||||
- name: ceph-conf
|
|
||||||
mountPath: /etc/ceph
|
|
||||||
- name: ceph-bootstrap-osd-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-osd
|
|
||||||
- name: ceph-bootstrap-mds-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-mds
|
|
||||||
- name: ceph-bootstrap-rgw-keyring
|
|
||||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
|
||||||
livenessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /
|
|
||||||
port: {{ .Values.network.port.rgw_target }}
|
|
||||||
initialDelaySeconds: 120
|
|
||||||
timeoutSeconds: 5
|
|
||||||
readinessProbe:
|
|
||||||
httpGet:
|
|
||||||
path: /
|
|
||||||
port: {{ .Values.network.port.rgw_target }}
|
|
||||||
timeoutSeconds: 5
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "500Mi"
|
|
||||||
cpu: ".5"
|
|
||||||
limits:
|
|
||||||
memory: "500Mi"
|
|
||||||
cpu: ".5"
|
|
||||||
{{ end }}
|
|
||||||
# end: rgw removed optionally
|
|
||||||
@ -15,6 +15,8 @@ spec:
|
|||||||
app: ceph
|
app: ceph
|
||||||
daemon: mon
|
daemon: mon
|
||||||
clusterIP: None
|
clusterIP: None
|
||||||
|
|
||||||
|
{{- if .Values.rgw.enabled }}
|
||||||
---
|
---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Service
|
kind: Service
|
||||||
@ -32,3 +34,4 @@ spec:
|
|||||||
app: ceph
|
app: ceph
|
||||||
daemon: rgw
|
daemon: rgw
|
||||||
type: LoadBalancer
|
type: LoadBalancer
|
||||||
|
{{- end }}
|
||||||
|
|||||||
105
ceph/templates/statefulset-mon.yaml
Normal file
105
ceph/templates/statefulset-mon.yaml
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
---
|
||||||
|
apiVersion: apps/v1beta1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: mon
|
||||||
|
name: ceph-mon
|
||||||
|
spec:
|
||||||
|
serviceName: {{ .Values.service.mon.name | quote }}
|
||||||
|
replicas: {{ .Values.replicas.mon }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
name: ceph-mon
|
||||||
|
labels:
|
||||||
|
app: ceph
|
||||||
|
daemon: mon
|
||||||
|
annotations:
|
||||||
|
# alanmeadows: this soft requirement allows single
|
||||||
|
# host deployments to spawn several ceph-mon
|
||||||
|
# containers
|
||||||
|
scheduler.alpha.kubernetes.io/affinity: >
|
||||||
|
{
|
||||||
|
"podAntiAffinity": {
|
||||||
|
"preferredDuringSchedulingIgnoredDuringExecution": [{
|
||||||
|
"labelSelector": {
|
||||||
|
"matchExpressions": [{
|
||||||
|
"key": "daemon",
|
||||||
|
"operator": "In",
|
||||||
|
"values":["mon"]
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
"topologyKey": "kubernetes.io/hostname",
|
||||||
|
"weight": 10
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||||
|
serviceAccount: default
|
||||||
|
volumes:
|
||||||
|
- name: ceph-conf
|
||||||
|
secret:
|
||||||
|
secretName: ceph-conf-combined
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-osd-keyring
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-mds-keyring
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ceph-bootstrap-rgw-keyring
|
||||||
|
- name: ceph-monfs
|
||||||
|
hostPath:
|
||||||
|
path: {{ .Values.storage.mon_directory }}
|
||||||
|
containers:
|
||||||
|
- name: ceph-mon
|
||||||
|
image: {{ .Values.images.daemon }}
|
||||||
|
imagePullPolicy: Always
|
||||||
|
lifecycle:
|
||||||
|
preStop:
|
||||||
|
exec:
|
||||||
|
# remove the mon on Pod stop.
|
||||||
|
command:
|
||||||
|
- "/remove-mon.sh"
|
||||||
|
ports:
|
||||||
|
- containerPort: 6789
|
||||||
|
env:
|
||||||
|
- name: CEPH_DAEMON
|
||||||
|
value: MON
|
||||||
|
- name: KV_TYPE
|
||||||
|
value: k8s
|
||||||
|
- name: NETWORK_AUTO_DETECT
|
||||||
|
value: "1"
|
||||||
|
- name: CLUSTER
|
||||||
|
value: ceph
|
||||||
|
volumeMounts:
|
||||||
|
- name: ceph-conf
|
||||||
|
mountPath: /etc/ceph
|
||||||
|
- name: ceph-bootstrap-osd-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-osd
|
||||||
|
- name: ceph-bootstrap-mds-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-mds
|
||||||
|
- name: ceph-bootstrap-rgw-keyring
|
||||||
|
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||||
|
- name: ceph-monfs
|
||||||
|
mountPath: /var/lib/ceph/mon
|
||||||
|
livenessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: 6789
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
timeoutSeconds: 5
|
||||||
|
readinessProbe:
|
||||||
|
tcpSocket:
|
||||||
|
port: 6789
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: {{ .Values.resources.mon.requests.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.mon.requests.cpu | quote }}
|
||||||
|
limits:
|
||||||
|
memory: {{ .Values.resources.mon.limits.memory | quote }}
|
||||||
|
cpu: {{ .Values.resources.mon.limits.cpu | quote }}
|
||||||
@ -10,7 +10,8 @@ parameters:
|
|||||||
monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }}
|
monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }}
|
||||||
adminId: {{ .Values.storageclass.admin_id }}
|
adminId: {{ .Values.storageclass.admin_id }}
|
||||||
adminSecretName: {{ .Values.storageclass.admin_secret_name }}
|
adminSecretName: {{ .Values.storageclass.admin_secret_name }}
|
||||||
## forcing namespace due to issue with -- default "{{ .Release.Namespace }}" }} --
|
# forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }}
|
||||||
|
# during helm lint
|
||||||
adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }}
|
adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }}
|
||||||
pool: {{ .Values.storageclass.pool }}
|
pool: {{ .Values.storageclass.pool }}
|
||||||
userId: {{ .Values.storageclass.user_id }}
|
userId: {{ .Values.storageclass.user_id }}
|
||||||
|
|||||||
@ -7,6 +7,15 @@
|
|||||||
# tunables available - parameterizing more of the elements
|
# tunables available - parameterizing more of the elements
|
||||||
# in the manifests is a work in progress
|
# in the manifests is a work in progress
|
||||||
|
|
||||||
|
replicas:
|
||||||
|
mon: 3
|
||||||
|
rgw: 3
|
||||||
|
mon_check: 1
|
||||||
|
|
||||||
|
service:
|
||||||
|
mon:
|
||||||
|
name: ceph-mon
|
||||||
|
|
||||||
images:
|
images:
|
||||||
daemon: quay.io/attcomdev/ceph-daemon:latest
|
daemon: quay.io/attcomdev/ceph-daemon:latest
|
||||||
|
|
||||||
@ -23,11 +32,52 @@ network:
|
|||||||
storage:
|
storage:
|
||||||
osd_directory: /var/lib/openstack-helm/ceph/osd
|
osd_directory: /var/lib/openstack-helm/ceph/osd
|
||||||
var_directory: /var/lib/openstack-helm/ceph/ceph
|
var_directory: /var/lib/openstack-helm/ceph/ceph
|
||||||
|
mon_directory: /var/lib/openstack-helm/ceph/mon
|
||||||
|
|
||||||
# rgw is optionall disabled
|
# rgw is optionall disabled
|
||||||
rgw:
|
rgw:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
|
rgw:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
resources:
|
||||||
|
osd:
|
||||||
|
requests:
|
||||||
|
memory: "512Mi"
|
||||||
|
cpu: "1000m"
|
||||||
|
limits:
|
||||||
|
memory: "1024Mi"
|
||||||
|
cpu: "2000m"
|
||||||
|
mds:
|
||||||
|
requests:
|
||||||
|
memory: "10Mi"
|
||||||
|
cpu: "250m"
|
||||||
|
limits:
|
||||||
|
memory: "50Mi"
|
||||||
|
cpu: "500m"
|
||||||
|
mon:
|
||||||
|
requests:
|
||||||
|
memory: "50Mi"
|
||||||
|
cpu: "1000m"
|
||||||
|
limits:
|
||||||
|
memory: "100Mi"
|
||||||
|
cpu: "2000m"
|
||||||
|
mon_check:
|
||||||
|
requests:
|
||||||
|
memory: "5Mi"
|
||||||
|
cpu: "250m"
|
||||||
|
limits:
|
||||||
|
memory: "50Mi"
|
||||||
|
cpu: "500m"
|
||||||
|
rgw:
|
||||||
|
requests:
|
||||||
|
memory: "5Mi"
|
||||||
|
cpu: "250m"
|
||||||
|
limits:
|
||||||
|
memory: "50Mi"
|
||||||
|
cpu: "500m"
|
||||||
|
|
||||||
# Setting this to false will assume you will
|
# Setting this to false will assume you will
|
||||||
# setup and orchestrate your own secrets and
|
# setup and orchestrate your own secrets and
|
||||||
# configmaps outside of this helm chart
|
# configmaps outside of this helm chart
|
||||||
@ -63,7 +113,7 @@ secrets:
|
|||||||
storageclass:
|
storageclass:
|
||||||
provision_storage_class: true
|
provision_storage_class: true
|
||||||
name: general
|
name: general
|
||||||
monitors: null
|
monitors: null
|
||||||
pool: rbd
|
pool: rbd
|
||||||
admin_id: admin
|
admin_id: admin
|
||||||
admin_secret_name: pvc-ceph-conf-combined-storageclass
|
admin_secret_name: pvc-ceph-conf-combined-storageclass
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user