From f3b7c7fc820f3027ae38d5120abc374a576d7236 Mon Sep 17 00:00:00 2001
From: Renis <renis.makadia@att.com>
Date: Wed, 8 Aug 2018 21:22:52 -0700
Subject: [PATCH] Make Ceph Mon port configurable
- make mon port overridable through env variable
- update ceph.conf to include fqdn with port number
- add mon_add: to ceph global conf
- updating ClusterRoleBinding name for cephfs and rbd
provisioner.
- remove unused network.port section as we specify ports
through endpoints
With these changes, ceph mon can run on different port
Change-Id: I8af2bbe3ee84fd3692bdff92d821b9621a6d53c9
---
.../templates/configmap-etc-client.yaml | 7 ++++++-
ceph-mon/templates/bin/mon/_start.sh.tpl | 10 +++++-----
ceph-mon/templates/configmap-etc.yaml | 7 ++++++-
ceph-mon/templates/daemonset-mon.yaml | 4 +++-
ceph-mon/templates/deployment-moncheck.yaml | 2 +-
ceph-osd/templates/configmap-etc.yaml | 7 ++++++-
ceph-osd/values.yaml | 4 ----
.../templates/configmap-etc-client.yaml | 7 ++++++-
.../deployment-cephfs-provisioner.yaml | 2 +-
.../templates/deployment-rbd-provisioner.yaml | 2 +-
ceph-provisioners/values.yaml | 3 ---
tools/deployment/developer/ceph/040-ceph.sh | 19 +++++++++++++++++++
12 files changed, 54 insertions(+), 20 deletions(-)
diff --git a/ceph-client/templates/configmap-etc-client.yaml b/ceph-client/templates/configmap-etc-client.yaml
index 0e1610b927..fb2a5adc8e 100644
--- a/ceph-client/templates/configmap-etc-client.yaml
+++ b/ceph-client/templates/configmap-etc-client.yaml
@@ -22,10 +22,15 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
{{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
{{- end -}}
diff --git a/ceph-mon/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl
index 24b6ed8e21..9588d9384b 100644
--- a/ceph-mon/templates/bin/mon/_start.sh.tpl
+++ b/ceph-mon/templates/bin/mon/_start.sh.tpl
@@ -39,9 +39,9 @@ function get_mon_config {
while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do
# Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params
if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
- MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}")
+ MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
else
- MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}")
+ MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
fi
(( timeout-- ))
sleep 1
@@ -53,7 +53,7 @@ function get_mon_config {
# if monmap exists and the mon is already there, don't overwrite monmap
if [ -f "${MONMAP}" ]; then
- monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":6789"
+ monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":${MON_PORT}"
if [ $? -eq 0 ]; then
echo "${MON_IP} already exists in monmap ${MONMAP}"
return
@@ -93,7 +93,7 @@ else
# no mons are up and running yet
timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true
ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
- timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:6789" || true
+ timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT}" || true
fi
# start MON
@@ -104,4 +104,4 @@ exec /usr/bin/ceph-mon \
-d \
-i ${MON_NAME} \
--mon-data "${MON_DATA_DIR}" \
- --public-addr "${MON_IP}:6789"
+ --public-addr "${MON_IP}:${MON_PORT}"
diff --git a/ceph-mon/templates/configmap-etc.yaml b/ceph-mon/templates/configmap-etc.yaml
index 148dcc2249..ac4b1e7b03 100644
--- a/ceph-mon/templates/configmap-etc.yaml
+++ b/ceph-mon/templates/configmap-etc.yaml
@@ -22,10 +22,15 @@ limitations under the License.
{{- if .Values.deployment.ceph }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "discovery" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
{{- if empty .Values.conf.ceph.global.fsid -}}
{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
{{- end -}}
diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml
index c70435b294..4709436309 100644
--- a/ceph-mon/templates/daemonset-mon.yaml
+++ b/ceph-mon/templates/daemonset-mon.yaml
@@ -106,6 +106,8 @@ spec:
value: {{ .Values.network.public | quote }}
- name: KUBECTL_PARAM
value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }}
+ - name: MON_PORT
+ value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_IP
valueFrom:
fieldRef:
@@ -126,7 +128,7 @@ spec:
command:
- /tmp/mon-stop.sh
ports:
- - containerPort: 6789
+ - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
livenessProbe:
exec:
command:
diff --git a/ceph-mon/templates/deployment-moncheck.yaml b/ceph-mon/templates/deployment-moncheck.yaml
index ffce5c52d6..70f8e109ae 100644
--- a/ceph-mon/templates/deployment-moncheck.yaml
+++ b/ceph-mon/templates/deployment-moncheck.yaml
@@ -60,7 +60,7 @@ spec:
command:
- /tmp/moncheck-start.sh
ports:
- - containerPort: 6789
+ - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
volumeMounts:
- name: ceph-mon-bin
mountPath: /tmp/moncheck-start.sh
diff --git a/ceph-osd/templates/configmap-etc.yaml b/ceph-osd/templates/configmap-etc.yaml
index e37eeb0765..85ab873b9e 100644
--- a/ceph-osd/templates/configmap-etc.yaml
+++ b/ceph-osd/templates/configmap-etc.yaml
@@ -20,10 +20,15 @@ limitations under the License.
{{- with $envAll }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
{{- if empty .Values.conf.ceph.global.fsid -}}
{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
{{- end -}}
diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml
index 7ee506b026..c40dd43dc1 100644
--- a/ceph-osd/values.yaml
+++ b/ceph-osd/values.yaml
@@ -79,10 +79,6 @@ secrets:
network:
public: 192.168.0.0/16
cluster: 192.168.0.0/16
- port:
- mon: 6789
- rgw: 8088
- mgr: 7000
conf:
ceph:
diff --git a/ceph-provisioners/templates/configmap-etc-client.yaml b/ceph-provisioners/templates/configmap-etc-client.yaml
index 3592990580..e3001bd686 100644
--- a/ceph-provisioners/templates/configmap-etc-client.yaml
+++ b/ceph-provisioners/templates/configmap-etc-client.yaml
@@ -22,10 +22,15 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
-{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
+{{- if empty .Values.conf.ceph.global.mon_addr -}}
+{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
+{{- end -}}
+
{{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
{{- end -}}
diff --git a/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml
index 6548a31738..4830fba4e9 100644
--- a/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml
+++ b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml
@@ -91,7 +91,7 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
- name: run-cephfs-provisioner
+ name: {{ $serviceAccountName }}-run-cephfs-provisioner
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
diff --git a/ceph-provisioners/templates/deployment-rbd-provisioner.yaml b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml
index ad21d36af0..99c7e3016b 100644
--- a/ceph-provisioners/templates/deployment-rbd-provisioner.yaml
+++ b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml
@@ -81,7 +81,7 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
- name: run-rbd-provisioner
+ name: {{ $serviceAccountName }}-run-rbd-provisioner
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml
index 2b8fd166e7..feb817e7d0 100644
--- a/ceph-provisioners/values.yaml
+++ b/ceph-provisioners/values.yaml
@@ -98,9 +98,6 @@ secrets:
network:
public: 192.168.0.0/16
cluster: 192.168.0.0/16
- port:
- mon: 6789
- mgr: 7000
conf:
ceph:
diff --git a/tools/deployment/developer/ceph/040-ceph.sh b/tools/deployment/developer/ceph/040-ceph.sh
index f389d06c62..53ba741b74 100755
--- a/tools/deployment/developer/ceph/040-ceph.sh
+++ b/tools/deployment/developer/ceph/040-ceph.sh
@@ -40,11 +40,29 @@ endpoints:
namespace: openstack
object_store:
namespace: ceph
+ port:
+ api:
+ default: 8088
+ public: 80
ceph_mon:
namespace: ceph
+ port:
+ mon:
+ default: 6789
+ ceph_mgr:
+ namespace: ceph
+ port:
+ mgr:
+ default: 7000
+ metrics:
+ default: 9283
network:
public: 172.17.0.1/16
cluster: 172.17.0.1/16
+ port:
+ mon: 6789
+ rgw: 8088
+ mgr: 7000
deployment:
storage_secrets: true
ceph: true
@@ -60,6 +78,7 @@ conf:
ceph:
global:
fsid: ${CEPH_FS_ID}
+ mon_addr: :6789
osd_pool_default_size: 1
osd:
osd_crush_chooseleaf_type: 0