From cd519db139347faaff2b0037c00785d507a1fa02 Mon Sep 17 00:00:00 2001 From: wangwei Date: Mon, 22 Apr 2019 19:11:48 +0900 Subject: [PATCH] Support mon and osd to be named with hostname In the current deployment of ceph, the node name of osd and the name of mon are both IP, and other daemons use hostname. This commit adds support for naming mon and osd nodes using hostname, and does not change the default ip-named way. Change-Id: I22bef72dcd8fc8bcd391ae30e4643520250fd556 --- ansible/group_vars/all.yml | 13 ++++++++++++ ansible/roles/ceph/defaults/main.yml | 21 +++++++++++++++++++ ansible/roles/ceph/tasks/bootstrap_osds.yml | 4 ++-- ansible/roles/ceph/tasks/generate_cluster.yml | 4 ++-- ansible/roles/ceph/tasks/start_mdss.yml | 13 ++++-------- ansible/roles/ceph/tasks/start_mgrs.yml | 13 ++++-------- ansible/roles/ceph/tasks/start_mons.yml | 2 +- ansible/roles/ceph/templates/ceph-mds.json.j2 | 6 +++--- ansible/roles/ceph/templates/ceph-mgr.json.j2 | 6 +++--- ansible/roles/ceph/templates/ceph-mon.json.j2 | 2 +- ansible/roles/ceph/templates/ceph-osd.json.j2 | 2 +- ansible/roles/ceph/templates/ceph.conf.j2 | 8 +++++++ 12 files changed, 63 insertions(+), 31 deletions(-) diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 9b5922f3e6..97aae5b901 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -68,6 +68,9 @@ container_proxy: # to the api_interface. Allow the bind address to be an override. api_interface_address: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}" +# This is used to get the ip corresponding to the storage_interface. +storage_interface_address: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + ################ # Chrony options ################ @@ -968,6 +971,16 @@ ceph_pool_pgp_num: 8 # Valid options are [ filestore, bluestore] ceph_osd_store_type: "bluestore" +# Set the host type for ceph daemons +# Valid options are [ IP, HOSTNAME, FQDN, INVENTORY ] +# Note: For existing clusters, please don't modify this parameter. Otherwise, +# the existing mon will be invalidated, and the existing osd crush map will +# be changed. +ceph_mon_host_type: "IP" +ceph_mgr_host_type: "INVENTORY" +ceph_osd_host_type: "IP" +ceph_mds_host_type: "INVENTORY" + ##################### # VMware support ###################### diff --git a/ansible/roles/ceph/defaults/main.yml b/ansible/roles/ceph/defaults/main.yml index 32fc9216b6..2c94b9da8a 100644 --- a/ansible/roles/ceph/defaults/main.yml +++ b/ansible/roles/ceph/defaults/main.yml @@ -77,6 +77,27 @@ partition_name_osd_bootstrap: "{{ 'KOLLA_CEPH_OSD_BOOTSTRAP_BS' if ceph_osd_stor partition_name_cache_bootstrap: "{{ 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' }}" partition_name_osd_data: "{{ 'KOLLA_CEPH_DATA_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_DATA' }}" +ceph_mon_hostname: "{%- if ceph_mon_host_type == 'HOSTNAME' -%}{{ ansible_hostname }} + {%- elif ceph_mon_host_type == 'FQDN' -%}{{ ansible_fqdn }} + {%- elif ceph_mon_host_type == 'INVENTORY' -%}{{ inventory_hostname }} + {%- else -%}{{ storage_interface_address }} + {%- endif %}" +ceph_mgr_hostname: "{%- if ceph_mgr_host_type == 'HOSTNAME' -%}{{ ansible_hostname }} + {%- elif ceph_mgr_host_type == 'FQDN' -%}{{ ansible_fqdn }} + {%- elif ceph_mgr_host_type == 'INVENTORY' -%}{{ inventory_hostname }} + {%- else -%}{{ storage_interface_address }} + {%- endif %}" +ceph_osd_hostname: "{%- if ceph_osd_host_type == 'HOSTNAME' -%}{{ ansible_hostname }} + {%- elif ceph_osd_host_type == 'FQDN' -%}{{ ansible_fqdn }} + {%- elif ceph_osd_host_type == 'INVENTORY' -%}{{ inventory_hostname }} + {%- else -%}{{ storage_interface_address }} + {%- endif %}" +ceph_mds_hostname: "{%- if ceph_mds_host_type == 'HOSTNAME' -%}{{ ansible_hostname }} + {%- elif ceph_mds_host_type == 'FQDN' -%}{{ ansible_fqdn }} + {%- elif ceph_mds_host_type == 'INVENTORY' -%}{{ inventory_hostname }} + {%- else -%}{{ storage_interface_address }} + {%- endif %}" + #################### ## Ceph_rgw_keystone #################### diff --git a/ansible/roles/ceph/tasks/bootstrap_osds.yml b/ansible/roles/ceph/tasks/bootstrap_osds.yml index aef33395e7..d6af5f1c5b 100644 --- a/ansible/roles/ceph/tasks/bootstrap_osds.yml +++ b/ansible/roles/ceph/tasks/bootstrap_osds.yml @@ -59,7 +59,7 @@ USE_EXTERNAL_JOURNAL: "{{ item.1.external_journal | bool }}" OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}" OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}" - HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + HOSTNAME: "{{ ceph_osd_hostname }}" OSD_STORETYPE: "{{ ceph_osd_store_type }}" OSD_BS_DEV: "{{ item.1.device | default('') }}" OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}" @@ -120,7 +120,7 @@ USE_EXTERNAL_JOURNAL: "{{ item.1.external_journal | bool }}" OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}" OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}" - HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + HOSTNAME: "{{ ceph_osd_hostname }}" OSD_STORETYPE: "{{ ceph_osd_store_type }}" OSD_BS_DEV: "{{ item.1.device | default('') }}" OSD_BS_LABEL: "{{ item.1.partition_label | default('') }}" diff --git a/ansible/roles/ceph/tasks/generate_cluster.yml b/ansible/roles/ceph/tasks/generate_cluster.yml index 121e36aaa4..c0f7a3dd24 100644 --- a/ansible/roles/ceph/tasks/generate_cluster.yml +++ b/ansible/roles/ceph/tasks/generate_cluster.yml @@ -8,8 +8,8 @@ environment: KOLLA_BOOTSTRAP: KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - MON_IP: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" - HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + MON_IP: "{{ storage_interface_address }}" + HOSTNAME: "{{ ceph_mon_hostname }}" image: "{{ ceph_mon_image_full }}" labels: BOOTSTRAP: diff --git a/ansible/roles/ceph/tasks/start_mdss.yml b/ansible/roles/ceph/tasks/start_mdss.yml index 8c3b7806ab..9e1a66ed9d 100644 --- a/ansible/roles/ceph/tasks/start_mdss.yml +++ b/ansible/roles/ceph/tasks/start_mdss.yml @@ -22,24 +22,19 @@ - name: Geting ceph mds keyring become: true kolla_ceph_keyring: - name: "mds.{{ hostvars[item]['inventory_hostname'] }}" + name: "mds.{{ ceph_mds_hostname }}" caps: "{{ ceph_client_mds_keyring_caps }}" register: ceph_mds_auth - run_once: true delegate_to: "{{ groups['ceph-mon'][0] }}" - with_items: "{{ groups['ceph-mds'] }}" - name: Pushing ceph mds keyring to ceph-mds become: true copy: content: | - [mds.{{ item.item }}] - key = {{ item.keyring.key }} - dest: "{{ node_config_directory }}/ceph-mds/ceph.mds.{{ inventory_hostname }}.keyring" + [mds.{{ ceph_mds_hostname }}] + key = {{ ceph_mds_auth.keyring.key }} + dest: "{{ node_config_directory }}/ceph-mds/ceph.mds.{{ ceph_mds_hostname }}.keyring" mode: "0600" - when: - - inventory_hostname == item.item - with_items: "{{ ceph_mds_auth.results }}" - name: Starting ceph-mds container become: true diff --git a/ansible/roles/ceph/tasks/start_mgrs.yml b/ansible/roles/ceph/tasks/start_mgrs.yml index 3bdddf100f..fe0b162de1 100644 --- a/ansible/roles/ceph/tasks/start_mgrs.yml +++ b/ansible/roles/ceph/tasks/start_mgrs.yml @@ -2,24 +2,19 @@ - name: Getting ceph mgr keyring become: true kolla_ceph_keyring: - name: "mgr.{{ item }}" + name: "mgr.{{ ceph_mgr_hostname }}" caps: "{{ ceph_client_mgr_keyring_caps }}" register: ceph_mgr_keyring - run_once: true delegate_to: "{{ groups['ceph-mon'][0] }}" - with_items: "{{ groups['ceph-mgr'] }}" - name: Pushing ceph mgr keyring to ceph-mgr become: true copy: content: | - [mgr.{{ item.item }}] - key = {{ item.keyring.key }} - dest: "{{ node_config_directory }}/ceph-mgr/ceph.mgr.{{ inventory_hostname }}.keyring" + [mgr.{{ ceph_mgr_hostname }}] + key = {{ ceph_mgr_keyring.keyring.key }} + dest: "{{ node_config_directory }}/ceph-mgr/ceph.mgr.{{ ceph_mgr_hostname }}.keyring" mode: "0600" - when: - - inventory_hostname == item.item - with_items: "{{ ceph_mgr_keyring.results }}" - name: Starting ceph-mgr container become: true diff --git a/ansible/roles/ceph/tasks/start_mons.yml b/ansible/roles/ceph/tasks/start_mons.yml index 4428e576ae..0cb4884364 100644 --- a/ansible/roles/ceph/tasks/start_mons.yml +++ b/ansible/roles/ceph/tasks/start_mons.yml @@ -6,7 +6,7 @@ common_options: "{{ docker_common_options }}" environment: KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" - HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}" + HOSTNAME: "{{ ceph_mon_hostname }}" image: "{{ ceph_mon_image_full }}" name: "ceph_mon" volumes: diff --git a/ansible/roles/ceph/templates/ceph-mds.json.j2 b/ansible/roles/ceph/templates/ceph-mds.json.j2 index dc7073c635..f6b2144deb 100644 --- a/ansible/roles/ceph/templates/ceph-mds.json.j2 +++ b/ansible/roles/ceph/templates/ceph-mds.json.j2 @@ -1,5 +1,5 @@ { - "command": "/usr/bin/ceph-mds -f -c /etc/ceph/ceph.conf -i {{ inventory_hostname }}", + "command": "/usr/bin/ceph-mds -f -c /etc/ceph/ceph.conf -i {{ ceph_mds_hostname }}", "config_files": [ { "source": "{{ container_config_directory }}/ceph.conf", @@ -8,8 +8,8 @@ "perm": "0600" }, { - "source": "{{ container_config_directory }}/ceph.mds.{{ inventory_hostname }}.keyring", - "dest": "/var/lib/ceph/mds/ceph-{{ inventory_hostname }}/keyring", + "source": "{{ container_config_directory }}/ceph.mds.{{ ceph_mds_hostname }}.keyring", + "dest": "/var/lib/ceph/mds/ceph-{{ ceph_mds_hostname }}/keyring", "owner": "root", "perm": "0644" } diff --git a/ansible/roles/ceph/templates/ceph-mgr.json.j2 b/ansible/roles/ceph/templates/ceph-mgr.json.j2 index 75c3885f85..48ce22b3b0 100644 --- a/ansible/roles/ceph/templates/ceph-mgr.json.j2 +++ b/ansible/roles/ceph/templates/ceph-mgr.json.j2 @@ -1,5 +1,5 @@ { - "command": "/usr/bin/ceph-mgr -f -i {{ inventory_hostname }}", + "command": "/usr/bin/ceph-mgr -f -i {{ ceph_mgr_hostname }}", "config_files": [ { "source": "{{ container_config_directory }}/ceph.conf", @@ -8,8 +8,8 @@ "perm": "0600" }, { - "source": "{{ container_config_directory }}/ceph.mgr.{{ inventory_hostname }}.keyring", - "dest": "/var/lib/ceph/mgr/ceph-{{ inventory_hostname }}/keyring", + "source": "{{ container_config_directory }}/ceph.mgr.{{ ceph_mgr_hostname }}.keyring", + "dest": "/var/lib/ceph/mgr/ceph-{{ ceph_mgr_hostname }}/keyring", "owner": "ceph", "perm": "0600" }, diff --git a/ansible/roles/ceph/templates/ceph-mon.json.j2 b/ansible/roles/ceph/templates/ceph-mon.json.j2 index 68e50cd523..b92e00395b 100644 --- a/ansible/roles/ceph/templates/ceph-mon.json.j2 +++ b/ansible/roles/ceph/templates/ceph-mon.json.j2 @@ -1,5 +1,5 @@ { - "command": "/usr/bin/ceph-mon -f {% if ceph_debug %}-d{% endif %} -i {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }} --public-addr {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}:6789", + "command": "/usr/bin/ceph-mon -f {% if ceph_debug %}-d{% endif %} -i {{ ceph_mon_hostname }} --public-addr {{ storage_interface_address }}:6789", "config_files": [ { "source": "{{ container_config_directory }}/ceph.conf", diff --git a/ansible/roles/ceph/templates/ceph-osd.json.j2 b/ansible/roles/ceph/templates/ceph-osd.json.j2 index fd4fc3e97f..fe6f754696 100644 --- a/ansible/roles/ceph/templates/ceph-osd.json.j2 +++ b/ansible/roles/ceph/templates/ceph-osd.json.j2 @@ -1,5 +1,5 @@ { - "command": "/usr/bin/ceph-osd -f {% if ceph_debug %}-d{% endif %} --public-addr {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }} --cluster-addr {{ hostvars[inventory_hostname]['ansible_' + cluster_interface]['ipv4']['address'] }}", + "command": "/usr/bin/ceph-osd -f {% if ceph_debug %}-d{% endif %} --public-addr {{ storage_interface_address }} --cluster-addr {{ hostvars[inventory_hostname]['ansible_' + cluster_interface]['ipv4']['address'] }}", "config_files": [ { "source": "{{ container_config_directory }}/ceph.conf", diff --git a/ansible/roles/ceph/templates/ceph.conf.j2 b/ansible/roles/ceph/templates/ceph.conf.j2 index db82800f8f..0f3423b929 100644 --- a/ansible/roles/ceph/templates/ceph.conf.j2 +++ b/ansible/roles/ceph/templates/ceph.conf.j2 @@ -6,7 +6,15 @@ log to stderr = false err to stderr = false fsid = {{ ceph_cluster_fsid }} +{% if ceph_mon_host_type == 'HOSTNAME' %} +mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_hostname'] }}{% if not loop.last %}, {% endif %}{% endfor %} +{% elif ceph_mon_host_type == 'FQDN' %} +mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_fqdn'] }}{% if not loop.last %}, {% endif %}{% endfor %} +{% elif ceph_mon_host_type == 'INVENTORY' %} +mon initial members = {% for host in groups['ceph-mon'] %}{{ host }}{% if not loop.last %}, {% endif %}{% endfor %} +{%- else %} mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + hostvars[host]['storage_interface']]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %} +{% endif %} mon host = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + hostvars[host]['storage_interface']]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %}