From 3397668d10aac69e016e7c6bd72d16c625e1e0d1 Mon Sep 17 00:00:00 2001
From: Jeffrey Zhang <zhang.lei.fly@gmail.com>
Date: Fri, 30 Mar 2018 10:49:12 +0800
Subject: [PATCH] Migrate ceph keyring creation to kolla_ceph_keyring module

In this way, keyring caps is updatable.

Change-Id: Idf7f222645b5073e2c72d59eecf3d47b3f1dc6ba
---
 ansible/roles/ceph/defaults/main.yml    | 10 ++++++++++
 ansible/roles/ceph/tasks/start_mdss.yml |  9 ++++++---
 ansible/roles/ceph/tasks/start_mgrs.yml |  9 ++++++---
 ansible/roles/cinder/defaults/main.yml  | 18 ++++++++++++++++++
 ansible/roles/cinder/tasks/ceph.yml     | 22 ++++++++++++----------
 ansible/roles/glance/defaults/main.yml  |  7 +++++++
 ansible/roles/glance/tasks/ceph.yml     | 10 ++++++----
 ansible/roles/gnocchi/defaults/main.yml |  7 +++++++
 ansible/roles/gnocchi/tasks/ceph.yml    | 10 ++++++----
 ansible/roles/manila/defaults/main.yml  | 15 +++++++++++++++
 ansible/roles/manila/tasks/ceph.yml     |  9 ++++++---
 ansible/roles/nova/defaults/main.yml    | 11 +++++++++++
 ansible/roles/nova/tasks/ceph.yml       | 22 ++++++++--------------
 13 files changed, 118 insertions(+), 41 deletions(-)

diff --git a/ansible/roles/ceph/defaults/main.yml b/ansible/roles/ceph/defaults/main.yml
index 6c312da401..950a8a7b18 100644
--- a/ansible/roles/ceph/defaults/main.yml
+++ b/ansible/roles/ceph/defaults/main.yml
@@ -46,6 +46,16 @@ ceph_client_admin_keyring_caps:
   osd: "allow *"
   mgr: "allow *"
 
+ceph_client_mgr_keyring_caps:
+  mon: 'allow profile mgr'
+  osd: 'allow *'
+  mds: 'allow *'
+
+ceph_client_mds_keyring_caps:
+  mds: 'allow '
+  osd: 'allow *'
+  mon: 'allow rwx'
+
 partition_name_osd_bootstrap: "{{ 'KOLLA_CEPH_OSD_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_BOOTSTRAP' }}"
 partition_name_cache_bootstrap: "{{ 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP_BS' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' }}"
 partition_name_osd_data: "{{ 'KOLLA_CEPH_BSDATA' if ceph_osd_store_type == 'bluestore' else 'KOLLA_CEPH_DATA' }}"
diff --git a/ansible/roles/ceph/tasks/start_mdss.yml b/ansible/roles/ceph/tasks/start_mdss.yml
index f09481130d..1d03ffc38b 100644
--- a/ansible/roles/ceph/tasks/start_mdss.yml
+++ b/ansible/roles/ceph/tasks/start_mdss.yml
@@ -20,17 +20,20 @@
       pool_pgp_num: "{{ cephfs_metadata_pool_pgp_num }}"
 
 - name: Geting ceph mds keyring
-  command: docker exec ceph_mon ceph auth get-or-create mds.{{ hostvars[item]['inventory_hostname'] }} mds 'allow ' osd 'allow *' mon 'allow rwx'
+  kolla_ceph_keyring:
+    name: "mds.{{ hostvars[item]['inventory_hostname'] }}"
+    caps: "{{ ceph_client_mds_keyring_caps }}"
   register: ceph_mds_auth
   run_once: true
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: false
   with_items: "{{ groups['ceph-mds'] }}"
 
 - name: Pushing ceph mds keyring to ceph-mds
   become: true
   copy:
-    content: "{{ item.stdout }}\n"
+    content: |
+      [mds.{{ item.item }}]
+          key = {{ item.keyring.key }}
     dest: "{{ node_config_directory }}/ceph-mds/ceph.mds.{{ inventory_hostname }}.keyring"
     mode: 0600
   when:
diff --git a/ansible/roles/ceph/tasks/start_mgrs.yml b/ansible/roles/ceph/tasks/start_mgrs.yml
index 1e566f04fc..d4f1af1ac6 100644
--- a/ansible/roles/ceph/tasks/start_mgrs.yml
+++ b/ansible/roles/ceph/tasks/start_mgrs.yml
@@ -1,16 +1,19 @@
 ---
 - name: Getting ceph mgr keyring
-  command: docker exec ceph_mon ceph auth get-or-create mgr.{{ item }} mon 'allow profile mgr' osd 'allow *' mds 'allow *'
+  kolla_ceph_keyring:
+    name: "mgr.{{ item }}"
+    caps: "{{ ceph_client_mgr_keyring_caps }}"
   register: ceph_mgr_keyring
   run_once: true
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: false
   with_items: "{{ groups['ceph-mgr'] }}"
 
 - name: Pushing ceph mgr keyring to ceph-mgr
   become: true
   copy:
-    content: "{{ item.stdout }}\n"
+    content: |
+      [mgr.{{ item.item }}]
+          key = {{ item.keyring.key }}
     dest: "{{ node_config_directory }}/ceph-mgr/ceph.mgr.{{ inventory_hostname }}.keyring"
     mode: 0600
   when:
diff --git a/ansible/roles/cinder/defaults/main.yml b/ansible/roles/cinder/defaults/main.yml
index ce3bb926db..9725988db2 100644
--- a/ansible/roles/cinder/defaults/main.yml
+++ b/ansible/roles/cinder/defaults/main.yml
@@ -77,6 +77,24 @@ cinder_backup_cache_mode: "{{ ceph_cinder_backup_cache_mode }}"
 cinder_backup_pool_pg_num: "{{ ceph_pool_pg_num }}"
 cinder_backup_pool_pgp_num: "{{ ceph_pool_pgp_num }}"
 
+ceph_client_cinder_keyring_caps:
+  mon: 'allow r'
+  osd: >-
+    allow class-read object_prefix rbd_children,
+    allow rwx pool={{ ceph_cinder_pool_name }},
+    allow rwx pool={{ ceph_cinder_pool_name }}-cache,
+    allow rwx pool={{ ceph_nova_pool_name }},
+    allow rwx pool={{ ceph_nova_pool_name }}-cache,
+    allow rx pool={{ ceph_glance_pool_name }},
+    allow rx pool={{ ceph_glance_pool_name }}-cache
+
+ceph_client_cinder_backup_keyring_caps:
+  mon: 'allow r'
+  osd: >-
+    allow class-read object_prefix rbd_children,
+    allow rwx pool={{ ceph_cinder_backup_pool_name }},
+    allow rwx pool={{ ceph_cinder_backup_pool_name }}-cache
+
 
 ####################
 # Database
diff --git a/ansible/roles/cinder/tasks/ceph.yml b/ansible/roles/cinder/tasks/ceph.yml
index 3303786d3c..71599a62b6 100644
--- a/ansible/roles/cinder/tasks/ceph.yml
+++ b/ansible/roles/cinder/tasks/ceph.yml
@@ -54,32 +54,34 @@
     pool_pgp_num: "{{ cinder_backup_pool_pgp_num }}"
     pool_application: "rbd"
 
-# TODO(SamYaple): Improve changed_when tests
 - name: Pulling cephx keyring for cinder
-  command: docker exec ceph_mon ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ ceph_cinder_pool_name }}, allow rwx pool={{ ceph_cinder_pool_name }}-cache, allow rwx pool={{ ceph_nova_pool_name }}, allow rwx pool={{ ceph_nova_pool_name }}-cache, allow rx pool={{ ceph_glance_pool_name }}, allow rx pool={{ ceph_glance_pool_name }}-cache'
+  kolla_ceph_keyring:
+    name: client.cinder
+    caps: "{{ ceph_client_cinder_keyring_caps }}"
   register: cephx_key_cinder
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: False
   run_once: True
 
-# TODO(SamYaple): Improve changed_when tests
 - name: Pulling cephx keyring for cinder-backup
-  command: docker exec ceph_mon ceph auth get-or-create client.cinder-backup mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ ceph_cinder_backup_pool_name }}, allow rwx pool={{ ceph_cinder_backup_pool_name }}-cache'
+  kolla_ceph_keyring:
+    name: client.cinder-backup
+    caps: "{{ ceph_client_cinder_backup_keyring_caps }}"
   register: cephx_key_cinder_backup
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: False
   run_once: True
 
 - name: Pushing cephx keyring
   copy:
-    content: "{{ item.content }}\n\r"
+    content: |
+      [client.{{ item.key_name }}]
+          key = {{ item.key }}
     dest: "{{ node_config_directory }}/{{ item.service_name }}/ceph.client.{{ item.key_name }}.keyring"
     mode: "0600"
   become: true
   with_items:
-    - { service_name: "cinder-volume", key_name: "cinder", content: "{{ cephx_key_cinder.stdout }}" }
-    - { service_name: "cinder-backup", key_name: "cinder", content: "{{ cephx_key_cinder.stdout }}" }
-    - { service_name: "cinder-backup", key_name: "cinder-backup", content: "{{ cephx_key_cinder_backup.stdout }}" }
+    - { service_name: "cinder-volume", key_name: "cinder", key: "{{ cephx_key_cinder.keyring.key }}" }
+    - { service_name: "cinder-backup", key_name: "cinder", key: "{{ cephx_key_cinder.keyring.key }}" }
+    - { service_name: "cinder-backup", key_name: "cinder-backup", key: "{{ cephx_key_cinder_backup.keyring.key }}" }
   when:
     - inventory_hostname in groups[item.service_name]
     - cinder_services[item.service_name].enabled | bool
diff --git a/ansible/roles/glance/defaults/main.yml b/ansible/roles/glance/defaults/main.yml
index 662c41eff6..3559d77ab6 100644
--- a/ansible/roles/glance/defaults/main.yml
+++ b/ansible/roles/glance/defaults/main.yml
@@ -47,6 +47,13 @@ glance_cache_mode: "{{ ceph_glance_cache_mode }}"
 glance_pool_pg_num: "{{ ceph_pool_pg_num }}"
 glance_pool_pgp_num: "{{ ceph_pool_pgp_num }}"
 
+ceph_client_glance_keyring_caps:
+  mon: 'allow r'
+  osd: >-
+    allow class-read object_prefix rbd_children,
+    allow rwx pool={{ ceph_glance_pool_name }},
+    allow rwx pool={{ ceph_glance_pool_name }}-cache
+
 
 ####################
 # Database
diff --git a/ansible/roles/glance/tasks/ceph.yml b/ansible/roles/glance/tasks/ceph.yml
index b8c246e265..0055f19425 100644
--- a/ansible/roles/glance/tasks/ceph.yml
+++ b/ansible/roles/glance/tasks/ceph.yml
@@ -25,17 +25,19 @@
     pool_pgp_num: "{{ glance_pool_pgp_num }}"
     pool_application: "rbd"
 
-# TODO(SamYaple): Improve changed_when tests
 - name: Pulling cephx keyring
-  command: docker exec ceph_mon ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ ceph_glance_pool_name }}, allow rwx pool={{ ceph_glance_pool_name }}-cache'
+  kolla_ceph_keyring:
+    name: client.glance
+    caps: "{{ ceph_client_glance_keyring_caps }}"
   register: cephx_key
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: False
   run_once: True
 
 - name: Pushing cephx keyring
   copy:
-    content: "{{ cephx_key.stdout }}\n\r"
+    content: |
+      [client.glance]
+          key = {{ cephx_key.keyring.key }}
     dest: "{{ node_config_directory }}/glance-api/ceph.client.glance.keyring"
     mode: "0600"
   when: inventory_hostname in groups['glance-api']
diff --git a/ansible/roles/gnocchi/defaults/main.yml b/ansible/roles/gnocchi/defaults/main.yml
index 613b91881a..a902032ee9 100644
--- a/ansible/roles/gnocchi/defaults/main.yml
+++ b/ansible/roles/gnocchi/defaults/main.yml
@@ -48,6 +48,13 @@ gnocchi_cache_mode: "{{ ceph_gnocchi_cache_mode }}"
 gnocchi_pool_pg_num: "{{ ceph_pool_pg_num }}"
 gnocchi_pool_pgp_num: "{{ ceph_pool_pgp_num }}"
 
+ceph_client_gnocchi_keyring_caps:
+  mon: 'allow r'
+  osd: >-
+    allow class-read object_prefix rbd_children,
+    allow rwx pool={{ ceph_gnocchi_pool_name }},
+    allow rwx pool={{ ceph_gnocchi_pool_name }}-cache
+
 
 ####################
 # Database
diff --git a/ansible/roles/gnocchi/tasks/ceph.yml b/ansible/roles/gnocchi/tasks/ceph.yml
index 60fa152cf3..e39537b3ae 100644
--- a/ansible/roles/gnocchi/tasks/ceph.yml
+++ b/ansible/roles/gnocchi/tasks/ceph.yml
@@ -31,17 +31,19 @@
     pool_pgp_num: "{{ gnocchi_pool_pgp_num }}"
     pool_application: "rgw"
 
-# TODO(SamYaple): Improve changed_when tests
 - name: Pulling cephx keyring
-  command: docker exec ceph_mon ceph auth get-or-create client.gnocchi mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ ceph_gnocchi_pool_name }}, allow rwx pool={{ ceph_gnocchi_pool_name }}-cache'
+  kolla_ceph_keyring:
+    name: client.gnocchi
+    caps: "{{ ceph_client_gnocchi_keyring_caps }}"
   register: cephx_key
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: False
   run_once: True
 
 - name: Pushing cephx keyring
   copy:
-    content: "{{ cephx_key.stdout }}\n\r"
+    content: |
+      [client.gnocchi]
+          key = {{ cephx_key.keyring.key }}
     dest: "{{ node_config_directory }}/{{ item }}/ceph.client.gnocchi.keyring"
     mode: "0600"
   when: inventory_hostname in groups[item]
diff --git a/ansible/roles/manila/defaults/main.yml b/ansible/roles/manila/defaults/main.yml
index 0d4a5d876e..c65f3b7aec 100644
--- a/ansible/roles/manila/defaults/main.yml
+++ b/ansible/roles/manila/defaults/main.yml
@@ -44,6 +44,21 @@ manila_services:
       - "kolla_logs:/var/log/kolla/"
 
 
+#####################
+## Ceph
+#####################
+
+ceph_client_manila_keyring_caps:
+  mon: >-
+    allow r,
+    allow command "auth del",
+    allow command "auth caps",
+    allow command "auth get",
+    allow command "auth get-or-create"
+  osd: 'allow rw'
+  mds: 'allow *'
+
+
 #####################
 ## Database
 #####################
diff --git a/ansible/roles/manila/tasks/ceph.yml b/ansible/roles/manila/tasks/ceph.yml
index 497dff917e..e0a3fe51f1 100644
--- a/ansible/roles/manila/tasks/ceph.yml
+++ b/ansible/roles/manila/tasks/ceph.yml
@@ -15,15 +15,18 @@
   become: true
 
 - name: Pulling cephx keyring for manila
-  command: docker exec ceph_mon ceph auth get-or-create client.manila mon 'allow r, allow command "auth del", allow command "auth caps", allow command "auth get", allow command "auth get-or-create"' osd 'allow rw' mds 'allow *'
+  kolla_ceph_keyring:
+    name: client.manila
+    caps: "{{ ceph_client_manila_keyring_caps }}"
   register: cephx_key_manila
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: False
   run_once: True
 
 - name: Pushing cephx keyring
   copy:
-    content: "{{ cephx_key_manila.stdout }}\n\r"
+    content: |
+      [client.manila]
+          key = {{ cephx_key_manila.keyring.key }}
     dest: "{{ node_config_directory }}/manila-share/ceph.client.manila.keyring"
     mode: "0600"
   become: true
diff --git a/ansible/roles/nova/defaults/main.yml b/ansible/roles/nova/defaults/main.yml
index 4bdd3fe0d9..8a22cbffdf 100644
--- a/ansible/roles/nova/defaults/main.yml
+++ b/ansible/roles/nova/defaults/main.yml
@@ -154,6 +154,17 @@ nova_pool_pgp_num: "{{ ceph_pool_pgp_num }}"
 # qemu (1, 6, 0) or later. Set to "" to disable.
 nova_hw_disk_discard: "unmap"
 
+ceph_client_nova_keyring_caps:
+  mon: 'allow r'
+  osd: >-
+    allow class-read object_prefix rbd_children,
+    allow rwx pool={{ ceph_cinder_pool_name }},
+    allow rwx pool={{ ceph_cinder_pool_name }}-cache,
+    allow rwx pool={{ ceph_nova_pool_name }},
+    allow rwx pool={{ ceph_nova_pool_name }}-cache,
+    allow rwx pool={{ ceph_glance_pool_name }},
+    allow rwx pool={{ ceph_glance_pool_name }}-cache
+
 
 ####################
 # Database
diff --git a/ansible/roles/nova/tasks/ceph.yml b/ansible/roles/nova/tasks/ceph.yml
index 1500f5fbde..625bc0c288 100644
--- a/ansible/roles/nova/tasks/ceph.yml
+++ b/ansible/roles/nova/tasks/ceph.yml
@@ -33,20 +33,12 @@
     pool_pgp_num: "{{ nova_pool_pgp_num }}"
     pool_application: "rbd"
 
-# TODO(SamYaple): Improve changed_when tests
 - name: Pulling cephx keyring for nova
-  command: docker exec ceph_mon ceph auth get-or-create client.nova mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ ceph_cinder_pool_name }}, allow rwx pool={{ ceph_cinder_pool_name }}-cache, allow rwx pool={{ ceph_nova_pool_name }}, allow rwx pool={{ ceph_nova_pool_name }}-cache, allow rwx pool={{ ceph_glance_pool_name }}, allow rwx pool={{ ceph_glance_pool_name }}-cache'
-  register: cephx_key
+  kolla_ceph_keyring:
+    name: client.nova
+    caps: "{{ ceph_client_nova_keyring_caps }}"
+  register: nova_cephx_key
   delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: False
-  run_once: True
-
-# TODO(SamYaple): Improve failed_when and changed_when tests
-- name: Pulling nova cephx keyring for libvirt
-  command: docker exec ceph_mon ceph auth get-key client.nova
-  register: nova_cephx_raw_key
-  delegate_to: "{{ groups['ceph-mon'][0] }}"
-  changed_when: False
   run_once: True
 
 - name: Pulling cinder cephx keyring for libvirt
@@ -61,7 +53,9 @@
 
 - name: Pushing cephx keyring for nova
   copy:
-    content: "{{ cephx_key.stdout }}\n\r"
+    content: |
+      [client.nova]
+          key = {{ nova_cephx_key.keyring.key }}
     dest: "{{ node_config_directory }}/nova-compute/ceph.client.nova.keyring"
     mode: "0600"
   when: inventory_hostname in groups['compute']
@@ -92,7 +86,7 @@
     - item.enabled | bool
   with_items:
     - uuid: "{{ rbd_secret_uuid }}"
-      content: "{{ nova_cephx_raw_key.stdout }}"
+      content: "{{ nova_cephx_key.keyring.key }}"
       enabled: true
     - uuid: "{{ cinder_rbd_secret_uuid }}"
       content: "{{ cinder_cephx_raw_key.stdout|default('') }}"