Merge "ceph: fixes to deployment and upgrade"

ansible/library/kolla_ceph_keyring.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import json
+import re
 import subprocess  # nosec
 
 
@@ -51,13 +52,16 @@ EXAMPLES = '''
     name: client.admin
     container_name: ceph_mon
     caps:
-      mds: 'allow'
+      mds: 'allow *'
       mon: 'allow *'
       osd: 'allow *'
       mgr: 'allow *'
 '''
 
 
+enoent_re = re.compile(r"\bENOENT\b")
+
+
 class CephKeyring(object):
     def __init__(self, name, caps, container_name='ceph_mon'):
         self.name = name
@@ -93,7 +97,10 @@ class CephKeyring(object):
     def ensure_keyring(self):
         try:
             stdout = self.get_keyring()
-        except subprocess.CalledProcessError:
+        except subprocess.CalledProcessError as e:
+            if e.returncode != 2 or not enoent_re.search(e.output):
+                # this is not a missing keyring case
+                raise
             # keyring doesn't exsit, try to create it
             stdout = self.create_keyring()
             self.changed = True

ansible/roles/ceph/tasks/deploy.yml

@@ -1,4 +1,6 @@
 ---
+# NOTE(yoctozepto): this file is used during upgrade as well
+
 - include_tasks: config.yml
 
 - include_tasks: bootstrap_mons.yml
@@ -9,19 +11,8 @@
 - include_tasks: start_mons.yml
   when: inventory_hostname in groups['ceph-mon']
 
-- include_tasks: start_mgrs.yml
+# NOTE(yoctozepto): this ensures caps for admin are always up-to-date (run as earliest as possible = after MONs start)
-  when: inventory_hostname in groups['ceph-mgr']
+# this is retried because the cluster might not be fully operational yet (quorum gathering)
-
-- include_tasks: start_ceph_dashboard.yml
-  when:
-    - enable_ceph_dashboard | bool
-    - inventory_hostname in groups['ceph-mon']
-
-- include_tasks: start_nfss.yml
-  when:
-    - enable_ceph_nfs | bool
-    - inventory_hostname in groups['ceph-nfs']
-
 - name: configuring client.admin caps
   become: true
   kolla_ceph_keyring:
@@ -29,6 +20,18 @@
     caps: "{{ ceph_client_admin_keyring_caps }}"
   run_once: True
   delegate_to: "{{ groups['ceph-mon'][0] }}"
+  register: result
+  until: result is success
+  retries: 3
+  delay: 15
+
+- include_tasks: start_mgrs.yml
+  when: inventory_hostname in groups['ceph-mgr']
+
+- include_tasks: start_ceph_dashboard.yml
+  when:
+    - enable_ceph_dashboard | bool
+    - inventory_hostname in groups['ceph-mon']
 
 - include_tasks: bootstrap_osds.yml
   when: inventory_hostname in groups['ceph-osd']
@@ -50,3 +53,9 @@
   when:
     - enable_ceph_mds | bool
     - inventory_hostname in groups['ceph-mds']
+
+# NOTE(yoctozepto): nfs (cephfs-based) depends on mds so start it after
+- include_tasks: start_nfss.yml
+  when:
+    - enable_ceph_nfs | bool
+    - inventory_hostname in groups['ceph-nfs']

ansible/roles/ceph/tasks/start_nfss.yml

@@ -9,5 +9,4 @@
     volumes:
       - "{{ node_config_directory }}/ceph-nfs/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
-      - "/var/run/dbus:/var/run/dbus"
       - "kolla_logs:/var/log/kolla/"

ansible/roles/ceph/tasks/upgrade.yml

@@ -1,39 +1,5 @@
 ---
-- include_tasks: config.yml
+- include_tasks: deploy.yml
-
-# NOTE(jeffrey4l): client.admin caps should be update when upgrade from Jewel
-# to Luminous
-- name: configuring client.admin caps
-  become: true
-  kolla_ceph_keyring:
-    name: client.admin
-    caps: "{{ ceph_client_admin_keyring_caps }}"
-  run_once: True
-  delegate_to: "{{ groups['ceph-mon'][0] }}"
-
-- include_tasks: start_mons.yml
-  when: inventory_hostname in groups['ceph-mon']
-
-- include_tasks: start_mgrs.yml
-  when: inventory_hostname in groups['ceph-mgr']
-
-- include_tasks: start_osds.yml
-  when: inventory_hostname in groups['ceph-osd']
-
-- include_tasks: start_rgws.yml
-  when:
-    - enable_ceph_rgw | bool
-    - inventory_hostname in groups['ceph-rgw']
-
-- include_tasks: start_mdss.yml
-  when:
-    - enable_ceph_mds | bool
-    - inventory_hostname in groups['ceph-mds']
-
-- include_tasks: start_nfss.yml
-  when:
-    - enable_ceph_nfs | bool
-    - inventory_hostname in groups['ceph-nfs']
 
 - name: Check final release (as running on MONs)
   become: true

ansible/roles/ceph/templates/ganesha.conf.j2

@@ -1,3 +1,14 @@
+NFS_CORE_PARAM {
+    Protocols = 4;
+    Enable_NLM = false;
+    Enable_RQUOTA = false;
+    Enable_UDP = false;
+}
+
+NFS_KRB5 {
+    Active_krb5 = false;
+}
+
 EXPORT
 {
 	Export_id=1;

tests/get_logs.sh

@@ -43,9 +43,11 @@ copy_logs() {
 
     # ceph related logs
     if [[ $(docker ps --filter name=ceph_mon --format "{{.Names}}") ]]; then
-        docker exec ceph_mon ceph -s > ${LOG_DIR}/kolla/ceph/ceph_s.txt
+        docker exec ceph_mon ceph --connect-timeout 5 -s > ${LOG_DIR}/kolla/ceph/ceph_s.txt
-        docker exec ceph_mon ceph osd df > ${LOG_DIR}/kolla/ceph/ceph_osd_df.txt
+        # NOTE(yoctozepto): osd df removed on purpose to avoid CI POST_FAILURE due to a possible hang:
-        docker exec ceph_mon ceph osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt
+        # as of ceph mimic it hangs when MON is operational but MGR not
+        # its usefulness is mediocre and having POST_FAILUREs is bad
+        docker exec ceph_mon ceph --connect-timeout 5 osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt
     fi
 
     # bifrost related logs