Improve metricbeat Ceph detection

When Ceph is deployed by OSA ceph_mons is a list of addresses
of the mons on the storage network populated from the OSA inventory.

When Ceph is deployed outside of OSA, ceph_mons is provided by
the deployer as a pointer to the mon addresses for integration.

To cover both these cases, the only OSA hosts in the control plane
which are guaranteed to be able to access the Ceph mons and the ELK
stack are those in the cinder_volumes host group.

This change configures metricbeat on the cinder_volumes hosts to
collect data from the ceph REST API on the mon hosts when possible.
Otherwise, collection is attempted from all hosts which have a
/etc/ceph directory.

Change-Id: Ib90f4c54636371b38d37e588ef6f7ba259fbe1b3

elk_metrics_6x/installMetricbeat.yml

@@ -45,6 +45,21 @@
         path: /etc/ceph
       register: ceph
 
+    # gather ceph stats from localhost
+    # except when a list of mons is provided
+    - name: Set ceph stats hosts
+      set_fact:
+        ceph_stats_hosts: |-
+          {% set ceph_stats = [] %}
+          {% if (ceph_mons is defined) and (ceph_mons | length > 0) %}
+          {%   for mon in ceph_mons %}
+          {%     set _ = ceph_stats.insert(loop.index, (mon + ":5000")) %}
+          {%   endfor %}
+          {% else %}
+          {%   set ceph_stats = [ ansible_hostname + ":5000" ] %}
+          {% endif %}
+          {{ ceph_stats }}
+
     - name: Check for etcd
       stat:
         path: /etc/etcd
@@ -105,7 +120,18 @@
     - name: Set discovery facts
       set_fact:
         apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}"
-        ceph_enabled: "{{ ceph.stat.exists | bool }}"
+
+        # enable ceph on:  cinder volume hosts when we have a list of ceph mons
+        #      otherwise:  all hosts which have /etc/ceph
+        ceph_enabled: |-
+          {% set ceph_detect = false %}
+          {% if (ceph_mons is defined) and (ceph_mons | length > 0) and (inventory_hostname in groups['cinder_volume'])  %}
+          {%   set ceph_detect = true %}
+          {% else %}
+          {%   set ceph_detect = ceph.stats.exists | bool %}
+          {% endif %}
+          {{ ceph_detect }}
+
         docker_enabled: "{{ docker.stat.exists | bool }}"
         etcd_enabled: "{{ etcd.stat.exists | bool }}"
         haproxy_enabled: "{{ haproxy.stat.exists | bool }}"

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -145,7 +145,7 @@ metricbeat.modules:
   metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"]
   enabled: true
   period: 30s
-  hosts: ["localhost:5000"]
+  hosts: {{ ceph_stats_hosts | to_json }}
 #
 {% endif %}
 ##------------------------------ Couchbase Module -----------------------------