Merge "Add curator and dynamic shard counts"

elk_metrics_6x/installCurator.yml

@@ -0,0 +1,101 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install Curator
+  hosts: "elastic-logstash"
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  pre_tasks:
+    - name: Ensure curator is installed
+      pip:
+        name: elasticsearch-curator
+        state: "{{ elk_package_state | default('present') }}"
+
+    - name: exit playbook after uninstall
+      meta: end_play
+      when:
+        - elk_package_state | default('present') == 'absent'
+
+  tasks:
+    - name: create the system group
+      group:
+        name: "curator"
+        state: "present"
+        system: "yes"
+
+    - name: Create the curator system user
+      user:
+        name: "curator"
+        group: "curator"
+        comment: "curator user"
+        shell: "/bin/false"
+        createhome: "yes"
+        home: "/var/lib/curator"
+
+    - name: Create curator data path
+      file:
+        path: "{{ item }}"
+        state: directory
+        owner: "curator"
+        group: "curator"
+        mode: "0755"
+        recurse: true
+      with_items:
+        - "/var/lib/curator"
+        - "/var/log/curator"
+        - "/etc/curator"
+
+    - name: Drop curator conf file
+      template:
+        src: templates/curator.yml.j2
+        dest: /var/lib/curator/curator.yml
+
+    - name: Drop curator action file
+      template:
+        src: templates/curator-actions.yml.j2
+        dest: /var/lib/curator/actions.yml
+
+  post_tasks:
+    - name: Run the systemd service role
+      include_role:
+        name: systemd_service
+        private: true
+      vars:
+        systemd_service_enabled: true
+        systemd_user_name: curator
+        systemd_group_name: curator
+        systemd_services:
+          - service_name: "curator"
+            execstarts:
+              - /usr/local/bin/curator
+                --config /var/lib/curator/curator.yml
+                /var/lib/curator/actions.yml
+            timer:
+              state: "started"
+              options:
+                OnBootSec: 30min
+                OnUnitActiveSec: 48h
+                Persistent: true
+
+    - name: Enable and restart curator.timer
+      systemd:
+        name: "curator.timer"
+        enabled: true
+        state: restarted

elk_metrics_6x/installJournalbeat.yml

@@ -49,7 +49,7 @@
         state: "present"
         system: "yes"
 
-    - name: Create the nova system user
+    - name: Create the journalbeat user
       user:
         name: "journalbeat"
         group: "journalbeat"

elk_metrics_6x/templates/apm-server.yml.j2

@@ -282,7 +282,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/auditbeat.yml.j2

@@ -723,7 +723,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/curator-actions.yml.j2

@@ -0,0 +1,168 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete indices older than 60 days (based on index name), for logstash-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: logstash-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_logstash_retention }}
+  2:
+    action: delete_indices
+    description: >-
+      Delete indices older than 10 days (based on index name), for apm-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: apm-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_apm_retention }}
+  3:
+    action: delete_indices
+    description: >-
+      Delete indices older than 15 days (based on index name), for auditbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: auditbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_auditbeat_retention }}
+  4:
+    action: delete_indices
+    description: >-
+      Delete indices older than 15 days (based on index name), for filebeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: filebeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_filebeat_retention }}
+  5:
+    action: delete_indices
+    description: >-
+      Delete indices older than 10 days (based on index name), for heartbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: heartbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_heartbeat_retention }}
+  6:
+    action: delete_indices
+    description: >-
+      Delete indices older than 15 days (based on index name), for journalbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: journalbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_journalbeat_retention }}
+  7:
+    action: delete_indices
+    description: >-
+      Delete indices older than 10 days (based on index name), for metricbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: metricbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_metricbeat_retention }}
+  8:
+    action: delete_indices
+    description: >-
+      Delete indices older than 5 days (based on index name), for packetbeat-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: packetbeat-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ elastic_packetbeat_retention }}

elk_metrics_6x/templates/curator.yml.j2

@@ -0,0 +1,32 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+client:
+  hosts:
+    - {{ ansible_host }}
+  port: 9200
+  url_prefix: ""
+  use_ssl: false
+  ssl_no_validate: true
+  http_auth: ""
+  timeout: 120
+  master_only: true
+
+logging:
+  loglevel: INFO
+  logfile: /var/log/curator/curator
+  logformat: default
+  blacklist:
+    - elasticsearch
+    - urllib3

elk_metrics_6x/templates/filebeat.yml.j2

@@ -1771,7 +1771,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/heartbeat.yml.j2

@@ -832,7 +832,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/journalbeat.yml.j2

@@ -638,7 +638,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -1092,7 +1092,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/templates/packetbeat.yml.j2

@@ -1111,7 +1111,8 @@ setup.template.settings:
   # of the Elasticsearch template. For more details, please check
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
   index:
-    number_of_shards: 3
+    # 3 shards per elasticsearch host with a modifier of 1.5 rounded to the nearest whole number.
+    number_of_shards: {{ (((groups["elastic-logstash"] | length) * 3) * 1.5) // 1 }}
     codec: best_compression
     #number_of_routing_shards: 30
 

elk_metrics_6x/vars/variables.yml

@@ -4,6 +4,16 @@ elastic_hap_port: 9201
 cluster_name: openstack_elk
 node_name: ${HOSTNAME}
 
+# elastic curator vars
+# all retention options are in days
+elastic_logstash_retention: 28
+elastic_apm_retention: 14
+elastic_auditbeat_retention: 14
+elastic_filebeat_retention: 14
+elastic_heartbeat_retention: 7
+elastic_journalbeat_retention: 14
+elastic_metricbeat_retention: 14
+elastic_packetbeat_retention: 7
 
 # kibana vars
 kibana_interface: 0.0.0.0