Convert the curator action file into multiple files

The curator action plugin does not use a logical OR when parsing
multiple filters. The only way to do this is to run curator with
different action filter files.

Change-Id: I97c93c87d6254f79831f2a177098ea52a3a3a49d
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/roles/elastic_curator/tasks/curator_systemd.yml

@@ -27,10 +27,21 @@
         execstarts:
           - /opt/elasticsearch-curator/bin/curator
             --config /var/lib/curator/curator.yml
-            /var/lib/curator/actions.yml
+            /var/lib/curator/actions-age.yml
         timer:
           state: "started"
           options:
             OnBootSec: 30min
-            OnUnitActiveSec: 6h
+            OnUnitActiveSec: 24h
+            Persistent: true
+      - service_name: "curator"
+        execstarts:
+          - /opt/elasticsearch-curator/bin/curator
+            --config /var/lib/curator/curator.yml
+            /var/lib/curator/actions-size.yml
+        timer:
+          state: "started"
+          options:
+            OnBootSec: 30min
+            OnUnitActiveSec: 5h
             Persistent: true

elk_metrics_6x/roles/elastic_curator/tasks/curator_upstart.yml

@@ -13,11 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Create cron job for curator
+- name: Create cron job for curator (age)
   cron:
     name: "Run curator"
     minute: 0
-    hour: */6
+    hour: 1
     user: "curator"
-    job: "/opt/elasticsearch-curator/bin/curator --config /var/lib/curator/curator.yml /var/lib/curator/actions.yml"
+    job: "/opt/elasticsearch-curator/bin/curator --config /var/lib/curator/curator.yml /var/lib/curator/actions-age.yml"
+    cron_file: "elasticsearch-curator"
+
+- name: Create cron job for curator (size)
+  cron:
+    name: "Run curator"
+    minute: 0
+    hour: */5
+    user: "curator"
+    job: "/opt/elasticsearch-curator/bin/curator --config /var/lib/curator/curator.yml /var/lib/curator/actions-size.yml"
     cron_file: "elasticsearch-curator"

elk_metrics_6x/roles/elastic_curator/tasks/main.yml

@@ -91,8 +91,10 @@
   with_items:
     - src: "curator.yml.j2"
       dest: /var/lib/curator/curator.yml
-    - src: "curator-actions.yml.j2"
+    - src: "curator-actions-age.yml.j2"
-      dest: /var/lib/curator/actions.yml
+      dest: /var/lib/curator/actions-age.yml
+    - src: "curator-actions-size.yml.j2"
+      dest: /var/lib/curator/actions-size.yml
   notify:
     - Enable and restart curator.timer
 

elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2

@@ -20,13 +20,10 @@
 {# Total retention size in days #}
 {%   set _index_retention = hostvars[inventory_hostname]['elastic_' + key + '_retention'] -%}
 {%   set index_retention = ((_index_retention | int) > 0) | ternary(_index_retention, 1) | int %}
-{# Total retention size in gigabytes #}
-{%   set _index_size = ((hostvars[inventory_hostname]['elastic_' + key + '_size'] | int) // 1024) -%}
-{%   set index_size = ((_index_size | int) > 0) | ternary(_index_size, 1) | int %}
 {%   set _ = delete_indices.update(
    {
     'action': 'delete_indices',
-    'description': 'Prune indices for ' + key + ' after ' ~ index_retention ~ ' days or index is > ' ~ index_size ~ 'gb',
+    'description': 'Prune indices for ' + key + ' after ' ~ index_retention ~ ' days',
     'options': {
       'ignore_empty_list': true,
       'disable_action': false
@@ -43,15 +40,6 @@
    }
   )
 -%}
-{%   set _ = filters.append(
-   {
-    'filtertype': 'space',
-    'disk_space': index_size,
-    'use_age': true,
-    'source': 'creation_date'
-   }
-  )
--%}
 {%   set _ = filters.append(
    {
     'filtertype': 'age',

elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2

@@ -0,0 +1,63 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{% set action_items = [] -%}
+{# Delete index loop #}
+{% for key in elastic_beat_retention_policy_keys -%}
+{%   set delete_indices = {} -%}
+{# Total retention size in gigabytes #}
+{%   set _index_size = ((hostvars[inventory_hostname]['elastic_' + key + '_size'] | int) // 1024) -%}
+{%   set index_size = ((_index_size | int) > 0) | ternary(_index_size, 1) | int %}
+{%   set _ = delete_indices.update(
+   {
+    'action': 'delete_indices',
+    'description': 'Prune indices for ' + key + ' after index is > ' ~ index_size ~ 'gb',
+    'options': {
+      'ignore_empty_list': true,
+      'disable_action': false
+    }
+   }
+  )
+-%}
+{%   set filters = [] -%}
+{%   set _ = filters.append(
+   {
+    'filtertype': 'pattern',
+    'kind': 'prefix',
+    'value': key + '-'
+   }
+  )
+-%}
+{%   set _ = filters.append(
+   {
+    'filtertype': 'space',
+    'disk_space': index_size,
+    'use_age': true,
+    'source': 'creation_date'
+   }
+  )
+-%}
+{%   set _ = delete_indices.update({'filters': filters}) -%}
+{%   set _ = action_items.append(delete_indices) -%}
+{% endfor -%}
+
+{% set actions = {} -%}
+{% for action_item in action_items -%}
+{%   set _ = actions.update({loop.index: action_item}) -%}
+{% endfor -%}
+
+{# Render all actions #}
+{% set curator_actions = {'actions': actions} -%}
+{{ curator_actions | to_nice_yaml(indent=2) }}