b6a9a6fc7a
The curator retention policies will now query the storage nodes within a given deployment and set a suitable index retention policy based on the total amount of storage each index is assumed to produce every day. To ensure we're minimizing the storage required and optimizing search performance several actions are now being taken: * Indexes will be shrunk after a quarter of their retention time. * Indexes will be deleted should they exceed the retention time. Change-Id: I8bf548620b5404d25deaadba8fda93452ef64fa0 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
125 lines
3.4 KiB
Django/Jinja
125 lines
3.4 KiB
Django/Jinja
---
|
|
# Copyright 2018, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
{% set action_items = [] -%}
|
|
{# Delete index loop #}
|
|
{% for key in elastic_beat_retention_policy_hosts.keys() -%}
|
|
{% set delete_indices = {} -%}
|
|
{% set index_retention = hostvars[inventory_hostname]['elastic_' + key + '_retention'] -%}
|
|
{% set _ = delete_indices.update(
|
|
{
|
|
'action': 'delete_indices',
|
|
'description': 'Prune indices for ' + key + ' after ' ~ ((index_retention | int) * 2) ~ ' days.',
|
|
'options': {
|
|
'ignore_empty_list': true,
|
|
'disable_action': false
|
|
}
|
|
}
|
|
)
|
|
-%}
|
|
{# add the filter loop #}
|
|
{% set filters = [] -%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'pattern',
|
|
'kind': 'prefix',
|
|
'value': key + '-'
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'age',
|
|
'source': 'name',
|
|
'direction': 'older',
|
|
'timestring': '%Y.%m.%d',
|
|
'unit': 'days',
|
|
'unit_count': (index_retention | int)
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = delete_indices.update({'filters': filters}) -%}
|
|
{% set _ = action_items.append(delete_indices) -%}
|
|
|
|
{# Set shrink curator options #}
|
|
{% set shrink_indices = {} -%}
|
|
{% set _ = shrink_indices.update(
|
|
{
|
|
'action': 'shrink',
|
|
'description': 'Shrink ' + key + ' indices older than ' ~ (index_retention | int) // 4 ~ ' days',
|
|
'options': {
|
|
"disable_action": false,
|
|
"ignore_empty_list": true,
|
|
"shrink_node": "DETERMINISTIC",
|
|
"node_filters": {
|
|
"permit_masters": ((master_nodes | length) < (data_nodes | length)) | ternary(true, false),
|
|
"exclude_nodes": (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list)
|
|
},
|
|
"number_of_shards": 1,
|
|
"number_of_replicas": 1,
|
|
"shrink_suffix": '-shrink',
|
|
"copy_aliases": true,
|
|
"delete_after": true,
|
|
"post_allocation": {
|
|
"allocation_type": "include",
|
|
"key": "node_tag",
|
|
"value": "cold"
|
|
},
|
|
"wait_for_active_shards": 1,
|
|
"extra_settings": {
|
|
"settings": {
|
|
"index.codec": "best_compression"
|
|
}
|
|
},
|
|
"wait_for_completion": true,
|
|
"wait_for_rebalance": true,
|
|
"wait_interval": 9,
|
|
"max_wait": -1
|
|
}
|
|
}
|
|
)
|
|
-%}
|
|
{% set filters = [] -%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'pattern',
|
|
'kind': 'prefix',
|
|
'value': key + '-'
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = filters.append(
|
|
{
|
|
'filtertype': 'age',
|
|
'source': 'creation_date',
|
|
'direction': 'older',
|
|
'unit': 'days',
|
|
'unit_count': (index_retention | int) // 4
|
|
}
|
|
)
|
|
-%}
|
|
{% set _ = shrink_indices.update({'filters': filters}) -%}
|
|
{% set _ = action_items.append(shrink_indices) -%}
|
|
{% endfor -%}
|
|
|
|
{% set actions = {} -%}
|
|
{% for action_item in action_items -%}
|
|
{% set _ = actions.update({loop.index: action_item}) -%}
|
|
{% endfor -%}
|
|
|
|
{# Render all actions #}
|
|
{% set curator_actions = {'actions': actions} -%}
|
|
{{ curator_actions | to_nice_yaml(indent=2) }}
|