b6343c57a4
The logstash groks were running in line using the legacy method which uses lexical sorting of all logstash filter files and loads them in order. While this works it makes it so all data has to travel through all filters. This change makes use of the logstash multi-pipeline capabilities using a distributor and fork pattern. This allows data to flow through logstash more quickly and not block whenever there's an issue with an output plugin. Finger-prints using SHA1 when there's a message and UUID when not. This will ensure we're duplicating log entries which will help speed up transations and further reduce the storage required. Change-Id: I38268e33b370da0f1e186ecf65911d4a312c3e6a Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
44 lines
2.0 KiB
Django/Jinja
44 lines
2.0 KiB
Django/Jinja
# A template is used to set the mapping in Elasticsearch
|
|
# By default template loading is enabled and the template is loaded.
|
|
# These settings can be adjusted to load your own template or overwrite existing ones.
|
|
|
|
# Set to false to disable template loading.
|
|
setup.template.enabled: false
|
|
|
|
# Template name. By default the template name is "{{ beat_name }}-%{[beat.version]}"
|
|
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
|
|
setup.template.name: "{{ beat_name }}-%{[beat.version]}"
|
|
|
|
# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings.
|
|
# The first part is the version of the beat and then -* is used to match all daily indices.
|
|
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
|
|
setup.template.pattern: "{{ beat_name }}-%{[beat.version]}-*"
|
|
|
|
# Path to fields.yml file to generate the template
|
|
setup.template.fields: "${path.config}/fields.yml"
|
|
|
|
# Overwrite existing template
|
|
setup.template.overwrite: false
|
|
|
|
{% set shards = ((data_nodes | length) * 3) | int %}
|
|
|
|
# Elasticsearch template settings
|
|
setup.template.settings:
|
|
|
|
# A dictionary of settings to place into the settings.index dictionary
|
|
# of the Elasticsearch template. For more details, please check
|
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
|
|
index:
|
|
number_of_shards: {{ shards }}
|
|
codec: best_compression
|
|
# This provides for an index split of up to 2 times the number of available shards
|
|
number_of_routing_shards: {{ (shards | int) * 2 }}
|
|
# The default number of replicas will be based on the number of data nodes
|
|
# within the environment with a limit of 2 replicas.
|
|
number_of_replicas: {{ elasticsearch_number_of_replicas }}
|
|
|
|
# A dictionary of settings for the _source field. For more details, please check
|
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
|
|
_source:
|
|
enabled: true
|