diff --git a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml index 936edccb..877df849 100644 --- a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml +++ b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -96,7 +96,10 @@ logstash_data_node_details: >- # based on the assignment of roles to hosts, set per host booleans master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}" data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}" -elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) >= 24) | ternary(24, ansible_processor_cores) }}" + +elastic_processors_floor: "{{ ((ansible_processor_count | int) - 1) }}" +elastic_processors_floor_set: "{{ ((elastic_processors_floor | int) > 0) | ternary(elastic_processors_floor, 1) }}" +elastic_thread_pool_size: "{{ ((ansible_processor_count | int) >= 24) | ternary(23, elastic_processors_floor_set) }}" # Set a data node facts. The data nodes, in the case of elasticsearch are also # ingest nodes. diff --git a/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml b/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml index d2574c21..ad8db767 100644 --- a/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml +++ b/elk_metrics_6x/roles/elastic_logstash/defaults/main.yml @@ -17,7 +17,7 @@ temp_dir: /var/lib/logstash/tmp logstash_pipelines: "{{lookup('template', 'logstash-pipelines.yml.j2') }}" # Set processor cores fact -q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_core | int) * 2 }}" +q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_core | int) * 2 }}" # Set logstash facts logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}" diff --git a/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 b/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 index 1fc02e2a..6db9650a 100644 --- a/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 +++ b/elk_metrics_6x/roles/elastic_logstash/templates/logstash.yml.j2 @@ -38,7 +38,10 @@ path.data: /var/lib/logstash # # This defaults to the number of the host's CPU cores. # -{% set processors = ((elastic_thread_pool_size | int) > 0) | ternary(elastic_thread_pool_size, 1) %} + +{% set _h_processors = ((ansible_processor_count | int) // 2) %} +{% set _processors = ((_h_processors | int) > 0) | ternary(_h_processors, 1) %} +{% set processors = ((_processors | int) > 8) | ternary(8, _processors) %} pipeline.workers: {{ processors | int }} # # How many events to retrieve from inputs before sending to filters+workers diff --git a/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml b/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml index 65056b9b..7ba4f17d 100644 --- a/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml +++ b/elk_metrics_6x/roles/elasticsearch/vars/vars_kibana.yml @@ -19,4 +19,9 @@ elasticsearch_node_data: false elasticsearch_node_ingest: false elastic_coordination_node: true elastic_heap_size: "{{ (elastic_heap_size_default | int) // 3 }}" -elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, (ansible_processor_cores // 2)) }}" + +# This variable is redefined because kibana runs elasticsearch but only in a +# load balancer capacity. +elastic_processors_half: "{{ ((ansible_processor_count | int) // 2) }}" +elastic_processors_half_set: "{{ ((elastic_processors_half | int) > 0) | ternary(elastic_processors_half, 1) }}" +elastic_thread_pool_size: "{{ ((elastic_processors_half_set | int) > 4) | ternary(4, elastic_processors_half_set) }}" diff --git a/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 b/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 index 582ca4a5..f0f0946c 100644 --- a/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 +++ b/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 @@ -43,7 +43,7 @@ output.logstash: # less than or equal to 0 disables the splitting of batches. When splitting # is disabled, the queue decides on the number of events to be contained in a # batch. - bulk_max_size: {{ (elastic_thread_pool_size | int) * 256 }} + bulk_max_size: {{ (ansible_processor_count | int) * 256 }} {% if named_index is defined %} # Optional index name. The default index name is set to {{ named_index }} diff --git a/elk_metrics_6x/templates/_include_xpack_elasticsearch_output.yml.j2 b/elk_metrics_6x/templates/_include_xpack_elasticsearch_output.yml.j2 index 771a29d7..5272e16a 100644 --- a/elk_metrics_6x/templates/_include_xpack_elasticsearch_output.yml.j2 +++ b/elk_metrics_6x/templates/_include_xpack_elasticsearch_output.yml.j2 @@ -45,7 +45,7 @@ xpack.monitoring.elasticsearch: # The maximum number of events to bulk in a single Elasticsearch bulk API index request. # The default is 50. - bulk_max_size: {{ (elastic_thread_pool_size | int) * 64 }} + bulk_max_size: {{ (ansible_processor_count | int) * 64 }} # Configure http request timeout before failing an request to Elasticsearch. timeout: 120 diff --git a/elk_metrics_6x/templates/logstash-pipelines.yml.j2 b/elk_metrics_6x/templates/logstash-pipelines.yml.j2 index 57dd325f..13372080 100644 --- a/elk_metrics_6x/templates/logstash-pipelines.yml.j2 +++ b/elk_metrics_6x/templates/logstash-pipelines.yml.j2 @@ -547,12 +547,6 @@ add_tag => ["apimetrics"] } } - } else if "swift-container" in [tags] { - grok { - match => { - "message" => "%{CISCOTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{CISCOTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{USER}%{SPACE}%{USERNAME}%{SPACE}%{NOTSPACE}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{HTTPDUSER}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{INT}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SECOND}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}" - } - } } else if "swift-account" in [tags] { grok { match => {