From 9170f7084fbe8b886968360bd86970dcb82b068b Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 23 Jul 2018 10:26:24 -0500 Subject: [PATCH] Add facts to better optimize thread pools The ansible thread pool is supposed to be based on available processor cores and not hyper-threads. This change updates our settings which will allow the cluster to run generally better and give us more control over JVM memory usage. Change-Id: Ie19e31689c9b18ee2071b3e6fb8e725694fb7897 Signed-off-by: Kevin Carter --- elk_metrics_6x/common_task_data_node_hosts.yml | 1 + elk_metrics_6x/installElastic.yml | 3 ++- .../templates/_include_log_stash_output.yml.j2 | 3 +-- elk_metrics_6x/templates/elasticsearch.yml.j2 | 13 ++++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/elk_metrics_6x/common_task_data_node_hosts.yml b/elk_metrics_6x/common_task_data_node_hosts.yml index 10fe3cb3..f700a4df 100644 --- a/elk_metrics_6x/common_task_data_node_hosts.yml +++ b/elk_metrics_6x/common_task_data_node_hosts.yml @@ -94,6 +94,7 @@ set_fact: master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}" data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}" + elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 24) | ternary(24, ansible_processor_cores) }}" tags: - always diff --git a/elk_metrics_6x/installElastic.yml b/elk_metrics_6x/installElastic.yml index c5aeda6e..0f7530cd 100644 --- a/elk_metrics_6x/installElastic.yml +++ b/elk_metrics_6x/installElastic.yml @@ -25,7 +25,7 @@ tags: - always - - name: Set logstash facts + - name: Set heap facts set_fact: elastic_heap_size: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}" tags: @@ -39,6 +39,7 @@ elasticsearch_node_data: false elasticsearch_node_ingest: false elastic_heap_size: "{{ (elastic_heap_size | int) // 3 }}" + elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, 1) }}" when: - inventory_hostname in (groups['kibana'] | difference(groups['elastic-logstash'])) tags: diff --git a/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 b/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 index 421b14ee..25e269da 100644 --- a/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 +++ b/elk_metrics_6x/templates/_include_log_stash_output.yml.j2 @@ -30,7 +30,6 @@ output.logstash: # if no error is encountered. slow_start: true -{% set thread_pool_size = ansible_processor_cores * ((ansible_processor_threads_per_core > 0) | ternary(ansible_processor_threads_per_core, 1)) %} # The maximum number of events to bulk in a single Logstash request. The # default is the number of cores multiplied by the number of threads, # the resultant is then multiplied again by 256 which results in a the defined @@ -44,7 +43,7 @@ output.logstash: # less than or equal to 0 disables the splitting of batches. When splitting # is disabled, the queue decides on the number of events to be contained in a # batch. - bulk_max_size: {{ thread_pool_size * 256 }} + bulk_max_size: {{ elastic_thread_pool_size * 256 }} {% if named_index is defined %} # Optional index name. The default index name is set to {{ named_index }} diff --git a/elk_metrics_6x/templates/elasticsearch.yml.j2 b/elk_metrics_6x/templates/elasticsearch.yml.j2 index b98e01a8..de4c76bc 100644 --- a/elk_metrics_6x/templates/elasticsearch.yml.j2 +++ b/elk_metrics_6x/templates/elasticsearch.yml.j2 @@ -94,17 +94,16 @@ gateway.recover_after_nodes: {{ ((master_node_count | int) // 2) + 1 }} # Thread pool settings. For more on this see the documentation at: # -{% set thread_pool_size = ansible_processor_cores * ((ansible_processor_threads_per_core > 0) | ternary(ansible_processor_threads_per_core, 1)) %} thread_pool: search: - size: {{ thread_pool_size }} - queue_size: {{ thread_pool_size * 256 }} + size: {{ (elastic_thread_pool_size | int) }} + queue_size: {{ (elastic_thread_pool_size | int) * 256 }} index: - size: {{ thread_pool_size }} - queue_size: {{ thread_pool_size * 256 }} + size: {{ (elastic_thread_pool_size | int) }} + queue_size: {{ (elastic_thread_pool_size | int) * 256 }} bulk: - size: {{ thread_pool_size }} - queue_size: {{ thread_pool_size * 512 }} + size: {{ (elastic_thread_pool_size | int) }} + queue_size: {{ (elastic_thread_pool_size | int) * 512 }} # Accepts either a percentage or a byte size value. Set to 30%, meaning that 30% # of the total heap allocated to a node will be used as the indexing buffer size