openstack/openstack-ansible-ops
Code Issues Proposed changes

Add facts to better optimize thread pools

Browse Source 
The ansible thread pool is supposed to be based on available processor
cores and not hyper-threads. This change updates our settings which will
allow the cluster to run generally better and give us more control over
JVM memory usage.

Change-Id: Ie19e31689c9b18ee2071b3e6fb8e725694fb7897
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2018-07-23 10:26:24 -05:00
parent 0ab9d82545
commit 9170f7084f
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
4 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
elk_metrics_6x/common_task_data_node_hosts.yml
View File

@ -94,6 +94,7 @@
set_fact:
master_node: "{{ (inventory_hostname in master_nodes) | ternary(true, false) }}"
data_node: "{{ (inventory_hostname in data_nodes) | ternary(true, false) }}"
elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 24) | ternary(24, ansible_processor_cores) }}"
tags:
- always

3
elk_metrics_6x/installElastic.yml
View File

@ -25,7 +25,7 @@
tags:
- always
- name: Set logstash facts
- name: Set heap facts
set_fact:
elastic_heap_size: "{{ ((h_mem | int) > 30720) | ternary(30720, h_mem) }}"
tags:
@ -39,6 +39,7 @@
elasticsearch_node_data: false
elasticsearch_node_ingest: false
elastic_heap_size: "{{ (elastic_heap_size | int) // 3 }}"
elastic_thread_pool_size: "{{ ((ansible_processor_cores | int) > 4) | ternary(4, 1) }}"
when:
- inventory_hostname in (groups['kibana'] | difference(groups['elastic-logstash']))
tags:

3
elk_metrics_6x/templates/_include_log_stash_output.yml.j2
View File

@ -30,7 +30,6 @@ output.logstash:
# if no error is encountered.
slow_start: true
{% set thread_pool_size = ansible_processor_cores * ((ansible_processor_threads_per_core > 0) | ternary(ansible_processor_threads_per_core, 1)) %}
# The maximum number of events to bulk in a single Logstash request. The
# default is the number of cores multiplied by the number of threads,
# the resultant is then multiplied again by 256 which results in a the defined
@ -44,7 +43,7 @@ output.logstash:
# less than or equal to 0 disables the splitting of batches. When splitting
# is disabled, the queue decides on the number of events to be contained in a
# batch.
bulk_max_size: {{ thread_pool_size * 256 }}
bulk_max_size: {{ elastic_thread_pool_size * 256 }}
{% if named_index is defined %}
# Optional index name. The default index name is set to {{ named_index }}

13
elk_metrics_6x/templates/elasticsearch.yml.j2
View File

@ -94,17 +94,16 @@ gateway.recover_after_nodes: {{ ((master_node_count | int) // 2) + 1 }}
# Thread pool settings. For more on this see the documentation at:
# <https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-threadpool.html>
{% set thread_pool_size = ansible_processor_cores * ((ansible_processor_threads_per_core > 0) | ternary(ansible_processor_threads_per_core, 1)) %}
thread_pool:
search:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }}
size: {{ (elastic_thread_pool_size | int) }}
queue_size: {{ (elastic_thread_pool_size | int) * 256 }}
index:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 256 }}
size: {{ (elastic_thread_pool_size | int) }}
queue_size: {{ (elastic_thread_pool_size | int) * 256 }}
bulk:
size: {{ thread_pool_size }}
queue_size: {{ thread_pool_size * 512 }}
size: {{ (elastic_thread_pool_size | int) }}
queue_size: {{ (elastic_thread_pool_size | int) * 512 }}
# Accepts either a percentage or a byte size value. Set to 30%, meaning that 30%
# of the total heap allocated to a node will be used as the indexing buffer size