openstack/openstack-ansible-ops
Code Issues Proposed changes

Exposed config for logstash elasticsearch endpoints

Browse Source 
The Elasticsearch endpoints in use by logstash are currently hardcoded
to localhost in both the logstash monitoring config and the default
logstash pipeline template. This commit allows this to be configured,
such that (multiple) remote endpoints can be specified for the case that
logstash is not co-located with elasticsearch. The default behaviour is
unchanged.

Change-Id: Id871f201168ff882a5b37677747c3484f908298f
This commit is contained in:
Duncan Martin Walker 2020-02-03 16:54:56 +00:00
parent 33d3822f99
commit b21034c4b8
3 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
elk_metrics_7x/roles/elastic_logstash/defaults/main.yml
View File

@ -15,6 +15,8 @@
temp_dir: /var/lib/logstash/tmp
logstash_pipelines: "{{lookup('template', 'logstash-pipelines.yml.j2') }}"
logstash_elasticsearch_endpoints:
- "127.0.0.1:{{ elastic_port }}"
# Set processor cores fact
q_storage: 1

3
elk_metrics_7x/roles/elastic_logstash/templates/logstash.yml.j2
View File

@ -231,7 +231,7 @@ path.logs: /var/log/logstash
xpack.monitoring.enabled: true
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: password
xpack.monitoring.elasticsearch.hosts: ["http://127.0.0.1:9200"]
xpack.monitoring.elasticsearch.hosts: {{ logstash_elasticsearch_endpoints | to_json }}
#xpack.monitoring.elasticsearch.ssl.certificate_authority: [ "/path/to/ca.crt" ]
#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file
#xpack.monitoring.elasticsearch.ssl.truststore.password: password
@ -257,4 +257,3 @@ xpack.monitoring.collection.pipeline.details.enabled: true
#xpack.management.elasticsearch.ssl.verification_mode: certificate
#xpack.management.elasticsearch.sniffing: false
#xpack.management.logstash.poll_interval: 5s

20
elk_metrics_7x/templates/logstash-pipelines.yml.j2
View File

@ -495,7 +495,7 @@
elasticsearch {
id => "elasticsearchDocIDOutputPipeline"
document_id => "%{[@metadata][fingerprint]}"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "%{[@metadata][beat]}-%{[@metadata][version]}"
@ -504,7 +504,7 @@
elasticsearch {
id => "elasticsearchLegacyDocIDOutputPipeline"
document_id => "%{[@metadata][fingerprint]}"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
@ -513,7 +513,7 @@
elasticsearch {
id => "elasticsearchSyslogDocIDOutputPipeline"
document_id => "%{[@metadata][fingerprint]}"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "syslog-%{+YYYY.MM.dd}"
@ -522,7 +522,7 @@
elasticsearch {
id => "elasticsearchCollectdDocIDOutputPipeline"
document_id => "%{[@metadata][fingerprint]}"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "collectd-%{+YYYY.MM.dd}"
@ -531,7 +531,7 @@
elasticsearch {
id => "elasticsearchUndefinedDocIDOutputPipeline"
document_id => "%{[@metadata][fingerprint]}"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "undefined-%{+YYYY.MM.dd}"
@ -541,7 +541,7 @@
if [@metadata][version] {
elasticsearch {
id => "elasticsearchOutputPipeline"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "%{[@metadata][beat]}-%{[@metadata][version]}"
@ -549,7 +549,7 @@
} else if [@metadata][beat] {
elasticsearch {
id => "elasticsearchLegacyOutputPipeline"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
@ -557,7 +557,7 @@
} else if "syslog" in [tags] {
elasticsearch {
id => "elasticsearchSyslogOutputPipeline"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "syslog-%{+YYYY.MM.dd}"
@ -565,7 +565,7 @@
} else if "collectd" in [tags] {
elasticsearch {
id => "elasticsearchCollectdOutputPipeline"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "collectd-%{+YYYY.MM.dd}"
@ -573,7 +573,7 @@
} else {
elasticsearch {
id => "elasticsearchUndefinedOutputPipeline"
hosts => ["{{ '127.0.0.1:' ~ elastic_port }}"]
hosts => {{ logstash_elasticsearch_endpoints | to_json }}
sniffing => {{ (elastic_sniffing_enabled | default(not data_node)) | bool | string | lower }}
manage_template => {{ (data_node | bool) | lower }}
index => "undefined-%{+YYYY.MM.dd}"