From 0c573062fc25e208bfa1206146fb31b401c8b7e5 Mon Sep 17 00:00:00 2001
From: Krzysztof Klimonda <kklimonda@cloudferro.com>
Date: Tue, 28 May 2019 12:05:48 +0000
Subject: [PATCH] Make fluentd-elasticsearch configuration more robust
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Enable reconnect_on_error option so that ES plugin re-establishes
a new session to the ES cluster on errors. Also, enable buffering
to the file, so that the buffer survives container restarts.
Co-Authored-By: Michal Nasiadka <mnasiadka@gmail.com>
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>
Co-Authored-By: Doug Szumski <doug@stackhpc.com>
Closes-Bug: #1830724
Change-Id: Ia40685b9d4fc02194e03c8791ddeb3d29d7f07f6
---
ansible/roles/common/defaults/main.yml | 1 +
ansible/roles/common/templates/conf/output/00-local.conf.j2 | 6 ++++++
ansible/roles/common/templates/conf/output/01-es.conf.j2 | 3 +++
ansible/roles/common/templates/fluentd.json.j2 | 5 +++++
4 files changed, 15 insertions(+)
diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml
index d8c6ce9b0d..4d2b46041c 100644
--- a/ansible/roles/common/defaults/main.yml
+++ b/ansible/roles/common/defaults/main.yml
@@ -84,6 +84,7 @@ fluentd_default_volumes:
- "{{ node_config_directory }}/fluentd/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla/"
+ - "fluentd_data:/var/lib/fluentd/data/"
kolla_toolbox_extra_volumes: "{{ default_extra_volumes }}"
cron_extra_volumes: "{{ default_extra_volumes }}"
fluentd_extra_volumes: "{{ default_extra_volumes }}"
diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2
index f9753eb9d6..550749c7ec 100644
--- a/ansible/roles/common/templates/conf/output/00-local.conf.j2
+++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2
@@ -27,6 +27,9 @@
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
+ reconnect_on_error true
+ buffer_type file
+ buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
</store>
{% elif enable_monasca | bool %}
type copy
@@ -75,6 +78,9 @@
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
+ reconnect_on_error true
+ buffer_type file
+ buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
</store>
{% elif enable_monasca | bool %}
type copy
diff --git a/ansible/roles/common/templates/conf/output/01-es.conf.j2 b/ansible/roles/common/templates/conf/output/01-es.conf.j2
index 178acf407a..38500e8e94 100644
--- a/ansible/roles/common/templates/conf/output/01-es.conf.j2
+++ b/ansible/roles/common/templates/conf/output/01-es.conf.j2
@@ -19,5 +19,8 @@
logstash_format true
logstash_prefix {{ kibana_log_prefix }}
flush_interval 15s
+ reconnect_on_error true
+ buffer_type file
+ buffer_path /var/lib/fluentd/data/elasticsearch.buffer/openstack.*
</store>
</match>
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 40d2d14fb5..9cef1f8a68 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -65,6 +65,11 @@
"path": "/var/log/kolla/swift",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
+ },
+ {
+ "path": "/var/lib/fluentd/data",
+ "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
+ "recurse": true
}
]