From 2c919bc61ca7a903e93372604fa0e770f95b0d52 Mon Sep 17 00:00:00 2001
From: Stig Telfer <stig.openstack@telfer.org>
Date: Wed, 1 Jul 2020 16:42:13 +0200
Subject: [PATCH] Set a chunk size for Fluentd bulk log upload to Monasca

In Fluentd v0.12, both the in memory and file buffer chunk size default
to 8MB. In v1.0 the file buffer defaults to 256MB. This can exceed the
Monasca Log or Unified API maximum chunk size which is set to 10MB.
This can result in logs being rejected and filling the local buffer
on disk.

Change-Id: I9c495773db726a3c5cd94b819dff4141737a1d6e
Closes-Bug: #1885885
Co-Authored-By: Sebastian Luna Valero <sebastian.luna.valero@gmail.com>
---
 .../roles/common/templates/conf/output/00-local.conf.j2  | 9 +++++++++
 .../common/templates/conf/output/02-monasca.conf.j2      | 3 +++
 ...r-chunk-size-for-monasca-output-882338103d2e1f13.yaml | 8 ++++++++
 3 files changed, 20 insertions(+)
 create mode 100644 releasenotes/notes/fix-fluentd-buffer-chunk-size-for-monasca-output-882338103d2e1f13.yaml

diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2
index 6d053513ee..8bb1f9c082 100644
--- a/ansible/roles/common/templates/conf/output/00-local.conf.j2
+++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2
@@ -52,6 +52,9 @@
        buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_swift_facility }}.*
        max_retry_wait 1800s
        disable_retry_limit true
+       <buffer>
+         chunk_limit_size 8m
+       </buffer>
     </store>
 {% endif %}
 </match>
@@ -112,6 +115,9 @@
        buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_haproxy_facility }}.*
        max_retry_wait 1800s
        disable_retry_limit true
+       <buffer>
+         chunk_limit_size 8m
+       </buffer>
     </store>
 {% endif %}
 </match>
@@ -169,6 +175,9 @@
        buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_glance_tls_proxy_facility }}.*
        max_retry_wait 1800s
        disable_retry_limit true
+       <buffer>
+         chunk_limit_size 8m
+       </buffer>
     </store>
 {% endif %}
 </match>
diff --git a/ansible/roles/common/templates/conf/output/02-monasca.conf.j2 b/ansible/roles/common/templates/conf/output/02-monasca.conf.j2
index f0d9cc5923..7a680784de 100644
--- a/ansible/roles/common/templates/conf/output/02-monasca.conf.j2
+++ b/ansible/roles/common/templates/conf/output/02-monasca.conf.j2
@@ -14,5 +14,8 @@
        buffer_path /var/lib/fluentd/data/monasca.buffer/openstack.*
        max_retry_wait 1800s
        disable_retry_limit true
+       <buffer>
+         chunk_limit_size 8m
+       </buffer>
     </store>
 </match>
diff --git a/releasenotes/notes/fix-fluentd-buffer-chunk-size-for-monasca-output-882338103d2e1f13.yaml b/releasenotes/notes/fix-fluentd-buffer-chunk-size-for-monasca-output-882338103d2e1f13.yaml
new file mode 100644
index 0000000000..d56706aede
--- /dev/null
+++ b/releasenotes/notes/fix-fluentd-buffer-chunk-size-for-monasca-output-882338103d2e1f13.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Fixes `LP#1885885
+    <https://bugs.launchpad.net/kolla-ansible/+bug/1885885>`__ where the
+    default chunk size in the Monasca Fluentd output plugin increased from
+    8MB to 256MB for file buffering which exceeded the limit allowed by
+    the Monasca Log / Unified API.