From ba736d9840b7c779340faaf0e7af30871e903c1a Mon Sep 17 00:00:00 2001
From: Steve Wilkerson <wilkers.steve@gmail.com>
Date: Thu, 9 Aug 2018 16:42:00 -0500
Subject: [PATCH] Fluent-logging: Update fluentd configuration

This updates the configuration for fluentd, providing a mechanism
for basic determination of the log level of a logged event via
entries from /var/log/containers. This log level is prepended to
the tag for that event, and also added as a new `level` key in
the resulting event. These two improvements allow for querying
specific log level events via the tag.

This also adds similar functionality to any events captured via
the oslo log fluentd handler/formatter. This allows for
elasticsearch queries akin to `error.openstack.keystone`, which
can be used by nagios or another alerting mechanism to raise
alerts when a particular level event has been captured.

Change-Id: I016ddcfcf7408de7b6511ddf7009e1e6a5f3a1d9
---
 fluent-logging/values.yaml | 89 +++++++++++++++++++++++++++++++-------
 1 file changed, 73 insertions(+), 16 deletions(-)

diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml
index 45b3925f39..e519a815c8 100644
--- a/fluent-logging/values.yaml
+++ b/fluent-logging/values.yaml
@@ -158,6 +158,79 @@ conf:
         header: match
         expression: "fluent.**"
         type: "null"
+      # NOTE(srwilkers): Look for specific keywords in the log key to determine
+      # log level of event
+    - tag_kubernetes_log_level:
+        header: match
+        type: rewrite_tag_filter
+        expression: "kube.var.log.containers.**.log"
+        rule:
+          -
+            - header: rule
+              key: log
+              pattern: /info/i
+              tag: info.${tag}
+            - header: rule
+              key: log
+              pattern: /warn/i
+              tag: warn.${tag}
+            - header: rule
+              key: log
+              pattern: /error/i
+              tag: error.${tag}
+            - header: rule
+              key: log
+              pattern: /critical/i
+              tag: critical.${tag}
+            - header: rule
+              key: log
+              pattern: (.+)
+              tag: info.${tag}
+      # NOTE(srwilkers): Create new key for log level, and use the tag prefix
+      # added previously
+    - add_kubernetes_log_level_and_application_key:
+        header: filter
+        type: record_transformer
+        enable_ruby: true
+        expression: "**.kube.var.log.containers.**.log"
+        record:
+          -
+            - header: record
+              level: ${tag_parts[0]}
+              application: ${record["kubernetes"]["labels"]["application"]}
+    - add_openstack_application_key:
+        header: filter
+        type: record_transformer
+        expression: "openstack.**"
+        record:
+          -
+            - header: record
+              application: ${tag_parts[1]}
+      #NOTE(srwilkers): This prefixes the tag for oslo.log entries from the
+      # fluent handler/formatter with the log level, allowing for lookups on
+      # openstack logs with a particular log level (ie: error.openstack.keystone)
+    - tag_openstack_log_level:
+        header: match
+        type: rewrite_tag_filter
+        expression: "openstack.**"
+        rule:
+          -
+            - header: rule
+              key: level
+              pattern: INFO
+              tag: info.${tag}
+            - header: rule
+              key: level
+              pattern: WARN
+              tag: warn.${tag}
+            - header: rule
+              key: level
+              pattern: ERROR
+              tag: error.${tag}
+            - header: rule
+              key: level
+              pattern: CRITICAL
+              tag: critical.${tag}
     - elasticsearch:
         header: match
         type: elasticsearch
@@ -199,20 +272,6 @@ conf:
                 host:
                   type: keyword
                   index: false
-                labels:
-                  properties:
-                    app:
-                      type: keyword
-                      index: false
-                    application:
-                      type: keyword
-                      index: false
-                    component:
-                      type: keyword
-                      index: false
-                    release_group:
-                      type: keyword
-                      index: false
                 namespace_name:
                   type: keyword
                   index: false
@@ -222,8 +281,6 @@ conf:
                 pod_name:
                   type: keyword
                   index: false
-            log:
-              type: text
 
 endpoints:
   cluster_domain_suffix: cluster.local