Fluentd: rework and fix tail source config

Actually a unique 'tail' source is used for all logs files in
kolla/*/*.log.

So log files from services like: chrony, elasticsearch, etcd, grafana,
influxdb, kibana, mongodb... are actually processed by our Fluentd
config, but should not cause all following config will failed to parse
them. It just uses CPU for nothing cause they will
never match the following filters:
* record_transformer => add Hostname, Logger, programname field
* rewrite_tag_filter => rewrite 'programname' TAG if needed

Hence these filters should only be applied to Openstack services logs to
avoid CPU usage.

Furthermore WGI and Apache logs files must be treated differently.

Change-Id: I425979160291e1123e3775e39243bca06ed22c2f
Closes-Bug: #1711390

ansible/roles/common/tasks/config.yml

@@ -33,6 +33,7 @@
     - "01-syslog"
     - "02-mariadb"
     - "03-rabbitmq"
+    - "04-openstack-wsgi"
   notify:
     - Restart fluentd container
 

ansible/roles/common/templates/conf/input/00-global.conf.j2

@@ -1,9 +1,49 @@
+#jinja2: trim_blocks: False
 {% set fluentd_dir = 'td-agent' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluentd' %}
+{% set services = [
+    ( 'aodh', enable_aodh ),
+    ( 'barbican', enable_barbican ),
+    ( 'ceilometer', enable_ceilometer ),
+    ( 'cinder', enable_cinder ),
+    ( 'cloudkitty', enable_cloudkitty ),
+    ( 'congress', enable_congress ),
+    ( 'designate', enable_designate ),
+    ( 'freezer', enable_freezer ),
+    ( 'glance', enable_glance ),
+    ( 'gnocchi', enable_gnocchi ),
+    ( 'heat', enable_heat ),
+    ( 'horizon', enable_horizon ),
+    ( 'ironic', enable_ironic ),
+    ( 'karbor', enable_karbor ),
+    ( 'keystone', enable_keystone ),
+    ( 'kuryr', enable_kuryr ),
+    ( 'magnum', enable_magnum ),
+    ( 'manila', enable_manila ),
+    ( 'mistral', enable_mistral ),
+    ( 'mongodb', enable_mongodb ),
+    ( 'murano', enable_murano ),
+    ( 'neutron', enable_neutron ),
+    ( 'nova', enable_nova ),
+    ( 'octavia', enable_octavia ),
+    ( 'opendaylight', enable_opendaylight ),
+    ( 'panko', enable_panko ),
+    ( 'rally', enable_rally ),
+    ( 'sahara', enable_sahara ),
+    ( 'searchlight', enable_searchlight ),
+    ( 'senlin', enable_senlin ),
+    ( 'solum', enable_solum ),
+    ( 'tacker', enable_tacker ),
+    ( 'trove', enable_trove ),
+    ( 'watcher', enable_watcher ),
+    ( 'zun', enable_zun )
+] %}
 <source>
   @type tail
-  path /var/log/kolla/*/*.log
-  exclude_path ["/var/log/kolla/rabbitmq/*.log", "/var/log/kolla/mariadb/mariadb.log", "/var/log/kolla/haproxy/*.log", "/var/log/kolla/swift/*.log"]
-  pos_file /var/run/{{ fluentd_dir }}/kolla.pos
+  path {% for service, enabled in services if enabled | bool %}/var/log/kolla/{{ service }}/*.log{% if not loop.last %},{% endif %}{% endfor %}
+  exclude_path ["/var/log/kolla/neutron/dnsmasq.log",
+                "/var/log/kolla/*/*-access.log",
+                "/var/log/kolla/*/*-error.log"]
+  pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
   tag kolla.*
   format /^(?<message>.*)$/
 </source>

ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2

@@ -0,0 +1,9 @@
+# Note (blallau): to manage Apache and WSGI log files
+{% set fluentd_dir = 'td-agent' if kolla_base_distro in ['ubuntu', 'debian'] else 'fluentd' %}
+<source>
+  @type tail
+  path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log
+  pos_file /var/run/{{ fluentd_dir }}/kolla-openstack-wsgi.pos
+  tag kolla.*
+  format /^(?<message>.*)$/
+</source>

ansible/roles/common/templates/fluentd.json.j2

@@ -17,6 +17,12 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {
+            "source": "{{ container_config_directory }}/input/01-syslog.conf",
+            "dest": "{{ fluentd_dir }}/input/01-syslog.conf",
+            "owner": "{{ fluentd_user }}",
+            "perm": "0600"
+        },
         {
             "source": "{{ container_config_directory }}/input/02-mariadb.conf",
             "dest": "{{ fluentd_dir }}/input/02-mariadb.conf",
@@ -30,8 +36,8 @@
             "perm": "0600"
         },
         {
-            "source": "{{ container_config_directory }}/input/01-syslog.conf",
-            "dest": "{{ fluentd_dir }}/input/01-syslog.conf",
+            "source": "{{ container_config_directory }}/input/04-openstack-wsgi.conf",
+            "dest": "{{ fluentd_dir }}/input/04-openstack-wsgi.conf",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },