From 063518d0720d56a5b70ba277f1c645decb554c9a Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Mon, 24 Sep 2018 17:06:51 +0000 Subject: [PATCH] Update Monasca Log Transformer config * Convert all log levels to lowercase * Make number of threads configurable * Improve naming * Update path and log options for Logstash 2.4 * Change worker count from string to integer for the Kafka output plugin supplied with Logstash 2.4. Partially-Implements: blueprint monasca-roles Change-Id: I9379969ee639dff0b525d7a1a03df59bcf18b98b --- .../log-transformer.conf.j2 | 16 +++++++++++++++- .../monasca-log-transformer.json.j2 | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 index c571623317..ac88a0d525 100644 --- a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 @@ -5,7 +5,9 @@ input { kafka { zk_connect => "{{ monasca_zookeeper_servers }}" topic_id => "{{ monasca_raw_logs_topic }}" - group_id => "transformer-logstash-consumer" + group_id => "log_transformer" + consumer_id => "log_transformer_{{ ansible_hostname }}" + consumer_threads => "{{ monasca_log_pipeline_threads }}" } } @@ -15,11 +17,23 @@ filter { match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"] remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] } + + # OpenStack log levels are uppercase, and syslog are lowercase. + # Furthermore, syslog has more log levels that OpenStack. To avoid + # mapping syslog log levels to OpenStack log levels, we standardise + # on the syslog style here. + if [log][dimensions][log_level] { + mutate { + lowercase => [ "[log][dimensions][log_level]" ] + } + } } output { kafka { bootstrap_servers => "{{ monasca_kafka_servers }}" topic_id => "{{ monasca_transformed_logs_topic }}" + client_id => "log_transformer_{{ ansible_hostname }}" + workers => {{ monasca_log_pipeline_threads|int }} } } diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 index 94cb1cf375..109326485c 100644 --- a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 +++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 @@ -1,5 +1,5 @@ { - "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", + "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", "config_files": [ { "source": "{{ container_config_directory }}/log-transformer.conf",