diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 index c571623317..ac88a0d525 100644 --- a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 @@ -5,7 +5,9 @@ input { kafka { zk_connect => "{{ monasca_zookeeper_servers }}" topic_id => "{{ monasca_raw_logs_topic }}" - group_id => "transformer-logstash-consumer" + group_id => "log_transformer" + consumer_id => "log_transformer_{{ ansible_hostname }}" + consumer_threads => "{{ monasca_log_pipeline_threads }}" } } @@ -15,11 +17,23 @@ filter { match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"] remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] } + + # OpenStack log levels are uppercase, and syslog are lowercase. + # Furthermore, syslog has more log levels that OpenStack. To avoid + # mapping syslog log levels to OpenStack log levels, we standardise + # on the syslog style here. + if [log][dimensions][log_level] { + mutate { + lowercase => [ "[log][dimensions][log_level]" ] + } + } } output { kafka { bootstrap_servers => "{{ monasca_kafka_servers }}" topic_id => "{{ monasca_transformed_logs_topic }}" + client_id => "log_transformer_{{ ansible_hostname }}" + workers => {{ monasca_log_pipeline_threads|int }} } } diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 index 94cb1cf375..109326485c 100644 --- a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 +++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 @@ -1,5 +1,5 @@ { - "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", + "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", "config_files": [ { "source": "{{ container_config_directory }}/log-transformer.conf",