From 5441963c9a25257e67d63762d993aaf07d9a1b4c Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Thu, 7 Jun 2018 15:29:24 +0100
Subject: [PATCH] Support deploying Monasca Log Persister
This is a Logstash component which reads processed logs from Kafka
and writes them to Elasticsearch (or some other backend supported by
Logstash).
Ingesting the logs from this service with Fluentd will be covered under
a different commit.
Change-Id: I2d722991ab2072c54c4715507b19a4c9279f921b
Partially-Implements: blueprint monasca-roles
---
ansible/inventory/all-in-one | 3 +
ansible/inventory/multinode | 3 +
ansible/roles/monasca/defaults/main.yml | 10 ++++
ansible/roles/monasca/handlers/main.yml | 21 +++++++
ansible/roles/monasca/tasks/config.yml | 39 +++++++++++++
ansible/roles/monasca/tasks/deploy.yml | 6 +-
.../elasticsearch-template.json | 56 +++++++++++++++++++
.../log-persister.conf.j2 | 19 +++++++
.../monasca-log-persister.json.j2 | 24 ++++++++
...onasca-log-persister-f4da4370a0c5777e.yaml | 6 ++
10 files changed, 185 insertions(+), 2 deletions(-)
create mode 100644 ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json
create mode 100644 ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
create mode 100644 ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
create mode 100644 releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml
diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index ee1da6efe5..12c845f67d 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -449,6 +449,9 @@ monasca
[monasca-log-transformer:children]
monasca
+[monasca-log-persister:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 1caccd0709..c4c9848654 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -458,6 +458,9 @@ monasca
[monasca-log-transformer:children]
monasca
+[monasca-log-persister:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml
index 4db2feb1c6..c8dc213438 100644
--- a/ansible/roles/monasca/defaults/main.yml
+++ b/ansible/roles/monasca/defaults/main.yml
@@ -27,6 +27,15 @@ monasca_services:
- "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla"
+ monasca-log-persister:
+ container_name: monasca_log_persister
+ group: monasca-log-persister
+ enabled: true
+ image: "{{ monasca_logstash_image_full }}"
+ volumes:
+ - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "kolla_logs:/var/log/kolla"
####################
# Databases
@@ -45,6 +54,7 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
monasca_metrics_topic: "metrics"
monasca_raw_logs_topic: "logs"
diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml
index 6caadb6df0..c9db3faf06 100644
--- a/ansible/roles/monasca/handlers/main.yml
+++ b/ansible/roles/monasca/handlers/main.yml
@@ -62,3 +62,24 @@
- config_json.changed | bool
or monasca_log_transformer_confs.changed | bool
or monasca_log_transformer_container.changed | bool
+
+- name: Restart monasca-log-persister container
+ vars:
+ service_name: "monasca-log-persister"
+ service: "{{ monasca_services[service_name] }}"
+ config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ monasca_log_persister_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ volumes: "{{ service.volumes }}"
+ when:
+ - action != "config"
+ - inventory_hostname in groups[service.group]
+ - service.enabled | bool
+ - config_json.changed | bool
+ or monasca_log_persister_confs.changed | bool
+ or monasca_log_persister_elasticsearch_template.changed | bool
+ or monasca_log_persister_container.changed | bool
diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml
index 42c40f38dc..66217b5124 100644
--- a/ansible/roles/monasca/tasks/config.yml
+++ b/ansible/roles/monasca/tasks/config.yml
@@ -118,6 +118,45 @@
notify:
- Restart monasca-log-transformer container
+- name: Copying over monasca-log-persister config
+ vars:
+ service: "{{ monasca_services['monasca-log-persister'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf"
+ mode: "0660"
+ become: true
+ register: monasca_log_persister_confs
+ with_first_found:
+ - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf"
+ - "{{ node_custom_config }}/monasca/log-persister.conf"
+ - "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2"
+ when:
+ - inventory_hostname in groups[service['group']]
+ - service.enabled | bool
+ notify:
+ - Restart monasca-log-persister container
+
+- name: Copying over monasca-log-persister elasticsearch template
+ vars:
+ service: "{{ monasca_services['monasca-log-persister'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json"
+ mode: "0660"
+ become: true
+ register: monasca_log_persister_elasticsearch_template
+ with_first_found:
+ - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json"
+ - "{{ node_custom_config }}/monasca/elasticsearch-template.json"
+ - "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json"
+ when:
+ - inventory_hostname in groups[service['group']]
+ - service.enabled | bool
+ notify:
+ - Restart monasca-log-persister container
+
+
- name: Check monasca containers
become: true
kolla_docker:
diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml
index b386d15e3f..d470d08cfa 100644
--- a/ansible/roles/monasca/tasks/deploy.yml
+++ b/ansible/roles/monasca/tasks/deploy.yml
@@ -6,7 +6,8 @@
- include: config.yml
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
- inventory_hostname in groups['monasca-log-transformer']
+ inventory_hostname in groups['monasca-log-transformer'] or
+ inventory_hostname in groups['monasca-log-persister']
- include: bootstrap.yml
when: inventory_hostname in groups['monasca-api']
@@ -17,4 +18,5 @@
- include: check.yml
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
- inventory_hostname in groups['monasca-log-transformer']
+ inventory_hostname in groups['monasca-log-transformer'] or
+ inventory_hostname in groups['monasca-log-persister']
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json
new file mode 100644
index 0000000000..c882454cb7
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json
@@ -0,0 +1,56 @@
+{
+ "aliases": {},
+ "mappings": {
+ "log": {
+ "_all": {
+ "enabled": true,
+ "omit_norms": true
+ },
+ "dynamic_templates": [
+ {
+ "message_field": {
+ "mapping": {
+ "fielddata": {
+ "format": "disabled"
+ },
+ "index": "analyzed",
+ "omit_norms": true,
+ "type": "string"
+ },
+ "match": "message",
+ "match_mapping_type": "string"
+ }
+ },
+ {
+ "other_fields": {
+ "mapping": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "match": "*",
+ "match_mapping_type": "string"
+ }
+ }
+ ],
+ "properties": {
+ "@timestamp": {
+ "type": "date"
+ },
+ "@version": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "creation_time": {
+ "type": "date"
+ }
+ }
+ }
+ },
+ "order": 0,
+ "settings": {
+ "index": {
+ "refresh_interval": "5s"
+ }
+ },
+ "template": "monasca-*"
+}
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
new file mode 100644
index 0000000000..31b08d3c5f
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
@@ -0,0 +1,19 @@
+# Persist transformed logs to Elasticsearch
+
+input {
+ kafka {
+ zk_connect => "{{ monasca_zookeeper_servers }}"
+ topic_id => "{{ monasca_transformed_logs_topic }}"
+ group_id => "transformer-logstash-consumer"
+ }
+}
+
+output {
+ elasticsearch {
+ index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}"
+ hosts => [{{ monasca_elasticsearch_servers }}]
+ document_type => "log"
+ template_name => "monasca"
+ template => "/etc/logstash/elasticsearch-template.json"
+ }
+}
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
new file mode 100644
index 0000000000..3eb19fa42d
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
@@ -0,0 +1,24 @@
+{
+ "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/monasca/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
+ "config_files": [
+ {
+ "source": "{{ container_config_directory }}/log-persister.conf",
+ "dest": "/etc/logstash/conf.d/log-persister.conf",
+ "owner": "logstash",
+ "perm": "0600"
+ },
+ {
+ "source": "{{ container_config_directory }}/elasticsearch-template.json",
+ "dest": "/etc/logstash/elasticsearch-template.json",
+ "owner": "logstash",
+ "perm": "0600"
+ }
+ ],
+ "permissions": [
+ {
+ "path": "/var/log/kolla/monasca",
+ "owner": "logstash:kolla",
+ "recurse": true
+ }
+ ]
+}
diff --git a/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml
new file mode 100644
index 0000000000..cc0a4a5c5b
--- /dev/null
+++ b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml
@@ -0,0 +1,6 @@
+---
+features:
+ - |
+ Add support for deploying the Monasca Log Persister. The Log
+ Persister is responsible for reading logs from the Kafka processed
+ logs topic and writing them to Elasticsearch.