diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one
index ee1da6efe5..12c845f67d 100644
--- a/ansible/inventory/all-in-one
+++ b/ansible/inventory/all-in-one
@@ -449,6 +449,9 @@ monasca
[monasca-log-transformer:children]
monasca
+[monasca-log-persister:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode
index 1caccd0709..c4c9848654 100644
--- a/ansible/inventory/multinode
+++ b/ansible/inventory/multinode
@@ -458,6 +458,9 @@ monasca
[monasca-log-transformer:children]
monasca
+[monasca-log-persister:children]
+monasca
+
# Ironic
[ironic-api:children]
ironic
diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml
index 4db2feb1c6..c8dc213438 100644
--- a/ansible/roles/monasca/defaults/main.yml
+++ b/ansible/roles/monasca/defaults/main.yml
@@ -27,6 +27,15 @@ monasca_services:
- "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "kolla_logs:/var/log/kolla"
+ monasca-log-persister:
+ container_name: monasca_log_persister
+ group: monasca-log-persister
+ enabled: true
+ image: "{{ monasca_logstash_image_full }}"
+ volumes:
+ - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro"
+ - "/etc/localtime:/etc/localtime:ro"
+ - "kolla_logs:/var/log/kolla"
####################
# Databases
@@ -45,6 +54,7 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}"
monasca_metrics_topic: "metrics"
monasca_raw_logs_topic: "logs"
diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml
index 6caadb6df0..c9db3faf06 100644
--- a/ansible/roles/monasca/handlers/main.yml
+++ b/ansible/roles/monasca/handlers/main.yml
@@ -62,3 +62,24 @@
- config_json.changed | bool
or monasca_log_transformer_confs.changed | bool
or monasca_log_transformer_container.changed | bool
+
+- name: Restart monasca-log-persister container
+ vars:
+ service_name: "monasca-log-persister"
+ service: "{{ monasca_services[service_name] }}"
+ config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ monasca_log_persister_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+ kolla_docker:
+ action: "recreate_or_restart_container"
+ common_options: "{{ docker_common_options }}"
+ name: "{{ service.container_name }}"
+ image: "{{ service.image }}"
+ volumes: "{{ service.volumes }}"
+ when:
+ - action != "config"
+ - inventory_hostname in groups[service.group]
+ - service.enabled | bool
+ - config_json.changed | bool
+ or monasca_log_persister_confs.changed | bool
+ or monasca_log_persister_elasticsearch_template.changed | bool
+ or monasca_log_persister_container.changed | bool
diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml
index 42c40f38dc..66217b5124 100644
--- a/ansible/roles/monasca/tasks/config.yml
+++ b/ansible/roles/monasca/tasks/config.yml
@@ -118,6 +118,45 @@
notify:
- Restart monasca-log-transformer container
+- name: Copying over monasca-log-persister config
+ vars:
+ service: "{{ monasca_services['monasca-log-persister'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf"
+ mode: "0660"
+ become: true
+ register: monasca_log_persister_confs
+ with_first_found:
+ - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf"
+ - "{{ node_custom_config }}/monasca/log-persister.conf"
+ - "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2"
+ when:
+ - inventory_hostname in groups[service['group']]
+ - service.enabled | bool
+ notify:
+ - Restart monasca-log-persister container
+
+- name: Copying over monasca-log-persister elasticsearch template
+ vars:
+ service: "{{ monasca_services['monasca-log-persister'] }}"
+ template:
+ src: "{{ item }}"
+ dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json"
+ mode: "0660"
+ become: true
+ register: monasca_log_persister_elasticsearch_template
+ with_first_found:
+ - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json"
+ - "{{ node_custom_config }}/monasca/elasticsearch-template.json"
+ - "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json"
+ when:
+ - inventory_hostname in groups[service['group']]
+ - service.enabled | bool
+ notify:
+ - Restart monasca-log-persister container
+
+
- name: Check monasca containers
become: true
kolla_docker:
diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml
index b386d15e3f..d470d08cfa 100644
--- a/ansible/roles/monasca/tasks/deploy.yml
+++ b/ansible/roles/monasca/tasks/deploy.yml
@@ -6,7 +6,8 @@
- include: config.yml
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
- inventory_hostname in groups['monasca-log-transformer']
+ inventory_hostname in groups['monasca-log-transformer'] or
+ inventory_hostname in groups['monasca-log-persister']
- include: bootstrap.yml
when: inventory_hostname in groups['monasca-api']
@@ -17,4 +18,5 @@
- include: check.yml
when: inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api'] or
- inventory_hostname in groups['monasca-log-transformer']
+ inventory_hostname in groups['monasca-log-transformer'] or
+ inventory_hostname in groups['monasca-log-persister']
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json
new file mode 100644
index 0000000000..c882454cb7
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json
@@ -0,0 +1,56 @@
+{
+ "aliases": {},
+ "mappings": {
+ "log": {
+ "_all": {
+ "enabled": true,
+ "omit_norms": true
+ },
+ "dynamic_templates": [
+ {
+ "message_field": {
+ "mapping": {
+ "fielddata": {
+ "format": "disabled"
+ },
+ "index": "analyzed",
+ "omit_norms": true,
+ "type": "string"
+ },
+ "match": "message",
+ "match_mapping_type": "string"
+ }
+ },
+ {
+ "other_fields": {
+ "mapping": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "match": "*",
+ "match_mapping_type": "string"
+ }
+ }
+ ],
+ "properties": {
+ "@timestamp": {
+ "type": "date"
+ },
+ "@version": {
+ "index": "not_analyzed",
+ "type": "string"
+ },
+ "creation_time": {
+ "type": "date"
+ }
+ }
+ }
+ },
+ "order": 0,
+ "settings": {
+ "index": {
+ "refresh_interval": "5s"
+ }
+ },
+ "template": "monasca-*"
+}
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
new file mode 100644
index 0000000000..31b08d3c5f
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2
@@ -0,0 +1,19 @@
+# Persist transformed logs to Elasticsearch
+
+input {
+ kafka {
+ zk_connect => "{{ monasca_zookeeper_servers }}"
+ topic_id => "{{ monasca_transformed_logs_topic }}"
+ group_id => "transformer-logstash-consumer"
+ }
+}
+
+output {
+ elasticsearch {
+ index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}"
+ hosts => [{{ monasca_elasticsearch_servers }}]
+ document_type => "log"
+ template_name => "monasca"
+ template => "/etc/logstash/elasticsearch-template.json"
+ }
+}
diff --git a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
new file mode 100644
index 0000000000..3eb19fa42d
--- /dev/null
+++ b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2
@@ -0,0 +1,24 @@
+{
+ "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/monasca/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf",
+ "config_files": [
+ {
+ "source": "{{ container_config_directory }}/log-persister.conf",
+ "dest": "/etc/logstash/conf.d/log-persister.conf",
+ "owner": "logstash",
+ "perm": "0600"
+ },
+ {
+ "source": "{{ container_config_directory }}/elasticsearch-template.json",
+ "dest": "/etc/logstash/elasticsearch-template.json",
+ "owner": "logstash",
+ "perm": "0600"
+ }
+ ],
+ "permissions": [
+ {
+ "path": "/var/log/kolla/monasca",
+ "owner": "logstash:kolla",
+ "recurse": true
+ }
+ ]
+}
diff --git a/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml
new file mode 100644
index 0000000000..cc0a4a5c5b
--- /dev/null
+++ b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml
@@ -0,0 +1,6 @@
+---
+features:
+ - |
+ Add support for deploying the Monasca Log Persister. The Log
+ Persister is responsible for reading logs from the Kafka processed
+ logs topic and writing them to Elasticsearch.