From b692ce7af1042b4431a04060e066237166531b4d Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 7 Apr 2021 10:09:12 +0000 Subject: [PATCH] Support monitoring Fluentd with Prometheus This patch adds support for integrating Prometheus with Fluentd. This can be used to extract useful information about the status of Fluentd, such as output buffer capacity and logging rate, and also to extract metrics from logs via custom Fluentd configuration. More information can be found here in [1]. [1] https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus Change-Id: I233d6dd744848ef1f1589a462dbf272ed0f3aaae --- ansible/group_vars/all.yml | 2 ++ ansible/roles/common/tasks/config.yml | 32 +++++++++++++------ .../conf/input/08-prometheus.conf.j2 | 14 ++++++++ .../prometheus/templates/prometheus.yml.j2 | 9 ++++++ etc/kolla/globals.yml | 1 + ...for-fluentd-exporter-ca02ff9182039e3e.yaml | 10 ++++++ 6 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 create mode 100644 releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index e7c467bc82..fce29782e5 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -434,6 +434,7 @@ prometheus_memcached_exporter_port: "9150" prometheus_rabbitmq_exporter_port: "15692" # Default cadvisor port of 8080 already in use prometheus_cadvisor_port: "18080" +prometheus_fluentd_integration_port: "24231" # Prometheus alertmanager ports prometheus_alertmanager_port: "9093" @@ -1082,6 +1083,7 @@ enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}" enable_prometheus_memcached_exporter: "{{ enable_memcached | bool }}" enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}" +enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable_fluentd | bool }}" enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" enable_prometheus_ceph_mgr_exporter: "no" enable_prometheus_openstack_exporter: "{{ enable_prometheus | bool }}" diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index fb699f819c..e97485f1c4 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -109,17 +109,29 @@ ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }} fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}" # Inputs - fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}" + fluentd_input_files: "{{ default_input_files_enabled | customise_fluentd(customised_input_files) }}" + default_input_files_enabled: "{{ default_input_files | selectattr('enabled') | map(attribute='name') | list }}" default_input_files: - - "conf/input/00-global.conf.j2" - - "conf/input/01-syslog.conf.j2" - - "conf/input/02-mariadb.conf.j2" - - "conf/input/03-rabbitmq.conf.j2" - - "conf/input/04-openstack-wsgi.conf.j2" - - "conf/input/05-libvirt.conf.j2" - - "conf/input/06-zookeeper.conf.j2" - - "conf/input/07-kafka.conf.j2" - - "conf/input/09-monasca.conf.j2" + - name: "conf/input/00-global.conf.j2" + enabled: true + - name: "conf/input/01-syslog.conf.j2" + enabled: true + - name: "conf/input/02-mariadb.conf.j2" + enabled: true + - name: "conf/input/03-rabbitmq.conf.j2" + enabled: true + - name: "conf/input/04-openstack-wsgi.conf.j2" + enabled: true + - name: "conf/input/05-libvirt.conf.j2" + enabled: true + - name: "conf/input/06-zookeeper.conf.j2" + enabled: true + - name: "conf/input/07-kafka.conf.j2" + enabled: true + - name: "conf/input/08-prometheus.conf.j2" + enabled: "{{ enable_prometheus_fluentd_integration | bool }}" + - name: "conf/input/09-monasca.conf.j2" + enabled: true customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}" # Filters fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}" diff --git a/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 b/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 new file mode 100644 index 0000000000..1aa5da895e --- /dev/null +++ b/ansible/roles/common/templates/conf/input/08-prometheus.conf.j2 @@ -0,0 +1,14 @@ + + @type prometheus + bind {{ api_interface_address }} + port {{ prometheus_fluentd_integration_port }} + metrics_path /metrics + + + + @type prometheus_output_monitor + interval 10 + + Hostname ${hostname} + + diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2 index 44e5226f5f..4dbce1852f 100644 --- a/ansible/roles/prometheus/templates/prometheus.yml.j2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.j2 @@ -74,6 +74,15 @@ scrape_configs: {% endfor %} {% endif %} +{% if enable_prometheus_fluentd_integration | bool %} + - job_name: fluentd + static_configs: + - targets: +{% for host in groups['fluentd'] %} + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}' +{% endfor %} +{% endif %} + {% if enable_prometheus_ceph_mgr_exporter | bool %} - job_name: ceph_mgr_exporter honor_labels: true diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index ccfa01a830..41938086f1 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -685,6 +685,7 @@ #enable_prometheus_mysqld_exporter: "{{ enable_mariadb | bool }}" #enable_prometheus_node_exporter: "{{ enable_prometheus | bool }}" #enable_prometheus_cadvisor: "{{ enable_prometheus | bool }}" +#enable_prometheus_fluentd_integration: "{{ enable_prometheus | bool and enable fluentd | bool }}" #enable_prometheus_memcached: "{{ enable_prometheus | bool }}" #enable_prometheus_alertmanager: "{{ enable_prometheus | bool }}" #enable_prometheus_ceph_mgr_exporter: "no" diff --git a/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml b/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml new file mode 100644 index 0000000000..2896fd2e72 --- /dev/null +++ b/releasenotes/notes/add-support-for-fluentd-exporter-ca02ff9182039e3e.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds support for integrating Fluentd metrics into Prometheus. By + default this is now enabled when Prometheus is enabled. This behaviour + can be overridden via the `enable_prometheus_fluentd_integration` flag. By + default the integration provides metrics relating to the processing of logs + by Fluentd. These metrics can be useful for monitoring the status of the + Fluentd service. Additional metrics can also be extracted from logs via + custom Fluentd config.