From 956a29f83a6936a1445c7b5212e2b9e9f84a8bc5 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Tue, 1 Oct 2019 16:08:06 +0100
Subject: [PATCH] Support customizing prometheus.cfg files
This provides a mechanism to scrape targets defined outside of kolla-ansible.
Depends-On: https://review.opendev.org/#/c/685671/
Change-Id: I0950341b147bb374b4128f09f807ef5a756f5dfa
Related: blueprint custom-prometheus-targets
---
ansible/roles/prometheus/tasks/config.yml | 17 +++++++--
.../prometheus-guide.rst | 36 +++++++++++++++++++
...etheus-customization-0b173ff7c63ca30f.yaml | 6 ++++
3 files changed, 57 insertions(+), 2 deletions(-)
create mode 100644 releasenotes/notes/add-mechanism-for-prometheus-customization-0b173ff7c63ca30f.yaml
diff --git a/ansible/roles/prometheus/tasks/config.yml b/ansible/roles/prometheus/tasks/config.yml
index bbc7d2b568..0b1d8d0d45 100644
--- a/ansible/roles/prometheus/tasks/config.yml
+++ b/ansible/roles/prometheus/tasks/config.yml
@@ -56,14 +56,27 @@
notify:
- Restart prometheus-server container
+- name: Find prometheus config overrides
+ find:
+ # NOTE(wszumski): Non-existent paths don't produce a failure
+ paths:
+ - "{{ node_custom_config }}/prometheus/prometheus.yml.d"
+ - "{{ node_custom_config }}/prometheus/{{ inventory_hostname }}/prometheus.yml.d"
+ patterns: "*.yml"
+ delegate_to: localhost
+ register: prometheus_config_overrides_result
+ run_once: true
+
- name: Copying over prometheus config file
become: true
vars:
service: "{{ prometheus_services['prometheus-server'] }}"
- template:
- src: "{{ item }}"
+ overrides: "{{ prometheus_config_overrides_result.files | map(attribute='path') | list }}"
+ merge_yaml:
+ sources: "{{ [prometheus_config_file] + overrides }}"
dest: "{{ node_config_directory }}/prometheus-server/prometheus.yml"
mode: "0660"
+ extend_lists: true
when:
- inventory_hostname in groups[service.group]
- service.enabled | bool
diff --git a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst
index 282f9089ee..7252286888 100644
--- a/doc/source/reference/logging-and-monitoring/prometheus-guide.rst
+++ b/doc/source/reference/logging-and-monitoring/prometheus-guide.rst
@@ -30,3 +30,39 @@ and data retention period to 2 days:
.. code-block:: yaml
prometheus_cmdline_extras: "-storage.remote.timeout 30s -storage.local.retention 48h"
+
+Extending prometheus.cfg
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to add extra targets to scrape, you can extend the default
+``prometheus.yml`` config file by placing additional configs in
+``{{ node_custom_config }}/prometheus/prometheus.yml.d``. These should have the
+same format as ``prometheus.yml``. These additional configs are merged so
+that any list items are extended. For example, if using the default value for
+``node_custom_config``, you could add additional targets to scape by defining
+``/etc/kolla/config/prometheus/prometheus.yml.d/10-custom.yml`` containing the
+following:
+
+.. code-block:: jinja
+
+ scrape_configs:
+ - job_name: custom
+ static_configs:
+ - targets:
+ - '10.0.0.111:1234'
+ - job_name: custom-template
+ static_configs:
+ - targets:
+ {% for host in groups['prometheus'] %}
+ - '{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ 3456 }}'
+ {% endfor %}
+
+The jobs, ``custom``, and ``custom_template`` would be appended to the default
+list of ``scrape_configs`` in the final ``prometheus.yml``. To customize on a per
+host basis, files can also be placed in
+``{{ node_custom_config }}/prometheus/<inventory_hostname>/prometheus.yml.d``
+where, ``inventory_hostname`` is one of the hosts in your inventory. These
+will be merged with any files in ``{{ node_custom_config }}/prometheus/prometheus.yml.d``,
+so in order to override a list value instead of extending it, you will need to make
+sure that no files in ``{{ node_custom_config }}/prometheus/prometheus.yml.d``
+set a key with an equivalent hierarchical path.
diff --git a/releasenotes/notes/add-mechanism-for-prometheus-customization-0b173ff7c63ca30f.yaml b/releasenotes/notes/add-mechanism-for-prometheus-customization-0b173ff7c63ca30f.yaml
new file mode 100644
index 0000000000..a7ded83164
--- /dev/null
+++ b/releasenotes/notes/add-mechanism-for-prometheus-customization-0b173ff7c63ca30f.yaml
@@ -0,0 +1,6 @@
+---
+features:
+ - |
+ Added a mechanism to customize ``prometheus.yml``. Please read the the
+ `documentation <https://docs.openstack.org/kolla-ansible/latest/reference/logging-and-monitoring/prometheus-guide.html>`__.
+ for more details.