From 26dd0b7a2e7b6bf657d0c1ac977a135e833ce705 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 27 Apr 2018 12:24:04 -0500 Subject: [PATCH] Change filebeat to only execute on hosts Filebeat has everything it needs to function on the physical host machines. This change ensures that filebeat is only ever installed and setup on the physical hosts which will save a lot of IO, and greatly shorten playbook runtimes. Change-Id: I5dd9d3ac9dc6871e14568c1591f3de475cba33e2 Signed-off-by: Kevin Carter --- elk_metrics_6x/installFilebeat.yml | 29 +++++++------- elk_metrics_6x/templates/filebeat.yml.j2 | 50 +++++++++++++++++++++++- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/elk_metrics_6x/installFilebeat.yml b/elk_metrics_6x/installFilebeat.yml index 16d9ed00..c2dacdc9 100644 --- a/elk_metrics_6x/installFilebeat.yml +++ b/elk_metrics_6x/installFilebeat.yml @@ -1,6 +1,6 @@ --- - name: Install Filebeat - hosts: all + hosts: hosts become: true vars: haproxy_ssl: false @@ -112,18 +112,19 @@ apache_enabled: "{{ (apache2.stat.exists | bool) or (httpd.stat.exists | bool) }}" nginx_enabled: "{{ nginx.stat.exists | bool }}" auditd_enabled: "{{ audit.stat.exists | bool }}" - mysql_enabled: "{{ mysql.stat.exists | bool }}" - cinder_enabled: "{{ cinder.stat.exists | bool }}" - glance_enabled: "{{ glance.stat.exists | bool }}" - heat_enabled: "{{ heat.stat.exists | bool }}" - horizon_enabled: "{{ horizon.stat.exists | bool }}" - keystone_enabled: "{{ keystone.stat.exists | bool }}" - neutron_enabled: "{{ neutron.stat.exists | bool }}" - nova_enabled: "{{ nova.stat.exists | bool }}" - octavia_enabled: "{{ octavia.stat.exists | bool }}" - swift_enabled: "{{ swift.stat.exists | bool }}" - rabbitmq_enabled: "{{ rabbitmq.stat.exists | bool }}" - ceph_enabled: "{{ ceph.stat.exists | bool }}" + mysql_enabled: "{{ (mysql.stat.exists | bool) or (inventory_hostname in groups['galera_all'] | default([])) }}" + ceph_enabled: "{{ (ceph.stat.exists | bool) or (inventory_hostname in groups['ceph_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*ceph.*') | list | length) > 0) }}" + cinder_enabled: "{{ (cinder.stat.exists | bool) or (inventory_hostname in groups['cinder_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*cinder.*') | list | length) > 0) }}" + glance_enabled: "{{ (glance.stat.exists | bool) or (inventory_hostname in groups['glance_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*glance.*') | list | length) > 0) }}" + heat_enabled: "{{ (heat.stat.exists | bool) or (inventory_hostname in groups['heat_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*heat.*') | list | length) > 0) }}" + horizon_enabled: "{{ (horizon.stat.exists | bool) or (inventory_hostname in groups['horizon_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*horizon.*') | list | length) > 0) }}" + keystone_enabled: "{{ (keystone.stat.exists | bool) or (inventory_hostname in groups['keystone_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*keystone.*') | list | length) > 0) }}" + neutron_enabled: "{{ (neutron.stat.exists | bool) or (inventory_hostname in groups['neutron_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*neutron.*') | list | length) > 0) }}" + nova_enabled: "{{ (nova.stat.exists | bool) or (inventory_hostname in groups['nova_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*nova.*') | list | length) > 0) }}" + octavia_enabled: "{{ (octavia.stat.exists | bool) or (inventory_hostname in groups['octavia_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*octavia.*') | list | length) > 0) }}" + swift_enabled: "{{ (swift.stat.exists | bool) or (inventory_hostname in groups['swift_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*swift.*') | list | length) > 0) }}" + rabbitmq_enabled: "{{ (rabbitmq.stat.exists | bool) or (inventory_hostname in groups['rabbitmq_all'] | default([])) or (((groups[inventory_hostname + '-host_containers'] | default([])) | select('match', '.*rabbit.*') | list | length) > 0) }}" + post_tasks: - name: Drop Filebeat conf file @@ -139,7 +140,7 @@ - name: Load Filebeat Dashboards - hosts: all[0] + hosts: hosts[0] become: true vars_files: - vars/variables.yml diff --git a/elk_metrics_6x/templates/filebeat.yml.j2 b/elk_metrics_6x/templates/filebeat.yml.j2 index 6d49718a..8b7696b9 100644 --- a/elk_metrics_6x/templates/filebeat.yml.j2 +++ b/elk_metrics_6x/templates/filebeat.yml.j2 @@ -297,6 +297,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/cinder/*.log + - /openstack/log/*cinder*/*.log ### Multiline options @@ -326,6 +327,8 @@ filebeat.prospectors: - openstack - cinder + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -338,6 +341,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/glance/*.log + - /openstack/log/*glance*/*.log ### Multiline options @@ -367,6 +371,8 @@ filebeat.prospectors: - openstack - glance + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -379,6 +385,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/heat/*.log + - /openstack/log/*heat*/*.log ### Multiline options @@ -408,6 +415,8 @@ filebeat.prospectors: - openstack - heat + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -420,6 +429,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/horizon/*.log + - /openstack/log/*horizon*/*.log ### Multiline options @@ -449,6 +459,8 @@ filebeat.prospectors: - openstack - horizon + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -461,6 +473,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/keystone/*.log + - /openstack/log/*keystone*/*.log ### Multiline options @@ -490,6 +503,8 @@ filebeat.prospectors: - openstack - keystone + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -502,6 +517,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/neutron/*.log + - /openstack/log/*neutron*/*.log ### Multiline options @@ -531,6 +547,8 @@ filebeat.prospectors: - openstack - neutron + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -543,6 +561,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/nova/*.log + - /openstack/log/*nova*/*.log ### Multiline options @@ -572,6 +591,8 @@ filebeat.prospectors: - openstack - nova + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -584,6 +605,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/octavia/*.log + - /openstack/log/*octavia*/*.log ### Multiline options @@ -613,6 +635,8 @@ filebeat.prospectors: - openstack - octavia + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -625,6 +649,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/swift/account*.log + - /openstack/log/*swift*/account*.log ### Multiline options @@ -655,6 +680,8 @@ filebeat.prospectors: - swift - swift-account + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -667,6 +694,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/swift/container*.log + - /openstack/log/*swift*/container*.log ### Multiline options @@ -697,6 +725,8 @@ filebeat.prospectors: - swift - swift-container + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -709,6 +739,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/swift/object*.log + - /openstack/log/*swift*/object*.log ### Multiline options @@ -739,6 +770,8 @@ filebeat.prospectors: - swift - swift-object + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -751,6 +784,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/swift/proxy*.log + - /openstack/log/*swift*/proxy*.log # Optional additional fields. These fields can be freely picked # to add additional information to the crawled log files for filtering @@ -759,6 +793,8 @@ filebeat.prospectors: - swift - swift-proxy + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -770,7 +806,8 @@ filebeat.prospectors: # For each file found under this path, a harvester is started. # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - - /var/log/rabbitmq/* + - /var/log/rabbit*/* + - /openstack/log/*rabbit*/* ### Multiline options @@ -799,6 +836,8 @@ filebeat.prospectors: - rabbitmq - infrastructure + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -811,6 +850,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/ceph/ceph-mon.*.log + - /openstack/log/*ceph*/ceph-mon*.log ### Multiline options @@ -840,6 +880,8 @@ filebeat.prospectors: - ceph - infrastructure + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -852,6 +894,7 @@ filebeat.prospectors: # Make sure not file is defined twice as this can lead to unexpected behaviour. paths: - /var/log/ceph/ceph-osd.*.log + - /openstack/log/*ceph*/ceph-osd*.log # Optional additional fields. These fields can be freely picked # to add additional information to the crawled log files for filtering @@ -860,6 +903,8 @@ filebeat.prospectors: - ceph - infrastructure + symlinks: false + - type: log # Change to true to enable this prospector configuration. @@ -876,6 +921,7 @@ filebeat.prospectors: - /var/log/libvirt/*.log - /var/log/libvirt/*/*.log - /var/log/lxc/*.log + - /openstack/log/ansible-logging/*.log #- c:\programdata\elasticsearch\logs\* # Configure the file encoding for reading files with international characters @@ -993,7 +1039,7 @@ filebeat.prospectors: # If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the # original for harvesting but will report the symlink name as source. - #symlinks: false + symlinks: false # Backoff values define how aggressively filebeat crawls new files for updates # The default values can be used in most cases. Backoff defines how long it is waited