Update filebeat

Several new options have been added to the filebeat template, while
these options have not been enabled, they have been added so operators
know which options they have.

Other logging options have been consolodated which will cut down on
boilerplate code.

Change-Id: I640878f41f782a088ed9edd49284f907ad3b6460
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/templates/_include_beat_logging.yml.j2

@@ -0,0 +1,49 @@
+# There are three options for the log output: syslog, file, stderr.
+# Under Windows systems, the log files are per default sent to the file output,
+# under all other system per default to syslog.
+
+# Sets log level. The default log level is info.
+# Available log levels are: critical, error, warning, info, debug
+#logging.level: info
+
+# Enable debug output for selected components. To enable all selectors use ["*"]
+# Other available selectors are "beat", "publish", "service"
+# Multiple selectors can be chained.
+#logging.selectors: [ ]
+
+# Send all logging output to syslog. The default is false.
+#logging.to_syslog: true
+
+# If enabled, apm-server periodically logs its internal metrics that have changed
+# in the last period. For each metric that changed, the delta from the value at
+# the beginning of the period is logged. Also, the total values for
+# all non-zero internal metrics are logged on shutdown. The default is true.
+#logging.metrics.enabled: true
+
+# The period after which to log the internal metrics. The default is 30s.
+#logging.metrics.period: 30s
+
+# Logging to rotating files. Set logging.to_files to false to disable logging to
+# files.
+logging.to_files: true
+logging.files:
+  # Configure the path where the logs are written. The default is the logs directory
+  # under the home path (the binary location).
+  path: /var/log/beats
+
+  # The name of the files where the logs are written to.
+  name: {{ beat_name }}.log
+
+  # Configure log file size limit. If limit is reached, log file will be
+  # automatically rotated
+  #rotateeverybytes: 10485760 # = 10MB
+
+  # Number of rotated log files to keep. Oldest files will be deleted first.
+  keepfiles: 2
+
+  # The permissions mask to apply when rotating log files. The default value is 0600.
+  # Must be a valid Unix-style file permissions mask expressed in octal notation.
+  #permissions: 0600
+
+# Set to true to log messages in json format.
+#logging.json: false

elk_metrics_6x/templates/apm-server.yml.j2

@@ -188,52 +188,6 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are three options for the log output: syslog, file, stderr.
-# Under Windows systems, the log files are per default sent to the file output,
-# under all other system per default to syslog.
-
-# Sets log level. The default log level is info.
-# Available log levels are: critical, error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: true
-
-# If enabled, apm-server periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/apm-server
-
-  # The name of the files where the logs are written to.
-  name: apm-server
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
+{% with beat_name="apm-server" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}

elk_metrics_6x/templates/auditbeat.yml.j2

@@ -722,58 +722,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, auditbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/auditbeat
-
-  # The name of the files where the logs are written to.
-  name: auditbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="auditbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # auditbeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/filebeat.yml.j2

@@ -120,6 +120,32 @@ filebeat.modules:
     # can be added under this section.
     #prospector:
 
+#--------------------------------- IIS Module --------------------------------
+#- module: iis
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:
+
 #-------------------------------- Kafka Module -------------------------------
 #- module: kafka
   # All logs
@@ -136,22 +162,36 @@ filebeat.modules:
 
 
 #------------------------------ logstash Module ------------------------------
-#- module: logstash
+- module: logstash
   # logs
-  #log:
-    #enabled: true
+  log:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     # var.paths:
 
   # Slow logs
-  #slowlog:
-   #enabled: true
+  slowlog:
+    enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+#------------------------------- mongodb Module ------------------------------
+#- module: mongodb
+  # Logs
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Input configuration (advanced). Any input configuration option
+    # can be added under this section.
+    #input:
+
 #-------------------------------- MySQL Module -------------------------------
 - module: mysql
   # Error logs
@@ -169,8 +209,8 @@ filebeat.modules:
     #prospector:
 
   # Slow logs
-  #slowlog:
-    #enabled: true
+  slowlog:
+    enabled: {{ mysql_enabled | bool }}
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
@@ -959,6 +999,29 @@ filebeat.prospectors:
 
   symlinks: false
 
+- type: log
+
+  # Change to true to enable this prospector configuration.
+  enabled: true
+
+  # Paths that should be crawled and fetched. Glob based paths.
+  # To fetch all ".log" files from a specific level of subdirectories
+  # /var/log/*/*.log can be used.
+  # For each file found under this path, a harvester is started.
+  # Make sure not file is defined twice as this can lead to unexpected behaviour.
+  paths:
+    - /var/log/beats/*.log
+    - /var/log/curator/curator
+    - /var/log/elasticsearch/*.log
+
+  # Optional additional fields. These fields can be freely picked
+  # to add additional information to the crawled log files for filtering
+  tags:
+    - beats
+
+  symlinks: false
+
+
 - type: log
 
   # Change to true to enable this prospector configuration.
@@ -1824,58 +1887,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, filebeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/filebeat
-
-  # The name of the files where the logs are written to.
-  name: filebeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="filebeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # filebeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/heartbeat.yml.j2

@@ -834,58 +834,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, heartbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/heartbeat
-
-  # The name of the files where the logs are written to.
-  name: heartbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="heartbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # heartbeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/journalbeat.yml.j2

@@ -640,54 +640,6 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, journalbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/journalbeat
-
-  # The name of the files where the logs are written to.
-  name: journalbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
+{% with beat_name="journalbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}

elk_metrics_6x/templates/metricbeat.yml.j2

@@ -1095,58 +1095,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, metricbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/metricbeat
-
-  # The name of the files where the logs are written to.
-  name: metricbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="metricbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # metricbeat can export internal metrics to a central Elasticsearch monitoring cluster.

elk_metrics_6x/templates/packetbeat.yml.j2

@@ -1110,58 +1110,9 @@ setup.template.overwrite: true
 {% include 'templates/_include_kibana_setup.yml.j2' %}
 
 #================================ Logging ======================================
-# There are four options for the log output: file, stderr, syslog, eventlog
-# The file output is the default.
-
-# Sets log level. The default log level is info.
-# Available log levels are: error, warning, info, debug
-#logging.level: info
-
-# Enable debug output for selected components. To enable all selectors use ["*"]
-# Other available selectors are "beat", "publish", "service"
-# Multiple selectors can be chained.
-#logging.selectors: [ ]
-
-# Send all logging output to syslog. The default is false.
-#logging.to_syslog: false
-
-# Send all logging output to Windows Event Logs. The default is false.
-#logging.to_eventlog: false
-
-# If enabled, packetbeat periodically logs its internal metrics that have changed
-# in the last period. For each metric that changed, the delta from the value at
-# the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
-#logging.metrics.enabled: true
-
-# The period after which to log the internal metrics. The default is 30s.
-#logging.metrics.period: 30s
-
-# Logging to rotating files. Set logging.to_files to false to disable logging to
-# files.
-logging.to_files: true
-logging.files:
-  # Configure the path where the logs are written. The default is the logs directory
-  # under the home path (the binary location).
-  path: /var/log/packetbeat
-
-  # The name of the files where the logs are written to.
-  name: packetbeat
-
-  # Configure log file size limit. If limit is reached, log file will be
-  # automatically rotated
-  #rotateeverybytes: 10485760 # = 10MB
-
-  # Number of rotated log files to keep. Oldest files will be deleted first.
-  keepfiles: 2
-
-  # The permissions mask to apply when rotating log files. The default value is 0600.
-  # Must be a valid Unix-style file permissions mask expressed in octal notation.
-  #permissions: 0600
-
-# Set to true to log messages in json format.
-#logging.json: false
-
+{% with beat_name="packetbeat" %}
+{%   include 'templates/_include_beat_logging.yml.j2' %}
+{% endwith %}
 
 #============================== Xpack Monitoring =====================================
 # packetbeat can export internal metrics to a central Elasticsearch monitoring cluster.