From 67ead4f46bd00091ca4a7a1a860cd839e21f2db8 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 14 Jun 2018 22:47:31 -0500 Subject: [PATCH] Update filebeat Several new options have been added to the filebeat template, while these options have not been enabled, they have been added so operators know which options they have. Other logging options have been consolodated which will cut down on boilerplate code. Change-Id: I640878f41f782a088ed9edd49284f907ad3b6460 Signed-off-by: Kevin Carter --- .../templates/_include_beat_logging.yml.j2 | 49 +++++++ elk_metrics_6x/templates/apm-server.yml.j2 | 52 +------ elk_metrics_6x/templates/auditbeat.yml.j2 | 55 +------- elk_metrics_6x/templates/filebeat.yml.j2 | 132 ++++++++++-------- elk_metrics_6x/templates/heartbeat.yml.j2 | 55 +------- elk_metrics_6x/templates/journalbeat.yml.j2 | 54 +------ elk_metrics_6x/templates/metricbeat.yml.j2 | 55 +------- elk_metrics_6x/templates/packetbeat.yml.j2 | 55 +------- 8 files changed, 140 insertions(+), 367 deletions(-) create mode 100644 elk_metrics_6x/templates/_include_beat_logging.yml.j2 diff --git a/elk_metrics_6x/templates/_include_beat_logging.yml.j2 b/elk_metrics_6x/templates/_include_beat_logging.yml.j2 new file mode 100644 index 00000000..e50e62b4 --- /dev/null +++ b/elk_metrics_6x/templates/_include_beat_logging.yml.j2 @@ -0,0 +1,49 @@ +# There are three options for the log output: syslog, file, stderr. +# Under Windows systems, the log files are per default sent to the file output, +# under all other system per default to syslog. + +# Sets log level. The default log level is info. +# Available log levels are: critical, error, warning, info, debug +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: true + +# If enabled, apm-server periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + path: /var/log/beats + + # The name of the files where the logs are written to. + name: {{ beat_name }}.log + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + keepfiles: 2 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + +# Set to true to log messages in json format. +#logging.json: false diff --git a/elk_metrics_6x/templates/apm-server.yml.j2 b/elk_metrics_6x/templates/apm-server.yml.j2 index 3b3afc97..c1da8a8b 100644 --- a/elk_metrics_6x/templates/apm-server.yml.j2 +++ b/elk_metrics_6x/templates/apm-server.yml.j2 @@ -188,52 +188,6 @@ setup.template.overwrite: true {% include 'templates/_include_kibana_setup.yml.j2' %} #================================ Logging ====================================== -# There are three options for the log output: syslog, file, stderr. -# Under Windows systems, the log files are per default sent to the file output, -# under all other system per default to syslog. - -# Sets log level. The default log level is info. -# Available log levels are: critical, error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: true - -# If enabled, apm-server periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/apm-server - - # The name of the files where the logs are written to. - name: apm-server - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 2 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false +{% with beat_name="apm-server" %} +{% include 'templates/_include_beat_logging.yml.j2' %} +{% endwith %} diff --git a/elk_metrics_6x/templates/auditbeat.yml.j2 b/elk_metrics_6x/templates/auditbeat.yml.j2 index 7a36bb15..4803a59b 100644 --- a/elk_metrics_6x/templates/auditbeat.yml.j2 +++ b/elk_metrics_6x/templates/auditbeat.yml.j2 @@ -722,58 +722,9 @@ setup.template.overwrite: true {% include 'templates/_include_kibana_setup.yml.j2' %} #================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, auditbeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/auditbeat - - # The name of the files where the logs are written to. - name: auditbeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 2 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false - +{% with beat_name="auditbeat" %} +{% include 'templates/_include_beat_logging.yml.j2' %} +{% endwith %} #============================== Xpack Monitoring ===================================== # auditbeat can export internal metrics to a central Elasticsearch monitoring cluster. diff --git a/elk_metrics_6x/templates/filebeat.yml.j2 b/elk_metrics_6x/templates/filebeat.yml.j2 index 10516395..f6c95430 100644 --- a/elk_metrics_6x/templates/filebeat.yml.j2 +++ b/elk_metrics_6x/templates/filebeat.yml.j2 @@ -120,6 +120,32 @@ filebeat.modules: # can be added under this section. #prospector: +#--------------------------------- IIS Module -------------------------------- +#- module: iis + # Access logs + #access: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: + + # Error logs + #error: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: + #-------------------------------- Kafka Module ------------------------------- #- module: kafka # All logs @@ -136,22 +162,36 @@ filebeat.modules: #------------------------------ logstash Module ------------------------------ -#- module: logstash +- module: logstash # logs - #log: - #enabled: true + log: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. # var.paths: # Slow logs - #slowlog: - #enabled: true + slowlog: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: +#------------------------------- mongodb Module ------------------------------ +#- module: mongodb + # Logs + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Input configuration (advanced). Any input configuration option + # can be added under this section. + #input: + #-------------------------------- MySQL Module ------------------------------- - module: mysql # Error logs @@ -169,8 +209,8 @@ filebeat.modules: #prospector: # Slow logs - #slowlog: - #enabled: true + slowlog: + enabled: {{ mysql_enabled | bool }} # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. @@ -959,6 +999,29 @@ filebeat.prospectors: symlinks: false +- type: log + + # Change to true to enable this prospector configuration. + enabled: true + + # Paths that should be crawled and fetched. Glob based paths. + # To fetch all ".log" files from a specific level of subdirectories + # /var/log/*/*.log can be used. + # For each file found under this path, a harvester is started. + # Make sure not file is defined twice as this can lead to unexpected behaviour. + paths: + - /var/log/beats/*.log + - /var/log/curator/curator + - /var/log/elasticsearch/*.log + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + tags: + - beats + + symlinks: false + + - type: log # Change to true to enable this prospector configuration. @@ -1824,58 +1887,9 @@ setup.template.overwrite: true {% include 'templates/_include_kibana_setup.yml.j2' %} #================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, filebeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/filebeat - - # The name of the files where the logs are written to. - name: filebeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 2 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false - +{% with beat_name="filebeat" %} +{% include 'templates/_include_beat_logging.yml.j2' %} +{% endwith %} #============================== Xpack Monitoring ===================================== # filebeat can export internal metrics to a central Elasticsearch monitoring cluster. diff --git a/elk_metrics_6x/templates/heartbeat.yml.j2 b/elk_metrics_6x/templates/heartbeat.yml.j2 index c04f148a..556c97b2 100644 --- a/elk_metrics_6x/templates/heartbeat.yml.j2 +++ b/elk_metrics_6x/templates/heartbeat.yml.j2 @@ -834,58 +834,9 @@ setup.template.overwrite: true {% include 'templates/_include_kibana_setup.yml.j2' %} #================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, heartbeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/heartbeat - - # The name of the files where the logs are written to. - name: heartbeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 2 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false - +{% with beat_name="heartbeat" %} +{% include 'templates/_include_beat_logging.yml.j2' %} +{% endwith %} #============================== Xpack Monitoring ===================================== # heartbeat can export internal metrics to a central Elasticsearch monitoring cluster. diff --git a/elk_metrics_6x/templates/journalbeat.yml.j2 b/elk_metrics_6x/templates/journalbeat.yml.j2 index b25a59f6..e84ecda8 100644 --- a/elk_metrics_6x/templates/journalbeat.yml.j2 +++ b/elk_metrics_6x/templates/journalbeat.yml.j2 @@ -640,54 +640,6 @@ setup.template.overwrite: true {% include 'templates/_include_kibana_setup.yml.j2' %} #================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, journalbeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/journalbeat - - # The name of the files where the logs are written to. - name: journalbeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 2 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false +{% with beat_name="journalbeat" %} +{% include 'templates/_include_beat_logging.yml.j2' %} +{% endwith %} diff --git a/elk_metrics_6x/templates/metricbeat.yml.j2 b/elk_metrics_6x/templates/metricbeat.yml.j2 index d796d04b..d6fe1485 100644 --- a/elk_metrics_6x/templates/metricbeat.yml.j2 +++ b/elk_metrics_6x/templates/metricbeat.yml.j2 @@ -1095,58 +1095,9 @@ setup.template.overwrite: true {% include 'templates/_include_kibana_setup.yml.j2' %} #================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, metricbeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/metricbeat - - # The name of the files where the logs are written to. - name: metricbeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 2 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false - +{% with beat_name="metricbeat" %} +{% include 'templates/_include_beat_logging.yml.j2' %} +{% endwith %} #============================== Xpack Monitoring ===================================== # metricbeat can export internal metrics to a central Elasticsearch monitoring cluster. diff --git a/elk_metrics_6x/templates/packetbeat.yml.j2 b/elk_metrics_6x/templates/packetbeat.yml.j2 index 9ba85b15..c732f035 100644 --- a/elk_metrics_6x/templates/packetbeat.yml.j2 +++ b/elk_metrics_6x/templates/packetbeat.yml.j2 @@ -1110,58 +1110,9 @@ setup.template.overwrite: true {% include 'templates/_include_kibana_setup.yml.j2' %} #================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: info - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, packetbeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/packetbeat - - # The name of the files where the logs are written to. - name: packetbeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - #rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 2 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false - +{% with beat_name="packetbeat" %} +{% include 'templates/_include_beat_logging.yml.j2' %} +{% endwith %} #============================== Xpack Monitoring ===================================== # packetbeat can export internal metrics to a central Elasticsearch monitoring cluster.