Merge "Add persisted logstash queues and tag journalbeat"

elk_metrics_6x/files/02-journald.conf

@@ -0,0 +1,22 @@
+filter {
+  if "journald" in [tags] {
+    if [systemd_slice] {
+      mutate {
+        copy => { "systemd_slice" => "systemd_slice_tag" }
+      }
+      mutate {
+        gsub => [ "systemd_slice_tag", ".slice", "" ]
+      }
+      if [systemd_slice_tag] != "-" {
+        mutate {
+          add_tag => [
+            "%{systemd_slice_tag}"
+          ]
+        }
+      }
+      mutate {
+        remove_field => [ "%{systemd_slice_tag}" ]
+      }
+    }
+  }
+}

elk_metrics_6x/installLogstash.yml

@@ -15,12 +15,14 @@
     - name: Set quarter memory fact
       set_fact:
         q_mem: "{{ (ansible_memtotal_mb | int) // 4 }}"
+        q_storage: "{{ ansible_processor_cores }}"
       tags:
         - always
 
     - name: Set logstash facts
       set_fact:
         elastic_heap_size: "{{ ((q_mem | int) > 30720) | ternary(30720, q_mem) }}"
+        logstash_queue_size: "{{ (((q_storage | int) > 16) | ternary(16, q_storage) | int) * 1024 }}"
       tags:
         - always
 
@@ -79,21 +81,6 @@
       tags:
         - package_install
 
-    - name: Drop Logstash conf for beats input
-      template:
-        src: templates/02-beats-input.conf.j2
-        dest: /etc/logstash/conf.d/02-beats-input.conf
-
-    - name: Drop Logstash conf for beats input
-      template:
-        src: templates/10-syslog-filter.conf.j2
-        dest: /etc/logstash/conf.d/10-syslog-filter.conf
-
-    - name: Drop Logstash conf for elasticsearch output
-      template:
-        src: templates/99-elasticsearch-output.conf.j2
-        dest: /etc/logstash/conf.d/99-elasticsearch-output.conf
-
     - name: Drop elasticsearch conf file
       template:
         src: "{{ item.src }}"
@@ -101,6 +88,14 @@
       with_items:
         - src: templates/jvm.options.j2
           dest: /etc/logstash/jvm.options
+        - src: templates/99-elasticsearch-output.conf.j2
+          dest: /etc/logstash/conf.d/99-elasticsearch-output.conf
+        - src: templates/02-beats-input.conf.j2
+          dest: /etc/logstash/conf.d/02-beats-input.conf
+        - src: templates/logstash.yml.j2
+          dest: /etc/logstash/logstash.yml
+      notify:
+        - Enable and restart logstash
       tags:
         - config
 
@@ -123,16 +118,19 @@
         - extras
       when:
         - logstash_deploy_filters
+      notify:
+        - Enable and restart logstash
       tags:
         - logstash-filters
         - config
 
     - name: Deploy Logstash configuration files
-      template:
-        src: "{{ item }}"
+      copy:
+        src: "files/{{ item }}"
         dest: "/etc/logstash/conf.d/{{ item }}"
       with_items:
         - 02-general.conf
+        - 02-journald.conf
         - 03-nova.conf
         - 04-neutron.conf
         - 05-glance.conf
@@ -141,6 +139,7 @@
         - 08-apache.conf
         - 09-heat.conf
         - 10-mysql.conf
+        - 10-syslog-filter.conf
         - 11-auth.conf
         - 12-logstash.conf
         - 13-swift.conf
@@ -154,6 +153,8 @@
         - 98-traceback.conf
       when:
         - logstash_deploy_filters
+      notify:
+        - Enable and restart logstash
       tags:
         - logstash-filters
         - config
@@ -178,6 +179,8 @@
           template:
             src: "templates/99-kafka-output.conf"
             dest: "/etc/logstash/conf.d/99-kafka-output.conf"
+          notify:
+            - Enable and restart logstash
       when:
         - logstash_kafka_options is defined
 
@@ -187,7 +190,12 @@
         owner: logstash
         group: logstash
         recurse: true
+      register: l_perms
+      until: l_perms is success
+      retries: 3
+      delay: 1
 
+  handlers:
     - name: Enable and restart logstash
       systemd:
         name: "logstash"

elk_metrics_6x/templates/journalbeat.yml.j2

@@ -83,7 +83,8 @@ name: journalbeat
 # The tags of the shipper are included in their own field with each
 # transaction published. Tags make it easy to group servers by different
 # logical properties.
-#tags: ["service-X", "web-tier"]
+tags:
+  - journald
 
 # Optional fields that you can specify to add additional information to the
 # output. Fields can be scalar values, arrays, dictionaries, or any nested

elk_metrics_6x/templates/logstash.yml.j2

@@ -0,0 +1,246 @@
+# Settings file in YAML
+#
+# Settings can be specified either in hierarchical form, e.g.:
+#
+#   pipeline:
+#     batch:
+#       size: 125
+#       delay: 5
+#
+# Or as flat keys:
+#
+#   pipeline.batch.size: 125
+#   pipeline.batch.delay: 5
+#
+# ------------  Node identity ------------
+#
+# Use a descriptive name for the node:
+#
+# node.name: test
+#
+# If omitted the node name will default to the machine's host name
+#
+# ------------ Data path ------------------
+#
+# Which directory should be used by logstash and its plugins
+# for any persistent needs. Defaults to LOGSTASH_HOME/data
+#
+path.data: /var/lib/logstash
+#
+# ------------ Pipeline Settings --------------
+#
+# The ID of the pipeline.
+#
+# pipeline.id: main
+#
+# Set the number of workers that will, in parallel, execute the filters+outputs
+# stage of the pipeline.
+#
+# This defaults to the number of the host's CPU cores.
+#
+# pipeline.workers: 2
+#
+# How many events to retrieve from inputs before sending to filters+workers
+#
+# pipeline.batch.size: 125
+#
+# How long to wait in milliseconds while polling for the next event
+# before dispatching an undersized batch to filters+outputs
+#
+# pipeline.batch.delay: 50
+#
+# Force Logstash to exit during shutdown even if there are still inflight
+# events in memory. By default, logstash will refuse to quit until all
+# received events have been pushed to the outputs.
+#
+# WARNING: enabling this can lead to data loss during shutdown
+#
+# pipeline.unsafe_shutdown: false
+#
+# ------------ Pipeline Configuration Settings --------------
+#
+# Where to fetch the pipeline configuration for the main pipeline
+#
+# path.config:
+#
+# Pipeline configuration string for the main pipeline
+#
+# config.string:
+#
+# At startup, test if the configuration is valid and exit (dry run)
+#
+# config.test_and_exit: false
+#
+# Periodically check if the configuration has changed and reload the pipeline
+# This can also be triggered manually through the SIGHUP signal
+#
+# config.reload.automatic: false
+#
+# How often to check if the pipeline configuration has changed (in seconds)
+#
+# config.reload.interval: 3s
+#
+# Show fully compiled configuration as debug log message
+# NOTE: --log.level must be 'debug'
+#
+# config.debug: false
+#
+# When enabled, process escaped characters such as \n and \" in strings in the
+# pipeline configuration files.
+#
+# config.support_escapes: false
+#
+# ------------ Module Settings ---------------
+# Define modules here.  Modules definitions must be defined as an array.
+# The simple way to see this is to prepend each `name` with a `-`, and keep
+# all associated variables under the `name` they are associated with, and
+# above the next, like this:
+#
+# modules:
+#   - name: MODULE_NAME
+#     var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
+#     var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
+#     var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
+#     var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
+#
+# Module variable names must be in the format of
+#
+# var.PLUGIN_TYPE.PLUGIN_NAME.KEY
+#
+# modules:
+#
+# ------------ Cloud Settings ---------------
+# Define Elastic Cloud settings here.
+# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
+# and it may have an label prefix e.g. staging:dXMtZ...
+# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
+# cloud.id: <identifier>
+#
+# Format of cloud.auth is: <user>:<pass>
+# This is optional
+# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
+# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
+# cloud.auth: elastic:<password>
+#
+# ------------ Queuing Settings --------------
+#
+# Internal queuing model, "memory" for legacy in-memory based queuing and
+# "persisted" for disk-based acked queueing. Defaults is memory
+#
+queue.type: persisted
+#
+# If using queue.type: persisted, the directory path where the data files will be stored.
+# Default is path.data/queue
+#
+# path.queue:
+#
+# If using queue.type: persisted, the page data files size. The queue data consists of
+# append-only data files separated into pages. Default is 64mb
+#
+# queue.page_capacity: 64mb
+#
+# If using queue.type: persisted, the maximum number of unread events in the queue.
+# Default is 0 (unlimited)
+#
+# queue.max_events: 0
+#
+# If using queue.type: persisted, the total capacity of the queue in number of bytes.
+# If you would like more unacked events to be buffered in Logstash, you can increase the
+# capacity using this setting. Please make sure your disk drive has capacity greater than
+# the size specified here. If both max_bytes and max_events are specified, Logstash will pick
+# whichever criteria is reached first
+# Default is 1024mb or 1gb
+#
+queue.max_bytes: {{ logstash_queue_size }}mb
+#
+# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint
+# Default is 1024, 0 for unlimited
+#
+# queue.checkpoint.acks: 1024
+#
+# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint
+# Default is 1024, 0 for unlimited
+#
+# queue.checkpoint.writes: 1024
+#
+# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page
+# Default is 1000, 0 for no periodic checkpoint.
+#
+# queue.checkpoint.interval: 1000
+#
+# ------------ Dead-Letter Queue Settings --------------
+# Flag to turn on dead-letter queue.
+#
+# dead_letter_queue.enable: false
+
+# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries
+# will be dropped if they would increase the size of the dead letter queue beyond this setting.
+# Default is 1024mb
+# dead_letter_queue.max_bytes: 1024mb
+
+# If using dead_letter_queue.enable: true, the directory path where the data files will be stored.
+# Default is path.data/dead_letter_queue
+#
+# path.dead_letter_queue:
+#
+# ------------ Metrics Settings --------------
+#
+# Bind address for the metrics REST endpoint
+#
+# http.host: "127.0.0.1"
+#
+# Bind port for the metrics REST endpoint, this option also accept a range
+# (9600-9700) and logstash will pick up the first available ports.
+#
+# http.port: 9600-9700
+#
+# ------------ Debugging Settings --------------
+#
+# Options for log.level:
+#   * fatal
+#   * error
+#   * warn
+#   * info (default)
+#   * debug
+#   * trace
+#
+# log.level: info
+path.logs: /var/log/logstash
+#
+# ------------ Other Settings --------------
+#
+# Where to find custom plugins
+# path.plugins: []
+#
+# ------------ X-Pack Settings (not applicable for OSS build)--------------
+#
+# X-Pack Monitoring
+# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html
+#xpack.monitoring.enabled: false
+#xpack.monitoring.elasticsearch.username: logstash_system
+#xpack.monitoring.elasticsearch.password: password
+#xpack.monitoring.elasticsearch.url: ["https://es1:9200", "https://es2:9200"]
+#xpack.monitoring.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ]
+#xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file
+#xpack.monitoring.elasticsearch.ssl.truststore.password: password
+#xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file
+#xpack.monitoring.elasticsearch.ssl.keystore.password: password
+#xpack.monitoring.elasticsearch.ssl.verification_mode: certificate
+#xpack.monitoring.elasticsearch.sniffing: false
+#xpack.monitoring.collection.interval: 10s
+#xpack.monitoring.collection.pipeline.details.enabled: true
+#
+# X-Pack Management
+# https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html
+#xpack.management.enabled: false
+#xpack.management.pipeline.id: ["main", "apache_logs"]
+#xpack.management.elasticsearch.username: logstash_admin_user
+#xpack.management.elasticsearch.password: password
+#xpack.management.elasticsearch.url: ["https://es1:9200", "https://es2:9200"]
+#xpack.management.elasticsearch.ssl.ca: [ "/path/to/ca.crt" ]
+#xpack.management.elasticsearch.ssl.truststore.path: /path/to/file
+#xpack.management.elasticsearch.ssl.truststore.password: password
+#xpack.management.elasticsearch.ssl.keystore.path: /path/to/file
+#xpack.management.elasticsearch.ssl.keystore.password: password
+#xpack.management.elasticsearch.sniffing: false
+#xpack.management.logstash.poll_interval: 5s