Add syslog input into logstash

The new option logstash_syslog_input_enabled has been added which will
allow users to enable a direct syslog input. When enabled, messages will
be processed via logstash and sent directly to elasticsearch.

Change-Id: Icb7712ecb8aae3d7f99df80ae1c5cd647a15ce83
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/roles/elastic_dependencies/tasks/main.yml

@@ -137,14 +137,21 @@
     system: "yes"
 
 - name: Create the system user
-  user:
+  block:
-    name: "{{ service_owner }}"
+    - name: Create the system user
-    uid: "{{ service_owner_uid | default(omit) }}"
+      user:
-    group: "{{ service_group }}"
+        name: "{{ service_owner }}"
-    shell: "/bin/false"
+        uid: "{{ service_owner_uid | default(omit) }}"
-    system: "yes"
+        group: "{{ service_group }}"
-    createhome: "yes"
+        shell: "/bin/false"
-    home: "/var/lib/{{ service_name }}"
+        system: "yes"
+        createhome: "yes"
+        home: "/var/lib/{{ service_name }}"
+  rescue:
+    - name: Ensure the system user exists
+      user:
+        name: "{{ service_owner }}"
+        group: "{{ service_group }}"
 
 - name: Ensure service directories exists
   file:

elk_metrics_6x/roles/elastic_logstash/defaults/main.yml

@@ -22,3 +22,12 @@ q_storage: "{{ (ansible_processor_cores | int) * (ansible_processor_threads_per_
 # Set logstash facts
 logstash_queue_size: "{{ ((((q_storage | int) >= 2) | ternary(q_storage, 2) | int) * 1024) // ((logstash_pipelines | from_yaml) | length) }}"
 elastic_log_rotate_path: "/var/log/logstash"
+
+# Enable direct syslog input into logstash. When this is enabled syslog messages
+# can be sent directly to logstash via TCP or UDP.
+logstash_syslog_input_enabled: false
+# The typical syslog port is 514 however that is not available to logstash
+# because it's a "privledged" port. For this reason 1514 is used as the default.
+# Changing this port to 514 will require overrides to the service files making
+# logstash run as root (not recommended).
+logstash_syslog_input_port: 1514

elk_metrics_6x/templates/logstash-pipelines.yml.j2

@@ -8,6 +8,35 @@
 {% endif %}
 {% set output_pipeline = output_pipeline | to_json %}
 
+{% if logstash_syslog_input_enabled | bool %}
+- pipeline.id: "syslog-intake"
+  queue.type: persisted
+  config.string: |
+    input {
+      tcp {
+        id => "inputSyslogTcp"
+        port => {{ logstash_syslog_input_port }}
+        type => syslog
+      }
+      udp {
+        id => "inputSyslogUdp"
+        port => {{ logstash_syslog_input_port }}
+        type => syslog
+      }
+    }
+    filter {
+      mutate {
+        add_tag => ["syslog"]
+      }
+    }
+    output {
+      pipeline {
+        id => "sendDistributorPipeline"
+        send_to => [distributor]
+      }
+    }
+{% endif %}
+
 - pipeline.id: "beats-intake"
   queue.type: persisted
   config.string: |
@@ -702,7 +731,7 @@
           manage_template => {{ (data_node | bool) | lower }}
           index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
         }
-      } else {
+      } else if [@metadata][beat] {
         elasticsearch {
           id => "elasticsearchLegacyOutputPipeline"
           document_id => "%{[@metadata][fingerprint]}"
@@ -711,6 +740,24 @@
           manage_template => {{ (data_node | bool) | lower }}
           index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
         }
+      } else if "syslog" in [tags] {
+        elasticsearch {
+          id => "elasticsearchSyslogOutputPipeline"
+          document_id => "%{[@metadata][fingerprint]}"
+          hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }}
+          sniffing => {{ (not data_node | bool) | lower }}
+          manage_template => {{ (data_node | bool) | lower }}
+          index => "syslog-%{+YYYY.MM.dd}"
+        }
+      } else {
+        elasticsearch {
+          id => "elasticsearchUndefinedOutputPipeline"
+          document_id => "%{[@metadata][fingerprint]}"
+          hosts => {{ elasticsearch_data_hosts | shuffle(seed=inventory_hostname) | to_json }}
+          sniffing => {{ (not data_node | bool) | lower }}
+          manage_template => {{ (data_node | bool) | lower }}
+          index => "undefined-%{+YYYY.MM.dd}"
+        }
       }
     }