From bd685327e5b2cbe13a4bc3a95b46a832662f28e5 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Thu, 30 May 2013 11:41:01 -0700
Subject: [PATCH] Add syslog logs to Logstash.

* modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml:
Add the syslog log file to the list of files to be processed by
logstash-worker1.

* modules/openstack_project/templates/logstash/indexer.conf.erb: Add
Logstash filters for syslog format files.

Change-Id: I0f8f58ab484949eb0506842bdb98385767a50333
Reviewed-on: https://review.openstack.org/31097
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Approved: James E. Blair <corvus@inaugust.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
---
 .../logstash/logstash-worker1/jenkins-log-pusher.yaml  |  5 ++++-
 .../templates/logstash/indexer.conf.erb                | 10 +++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
index 47ed63b519..287bc3ff6d 100644
--- a/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
+++ b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
@@ -30,4 +30,7 @@ source-files:
     tags:
       - screen
       - keystonefmt
-# TODO(clarkb) Add swift and syslog logs here.
+  - name: logs/syslog.txt
+    tags:
+      - syslog
+# TODO(clarkb) Add swift logs here.
diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index 60f8739472..bd651b97e6 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -84,12 +84,20 @@ filter {
     pattern => [ "(?m)^\(\b%{NOTSPACE:module}\b\):%{SPACE}%{DATESTAMP:logdate}%{SPACE}(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR)%{SPACE}%{GREEDYDATA:logmessage}" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
+  grok {
+    type => "jenkins"
+    tags => ["syslog"]
+    # Syslog grok filter adapted from
+    # http://cookbook.logstash.net/recipes/syslog-pri/syslog.conf
+    pattern => [ "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
 
   # Filters below here should be consistent for all Jenkins log formats.
   date {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss" ]
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
   }
   mutate {
     type => "jenkins"