From 3ca843d707d9f9698be495c898912048a04c278e Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 9 Apr 2013 11:48:55 -0700 Subject: [PATCH] Clean up logstash Jenkins console logs. * modules/openstack_project/files/logstash/log-pusher.py: Change @message to event_messsage. Using @message here appears to confuse logstash and results in duplicate events. * modules/openstack_project/templates/logstash/agent.conf.erb: Change @message to event_messsage. Using @message here appears to confuse logstash and results in duplicate events. Also drop the opening and closing pre tags that end up in the html log files. Change-Id: I3cf679dc644392cfb8ef9002cf149d8b449ad4f5 Reviewed-on: https://review.openstack.org/26518 Reviewed-by: James E. Blair Approved: Clark Boylan Reviewed-by: Clark Boylan Tested-by: Jenkins --- modules/openstack_project/files/logstash/log-pusher.py | 2 +- .../openstack_project/templates/logstash/agent.conf.erb | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/openstack_project/files/logstash/log-pusher.py b/modules/openstack_project/files/logstash/log-pusher.py index 60c34ec634..583f6eeda0 100644 --- a/modules/openstack_project/files/logstash/log-pusher.py +++ b/modules/openstack_project/files/logstash/log-pusher.py @@ -92,7 +92,7 @@ class LogRetriever(threading.Thread): for line in log_lines: out_event = {} out_event["@fields"] = fields - out_event["@message"] = line + out_event["event_message"] = line self.logq.put(out_event) def _parse_fields(self, event): diff --git a/modules/openstack_project/templates/logstash/agent.conf.erb b/modules/openstack_project/templates/logstash/agent.conf.erb index 6a13b0fccf..7bf8246ab4 100644 --- a/modules/openstack_project/templates/logstash/agent.conf.erb +++ b/modules/openstack_project/templates/logstash/agent.conf.erb @@ -2,7 +2,7 @@ input { pipe { command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html" format => "json" - message_format => "%{@message}" + message_format => "%{event_message}" tags => ["jenkins", "console"] type => "jenkins_console" } @@ -10,6 +10,12 @@ input { # You can check grok patterns at http://grokdebug.herokuapp.com/ filter { + grep { + type => "jenkins_console" + # Drop matches. + negate => true + match => ["@message", "^$"] + } grok { type => "jenkins_console" pattern => [ "%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]