opendev/system-config
Code Issues Proposed changes

Add keystone logs to Logstash.

Browse Source 
* modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml:
Add the keystone log file to the list of files to be processed by
logstash-worker1.

* modules/openstack_project/templates/logstash/indexer.conf.erb: Add
Logstash filters for keystone format files.

Change-Id: I5a72fc17ed1f37b816581faabe44f26f8cc36db2
Reviewed-on: https://review.openstack.org/31096
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Approved: James E. Blair <corvus@inaugust.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
This commit is contained in:
Clark Boylan 2013-05-30 11:28:14 -07:00 committed by Jenkins
parent c9d484e797
commit f8e624d1c2
2 changed files with 33 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
View File

@ -26,4 +26,8 @@ source-files:
tags:
- screen
- oslofmt
# TODO(clarkb) Add keystone, swift, and syslog logs here.
- name: logs/screen-key.txt
tags:
- screen
- keystonefmt
# TODO(clarkb) Add swift and syslog logs here.

29
modules/openstack_project/templates/logstash/indexer.conf.erb
View File

@ -26,6 +26,14 @@ filter {
negate => true
match => ["@message", "^\+ "]
}
grep {
# Remove blank lines.
type => "jenkins"
tags => ["keystonefmt"]
# Drop matches.
negate => true
match => ["@message", "^$"]
}
multiline {
type => "jenkins"
tags => ["console.html"]
@ -42,6 +50,14 @@ filter {
what => "previous"
stream_identity => "%{@source_host}.%{filename}"
}
multiline {
type => "jenkins"
tags => ["keystonefmt"]
negate => true
pattern => "^\(\b%{NOTSPACE}\b\):"
what => "previous"
stream_identity => "%{@source_host}.%{filename}"
}
grok {
type => "jenkins"
tags => ["console.html"]
@ -59,10 +75,21 @@ filter {
pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
add_field => [ "received_at", "%{@timestamp}" ]
}
grok {
type => "jenkins"
tags => ["keystonefmt"]
# Do multiline matching as the above mutliline filter may add newlines
# to the log messages.
# TODO move the LOGLEVELs into a proper grok pattern.
pattern => [ "(?m)^\(\b%{NOTSPACE:module}\b\):%{SPACE}%{DATESTAMP:logdate}%{SPACE}(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR)%{SPACE}%{GREEDYDATA:logmessage}" ]
add_field => [ "received_at", "%{@timestamp}" ]
}
# Filters below here should be consistent for all Jenkins log formats.
date {
type => "jenkins"
exclude_tags => "_grokparsefailure"
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss" ]
}
mutate {
type => "jenkins"