From 23ac2aa98504245bf370c08e25d7332fe668f1f3 Mon Sep 17 00:00:00 2001 From: Dave Wilde Date: Thu, 7 Jun 2018 10:35:50 -0500 Subject: [PATCH] Add logstash filters This adds the ability to include logstash log parsing filters for various openstack and service logs. These filters are disabled by default and can be enabled by toggling the deploy_logstash_filters variable. Change-Id: I5c46f78f232d3fb604283ae623cd3975a8346c7c --- elk_metrics_6x/installLogstash.yml | 45 +++++++++++++++ elk_metrics_6x/templates/02-general.conf | 16 ++++++ elk_metrics_6x/templates/03-nova.conf | 23 ++++++++ elk_metrics_6x/templates/04-neutron.conf | 28 ++++++++++ elk_metrics_6x/templates/05-glance.conf | 16 ++++++ elk_metrics_6x/templates/06-cinder.conf | 18 ++++++ elk_metrics_6x/templates/07-libvirt.conf | 11 ++++ elk_metrics_6x/templates/08-apache.conf | 29 ++++++++++ elk_metrics_6x/templates/09-heat.conf | 23 ++++++++ elk_metrics_6x/templates/10-mysql.conf | 33 +++++++++++ elk_metrics_6x/templates/11-auth.conf | 10 ++++ elk_metrics_6x/templates/12-logstash.conf | 25 +++++++++ elk_metrics_6x/templates/13-swift.conf | 56 +++++++++++++++++++ elk_metrics_6x/templates/14-keystone.conf | 28 ++++++++++ .../templates/16-elasticsearch.conf | 10 ++++ elk_metrics_6x/templates/17-rabbitmq.conf | 20 +++++++ elk_metrics_6x/templates/18-ceph.conf | 12 ++++ elk_metrics_6x/templates/19-nginx.conf | 20 +++++++ elk_metrics_6x/templates/20-magnum.conf | 16 ++++++ elk_metrics_6x/templates/21-octavia.conf | 16 ++++++ elk_metrics_6x/templates/98-traceback.conf | 8 +++ ...onf.j2 => 99-elasticsearch-output.conf.j2} | 0 elk_metrics_6x/vars/variables.yml | 2 +- 23 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 elk_metrics_6x/templates/02-general.conf create mode 100644 elk_metrics_6x/templates/03-nova.conf create mode 100644 elk_metrics_6x/templates/04-neutron.conf create mode 100644 elk_metrics_6x/templates/05-glance.conf create mode 100644 elk_metrics_6x/templates/06-cinder.conf create mode 100644 elk_metrics_6x/templates/07-libvirt.conf create mode 100644 elk_metrics_6x/templates/08-apache.conf create mode 100644 elk_metrics_6x/templates/09-heat.conf create mode 100644 elk_metrics_6x/templates/10-mysql.conf create mode 100644 elk_metrics_6x/templates/11-auth.conf create mode 100644 elk_metrics_6x/templates/12-logstash.conf create mode 100644 elk_metrics_6x/templates/13-swift.conf create mode 100644 elk_metrics_6x/templates/14-keystone.conf create mode 100644 elk_metrics_6x/templates/16-elasticsearch.conf create mode 100644 elk_metrics_6x/templates/17-rabbitmq.conf create mode 100644 elk_metrics_6x/templates/18-ceph.conf create mode 100644 elk_metrics_6x/templates/19-nginx.conf create mode 100644 elk_metrics_6x/templates/20-magnum.conf create mode 100644 elk_metrics_6x/templates/21-octavia.conf create mode 100644 elk_metrics_6x/templates/98-traceback.conf rename elk_metrics_6x/templates/{30-elasticsearch-output.conf.j2 => 99-elasticsearch-output.conf.j2} (100%) diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index 509a4815..07aba9d7 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -86,6 +86,51 @@ tags: - config + - name: Logstash Extra Patterns + template: + src: "{{ item }}" + dest: "/opt/logstash/patterns/{{ item }}" + owner: "logstash" + group: "logstash" + with_items: + - extras + when: + - logstash_deploy_filters + tags: + - logstash-filters + - config + + - name: Deploy Logstash configuration files + template: + src: "{{ item }}" + dest: "/etc/logstash/conf.d/{{ item }}" + with_items: + - 02-general.conf + - 03-nova.conf + - 04-neutron.conf + - 05-glance.conf + - 06-cinder.conf + - 07-libvirt.conf + - 08-apache.conf + - 09-heat.conf + - 10-mysql.conf + - 11-auth.conf + - 12-logstash.conf + - 13-swift.conf + - 14-keystone.conf + - 16-elasticsearch.conf + - 17-rabbitmq.conf + - 18-ceph.conf + - 19-nginx.conf + - 20-magnum.conf + - 21-octavia.conf + - 98-traceback.conf + when: + - logstash_deploy_filters + tags: + - logstash-filters + - config + - name: Ensure logstash ownership file: path: /var/lib/logstash diff --git a/elk_metrics_6x/templates/02-general.conf b/elk_metrics_6x/templates/02-general.conf new file mode 100644 index 00000000..16bba6bf --- /dev/null +++ b/elk_metrics_6x/templates/02-general.conf @@ -0,0 +1,16 @@ +filter { + if "oslofmt" in [tags] or "openstack" in [tags] { + if "Can not find policy directory: policy.d" in [message] { + drop{} + } + grok { + match => { + "message" => [ + "^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?", + "^%{CISCOTIMESTAMP:journalddate}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{GREEDYDATA:logmessage}" + ] + } + add_field => { "received_at" => "%{@timestamp}" } + } + } +} diff --git a/elk_metrics_6x/templates/03-nova.conf b/elk_metrics_6x/templates/03-nova.conf new file mode 100644 index 00000000..c020994c --- /dev/null +++ b/elk_metrics_6x/templates/03-nova.conf @@ -0,0 +1,23 @@ +filter { + if "nova" in [tags] { + mutate { + gsub => ['logmessage',"\"",""] + } + if [module] == "nova.osapi_compute.wsgi.server" { + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes:int} time\: %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } else if [module] == "nova.api.ec2" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NUMBER:seconds}s %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} None\:None %{NUMBER:response} %{GREEDYDATA:user_agent}" } + add_tag => ["apimetrics"] + } + } else if [module] == "nova.metadata.wsgi.server" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" } + add_tag => ["apimetrics"] + } + } + } +} diff --git a/elk_metrics_6x/templates/04-neutron.conf b/elk_metrics_6x/templates/04-neutron.conf new file mode 100644 index 00000000..13acfdfc --- /dev/null +++ b/elk_metrics_6x/templates/04-neutron.conf @@ -0,0 +1,28 @@ +filter { + if "neutron" in [tags] { + if [module] == "neutron.wsgi" { + if "accepted" not in [logmessage] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } + } else if "neutron-ha-tool" in [source] { + mutate { + add_tag => ["neutron-ha-tool"] + remove_tag => ["_grokparsefailure"] + } + } + if "starting" in [message] and "_grokparsefailure" in [tags] { + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid}|\-)\](%{SPACE}\(%{NUMBER:pid}\)) %{GREEDYDATA:servicemessage}" } + } + mutate { + remove_tag => ["_grokparsefailure"] + } + } + } +} diff --git a/elk_metrics_6x/templates/05-glance.conf b/elk_metrics_6x/templates/05-glance.conf new file mode 100644 index 00000000..2eaec5dd --- /dev/null +++ b/elk_metrics_6x/templates/05-glance.conf @@ -0,0 +1,16 @@ +filter { + if "glance" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "glance.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/06-cinder.conf b/elk_metrics_6x/templates/06-cinder.conf new file mode 100644 index 00000000..09c62fe9 --- /dev/null +++ b/elk_metrics_6x/templates/06-cinder.conf @@ -0,0 +1,18 @@ +filter { + if "cinder" in [tags] { + if [module] == "cinder.eventlet.wsgi.server" { + if "accepted" not in [logmessage] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } + mutate { + replace => { "module" => "cinder.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/07-libvirt.conf b/elk_metrics_6x/templates/07-libvirt.conf new file mode 100644 index 00000000..c417c155 --- /dev/null +++ b/elk_metrics_6x/templates/07-libvirt.conf @@ -0,0 +1,11 @@ +filter { + if "libvirt" in [tags] { + grok { + match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" } + add_field => { "received_at" => "%{@timestamp}"} + } + mutate { + uppercase => [ "loglevel" ] + } + } +} diff --git a/elk_metrics_6x/templates/08-apache.conf b/elk_metrics_6x/templates/08-apache.conf new file mode 100644 index 00000000..ba1ad58f --- /dev/null +++ b/elk_metrics_6x/templates/08-apache.conf @@ -0,0 +1,29 @@ +filter { + if "horizon" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { + "message" => [ + "%{COMMONAPACHELOG}", + "\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}", + "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}" + ] + } + } + + if ![loglevel] { + mutate { + add_field => { "logmessage" => "%{request}" } + add_field => { "module" => "horizon.access" } + add_field => { "loglevel" => "INFO" } + add_tag => [ "apache-access" ] + } + } else { + mutate { + replace => { "module" => "horizon.error.%{module}" } + add_tag => [ "apache-error" ] + uppercase => [ "loglevel" ] + } + } + } +} diff --git a/elk_metrics_6x/templates/09-heat.conf b/elk_metrics_6x/templates/09-heat.conf new file mode 100644 index 00000000..8c22e39a --- /dev/null +++ b/elk_metrics_6x/templates/09-heat.conf @@ -0,0 +1,23 @@ +filter { + if "heat" in [tags] { + if [module] == "eventlet.wsgi.server" { + if "accepted" not in [logmessage] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE}\] %{NOTSPACE:requesterip} %{NOTSPACE} %{NOTSPACE} \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{BASE10NUM:httptime}" } + add_tag => ["apimetrics"] + } + } + mutate { + replace => { "module" => "heat.%{module}" } + } + } else if [module] == "heat.engine.service" { + grok { + match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE} %{GREEDYDATA:servicemessage}" } + add_tag => ["apimetrics"] + } + } + } +} diff --git a/elk_metrics_6x/templates/10-mysql.conf b/elk_metrics_6x/templates/10-mysql.conf new file mode 100644 index 00000000..861b240b --- /dev/null +++ b/elk_metrics_6x/templates/10-mysql.conf @@ -0,0 +1,33 @@ +filter { + if "mysql" in [tags] { + grok { + match => { "message" => "# User@Host: %{WORD:user}\[%{WORD}\] @ (%{HOSTNAME:client_hostname}|) \[(%{IP:client_ip}|)\]" } + } + + grok { + match => { "message" => "# Thread_id: %{NUMBER:thread_id:int} \s*Schema: (%{WORD:schema}| ) \s*QC_hit: %{WORD:qc_hit}" } + } + + grok { + match => { "message" => "# Query_time: %{NUMBER:query_time:float} \s*Lock_time: %{NUMBER:lock_time:float} \s*Rows_sent: %{NUMBER:rows_sent:int} \s*Rows_examined: %{NUMBER:rows_examined:int}" } + } + + grok { + match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" } + } + + date { + match => [ "timestamp", "UNIX" ] + } + + mutate { + remove_field => "timestamp" + } + + mutate { + gsub => [ "logmessage", "^\n", "" ] + add_field => { "module" => "mysql" } + add_field => { "loglevel" => "WARNING" } + } + } +} diff --git a/elk_metrics_6x/templates/11-auth.conf b/elk_metrics_6x/templates/11-auth.conf new file mode 100644 index 00000000..d9d1ee45 --- /dev/null +++ b/elk_metrics_6x/templates/11-auth.conf @@ -0,0 +1,10 @@ +filter { + if "auth" in [tags] { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" } + } + mutate { + add_field => { "module" => "auth" } + } + } +} diff --git a/elk_metrics_6x/templates/12-logstash.conf b/elk_metrics_6x/templates/12-logstash.conf new file mode 100644 index 00000000..e8ff8d3e --- /dev/null +++ b/elk_metrics_6x/templates/12-logstash.conf @@ -0,0 +1,25 @@ +filter { + if "logstash" in [tags] { + grok { + match => { + "message" => "\{\:timestamp=>\"%{TIMESTAMP_ISO8601:timestamp}\", \:message=>\"%{DATA:logmessage}\"(;|)(, \:address=>\"%{URIHOST:address}\", \:exception=>#<%{DATA:exception}>, \:backtrace=>\[%{DATA:backtrace}\]|)(, \:level=>:%{LOGLEVEL:loglevel}|)\}" + } + } + + mutate { + add_field => { "module" => "logstash" } + uppercase => [ "loglevel" ] + } + + if [loglevel] == "WARN" { + mutate { + replace => { "loglevel" => "WARNING" } + } + } else if ![loglevel] { + mutate { + add_field => { "loglevel" => "ERROR" } + } + } + + } +} diff --git a/elk_metrics_6x/templates/13-swift.conf b/elk_metrics_6x/templates/13-swift.conf new file mode 100644 index 00000000..aaa577d5 --- /dev/null +++ b/elk_metrics_6x/templates/13-swift.conf @@ -0,0 +1,56 @@ +filter { + if "swift-container" in [tags] { + grok { + match => { + "message" => "%{CISCOTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{CISCOTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{USER}%{SPACE}%{USERNAME}%{SPACE}%{NOTSPACE}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{HTTPDUSER}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{INT}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SECOND}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}" + } + } + } + if "swift-account" in [tags] { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{SYSLOGTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{IP}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{POSINT}%{SPACE}%{NOTSPACE}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{SECOND}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NOTSPACE}" + } + } + } + if "swift" in [tags] { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG:module}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" + } + } + + grok { + patterns_dir => ['/opt/logstash/patterns'] + match => { + "logmessage" => [ + "%{COMBINEDAPACHELOG}", + "%{SWIFTPROXY_ACCESS}", + "%{GREEDYDATA:logmessage} \(txn\: %{DATA:swift_txn}\)" + ] + } + tag_on_failure => [] + overwrite => [ "logmessage" ] + } + + if [request] { + mutate { + replace => { "logmessage" => "%{request}" } + } + } + + mutate { + replace => { "module" => "swift.%{module}" } + } + + if [file] =~ "error.log$" { + mutate { + add_field => { "loglevel" => "NOTICE" } + } + } else { + mutate { + add_field => { "loglevel" => "INFO" } + } + } + } +} diff --git a/elk_metrics_6x/templates/14-keystone.conf b/elk_metrics_6x/templates/14-keystone.conf new file mode 100644 index 00000000..e8cc6eca --- /dev/null +++ b/elk_metrics_6x/templates/14-keystone.conf @@ -0,0 +1,28 @@ +filter { + if "keystone-access" in [tags] { + grok { + match => { "message" => "%{CISCOTIMESTAMP:keystone_access_timestamp}%{SPACE}%{SYSLOGHOST:log_host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:keystone_timestmp}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{SYSLOG5424SD:requestid}%{SPACE}%{WORD:verb}%{SPACE}%{NOTSPACE:request}" } + } + } + if "keystone" in [tags] { + if "apache-access" in [tags] { + grok { + match => { "message" => "%{COMMONAPACHELOG}" } + } + mutate { + add_field => { "logmessage" => "%{request}" } + add_field => { "module" => "keystone.access" } + add_field => { "loglevel" => "INFO" } + } + } else if "apache-error" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { "message" => "%{KEYSTONE_SUBSECOND_TIMESTAMP:keystone_subsecond_timestamp} %{STANDARD_TIMESTAMP:standard_timestamp} %{NUMBER:pid} %{DATA:loglevel} %{DATA:module} \[%{DATA:requestid}\] %{WORD:verb} %{NOTSPACE:request}" } + } + mutate { + replace => { "module" => "keystone.error.%{module}" } + uppercase => [ "loglevel" ] + } + } + } +} diff --git a/elk_metrics_6x/templates/16-elasticsearch.conf b/elk_metrics_6x/templates/16-elasticsearch.conf new file mode 100644 index 00000000..8ca5890d --- /dev/null +++ b/elk_metrics_6x/templates/16-elasticsearch.conf @@ -0,0 +1,10 @@ +filter { + if "elasticsearch" in [tags] { + grok { + match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{LOGLEVEL:loglevel}\s*\]\[%{NOTSPACE:module}\s*\] %{GREEDYDATA:logmessage}" } + } + mutate { + replace => { "module" => "elasticsearch.%{module}" } + } + } +} diff --git a/elk_metrics_6x/templates/17-rabbitmq.conf b/elk_metrics_6x/templates/17-rabbitmq.conf new file mode 100644 index 00000000..c46f2551 --- /dev/null +++ b/elk_metrics_6x/templates/17-rabbitmq.conf @@ -0,0 +1,20 @@ +filter { + if "rabbitmq" in [tags] { + if [message] == "" { + drop { } + } + grok { + match => { "message" => "^\=%{LOGLEVEL:loglevel} REPORT\=\=\=\= %{MONTHDAY:event_day}\-%{MONTH:event_month}\-%{YEAR:event_year}\:\:%{TIME:event_time} \=\=\=\n%{GREEDYDATA:logmessage}" } + } + + mutate { + replace => { "module" => "rabbitmq" } + add_field => { "timestamp" => "%{event_day} %{event_month} %{event_year} %{event_time}" } + } + + date { + match => [ "timestamp", "dd MMM YYYY HH:mm:ss" ] + remove_field => [ "event_day", "event_month", "event_year", "event_time", "timestamp" ] + } + } +} diff --git a/elk_metrics_6x/templates/18-ceph.conf b/elk_metrics_6x/templates/18-ceph.conf new file mode 100644 index 00000000..5eb0d700 --- /dev/null +++ b/elk_metrics_6x/templates/18-ceph.conf @@ -0,0 +1,12 @@ +filter { + if "ceph" in [tags] { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:date} %{NOTSPACE:osd_epoch} ?%{SPACE}?%{NOTSPACE:error_bool} %{GREEDYDATA:logmessage}" } + } + } + if "ceph-osd" in [tags] { + grok { + match => { "message" => "-- (?(%{IPORHOST}\:%{POSINT}/%{POSINT})) (?:[<|>]){1,2} (?(%{IPORHOST}\:%{POSINT}/%{POSINT}))" } + } + } +} diff --git a/elk_metrics_6x/templates/19-nginx.conf b/elk_metrics_6x/templates/19-nginx.conf new file mode 100644 index 00000000..0c3f5ba9 --- /dev/null +++ b/elk_metrics_6x/templates/19-nginx.conf @@ -0,0 +1,20 @@ +filter { + if "nginx" in [tags] { + if "nginx-access" in [tags] { + grok { + patterns_dir => ['/opt/logstash/patterns'] + match => { + "message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}" + } + } + } + if "nginx-error" in [tags] { + grok { + patterns_dir => ['/opt/logstash/patterns'] + match => { + "message" => "%{NGINX_ERROR_TIMESTAMP:timestamp} \[%{LOGLEVEL:loglevel}\] %{GREEDYDATA:error_msg}" + } + } + } + } +} diff --git a/elk_metrics_6x/templates/20-magnum.conf b/elk_metrics_6x/templates/20-magnum.conf new file mode 100644 index 00000000..0b827b0a --- /dev/null +++ b/elk_metrics_6x/templates/20-magnum.conf @@ -0,0 +1,16 @@ +filter { + if "magnum" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "magnum.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/21-octavia.conf b/elk_metrics_6x/templates/21-octavia.conf new file mode 100644 index 00000000..e797f6e1 --- /dev/null +++ b/elk_metrics_6x/templates/21-octavia.conf @@ -0,0 +1,16 @@ +filter { + if "octavia" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "octavia.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/98-traceback.conf b/elk_metrics_6x/templates/98-traceback.conf new file mode 100644 index 00000000..102dd3b6 --- /dev/null +++ b/elk_metrics_6x/templates/98-traceback.conf @@ -0,0 +1,8 @@ +filter { + if "Traceback" in [message] { + mutate { + add_tag => ["traceback"] + remove_tag => ["_grokparsefailure"] + } + } +} diff --git a/elk_metrics_6x/templates/30-elasticsearch-output.conf.j2 b/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 similarity index 100% rename from elk_metrics_6x/templates/30-elasticsearch-output.conf.j2 rename to elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 diff --git a/elk_metrics_6x/vars/variables.yml b/elk_metrics_6x/vars/variables.yml index 46b8dca9..25b759e3 100644 --- a/elk_metrics_6x/vars/variables.yml +++ b/elk_metrics_6x/vars/variables.yml @@ -35,7 +35,7 @@ kibana_server_name: "{{ ansible_hostname }}" # logstash vars logstash_beat_input_port: 5044 - +logstash_deploy_filters: true # Beat options heartbeat_services: