diff --git a/elk_metrics_6x/installLogstash.yml b/elk_metrics_6x/installLogstash.yml index 509a4815..07aba9d7 100644 --- a/elk_metrics_6x/installLogstash.yml +++ b/elk_metrics_6x/installLogstash.yml @@ -86,6 +86,51 @@ tags: - config + - name: Logstash Extra Patterns + template: + src: "{{ item }}" + dest: "/opt/logstash/patterns/{{ item }}" + owner: "logstash" + group: "logstash" + with_items: + - extras + when: + - logstash_deploy_filters + tags: + - logstash-filters + - config + + - name: Deploy Logstash configuration files + template: + src: "{{ item }}" + dest: "/etc/logstash/conf.d/{{ item }}" + with_items: + - 02-general.conf + - 03-nova.conf + - 04-neutron.conf + - 05-glance.conf + - 06-cinder.conf + - 07-libvirt.conf + - 08-apache.conf + - 09-heat.conf + - 10-mysql.conf + - 11-auth.conf + - 12-logstash.conf + - 13-swift.conf + - 14-keystone.conf + - 16-elasticsearch.conf + - 17-rabbitmq.conf + - 18-ceph.conf + - 19-nginx.conf + - 20-magnum.conf + - 21-octavia.conf + - 98-traceback.conf + when: + - logstash_deploy_filters + tags: + - logstash-filters + - config + - name: Ensure logstash ownership file: path: /var/lib/logstash diff --git a/elk_metrics_6x/templates/02-general.conf b/elk_metrics_6x/templates/02-general.conf new file mode 100644 index 00000000..16bba6bf --- /dev/null +++ b/elk_metrics_6x/templates/02-general.conf @@ -0,0 +1,16 @@ +filter { + if "oslofmt" in [tags] or "openstack" in [tags] { + if "Can not find policy directory: policy.d" in [message] { + drop{} + } + grok { + match => { + "message" => [ + "^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?", + "^%{CISCOTIMESTAMP:journalddate}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{GREEDYDATA:logmessage}" + ] + } + add_field => { "received_at" => "%{@timestamp}" } + } + } +} diff --git a/elk_metrics_6x/templates/03-nova.conf b/elk_metrics_6x/templates/03-nova.conf new file mode 100644 index 00000000..c020994c --- /dev/null +++ b/elk_metrics_6x/templates/03-nova.conf @@ -0,0 +1,23 @@ +filter { + if "nova" in [tags] { + mutate { + gsub => ['logmessage',"\"",""] + } + if [module] == "nova.osapi_compute.wsgi.server" { + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes:int} time\: %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } else if [module] == "nova.api.ec2" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NUMBER:seconds}s %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} None\:None %{NUMBER:response} %{GREEDYDATA:user_agent}" } + add_tag => ["apimetrics"] + } + } else if [module] == "nova.metadata.wsgi.server" { + grok { + match => { "logmessage" => "\[%{GREEDYDATA:requestid}\] %{NOTSPACE:requesterip} %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} status\: %{NUMBER:response} len\: %{NUMBER:bytes} time\: %{NUMBER:seconds}" } + add_tag => ["apimetrics"] + } + } + } +} diff --git a/elk_metrics_6x/templates/04-neutron.conf b/elk_metrics_6x/templates/04-neutron.conf new file mode 100644 index 00000000..13acfdfc --- /dev/null +++ b/elk_metrics_6x/templates/04-neutron.conf @@ -0,0 +1,28 @@ +filter { + if "neutron" in [tags] { + if [module] == "neutron.wsgi" { + if "accepted" not in [logmessage] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } + } else if "neutron-ha-tool" in [source] { + mutate { + add_tag => ["neutron-ha-tool"] + remove_tag => ["_grokparsefailure"] + } + } + if "starting" in [message] and "_grokparsefailure" in [tags] { + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid}|\-)\](%{SPACE}\(%{NUMBER:pid}\)) %{GREEDYDATA:servicemessage}" } + } + mutate { + remove_tag => ["_grokparsefailure"] + } + } + } +} diff --git a/elk_metrics_6x/templates/05-glance.conf b/elk_metrics_6x/templates/05-glance.conf new file mode 100644 index 00000000..2eaec5dd --- /dev/null +++ b/elk_metrics_6x/templates/05-glance.conf @@ -0,0 +1,16 @@ +filter { + if "glance" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "glance.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/06-cinder.conf b/elk_metrics_6x/templates/06-cinder.conf new file mode 100644 index 00000000..09c62fe9 --- /dev/null +++ b/elk_metrics_6x/templates/06-cinder.conf @@ -0,0 +1,18 @@ +filter { + if "cinder" in [tags] { + if [module] == "cinder.eventlet.wsgi.server" { + if "accepted" not in [logmessage] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + } + mutate { + replace => { "module" => "cinder.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/07-libvirt.conf b/elk_metrics_6x/templates/07-libvirt.conf new file mode 100644 index 00000000..c417c155 --- /dev/null +++ b/elk_metrics_6x/templates/07-libvirt.conf @@ -0,0 +1,11 @@ +filter { + if "libvirt" in [tags] { + grok { + match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" } + add_field => { "received_at" => "%{@timestamp}"} + } + mutate { + uppercase => [ "loglevel" ] + } + } +} diff --git a/elk_metrics_6x/templates/08-apache.conf b/elk_metrics_6x/templates/08-apache.conf new file mode 100644 index 00000000..ba1ad58f --- /dev/null +++ b/elk_metrics_6x/templates/08-apache.conf @@ -0,0 +1,29 @@ +filter { + if "horizon" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { + "message" => [ + "%{COMMONAPACHELOG}", + "\[%{APACHE_ERROR_TIMESTAMP:timestamp}\] \[%{DATA:module}:%{DATA:loglevel}\] \[pid %{POSINT:apache_pid}\:tid %{POSINT:apache_tid}\] ?(?:\[client %{IP:clientip}:%{POSINT:clientport}\] )?%{GREEDYDATA:logmessage}", + "%{SYSLOGTIMESTAMP:timestamp}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{PROG:prog}%{SPACE}%{IP:clientip}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NUMBER}%{SPACE}%{QS}%{SPACE}%{QS}" + ] + } + } + + if ![loglevel] { + mutate { + add_field => { "logmessage" => "%{request}" } + add_field => { "module" => "horizon.access" } + add_field => { "loglevel" => "INFO" } + add_tag => [ "apache-access" ] + } + } else { + mutate { + replace => { "module" => "horizon.error.%{module}" } + add_tag => [ "apache-error" ] + uppercase => [ "loglevel" ] + } + } + } +} diff --git a/elk_metrics_6x/templates/09-heat.conf b/elk_metrics_6x/templates/09-heat.conf new file mode 100644 index 00000000..8c22e39a --- /dev/null +++ b/elk_metrics_6x/templates/09-heat.conf @@ -0,0 +1,23 @@ +filter { + if "heat" in [tags] { + if [module] == "eventlet.wsgi.server" { + if "accepted" not in [logmessage] { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE}\] %{NOTSPACE:requesterip} %{NOTSPACE} %{NOTSPACE} \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes} %{BASE10NUM:httptime}" } + add_tag => ["apimetrics"] + } + } + mutate { + replace => { "module" => "heat.%{module}" } + } + } else if [module] == "heat.engine.service" { + grok { + match => { "logmessage" => "\[%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} %{NOTSPACE} %{NOTSPACE} %{NOTSPACE} %{GREEDYDATA:servicemessage}" } + add_tag => ["apimetrics"] + } + } + } +} diff --git a/elk_metrics_6x/templates/10-mysql.conf b/elk_metrics_6x/templates/10-mysql.conf new file mode 100644 index 00000000..861b240b --- /dev/null +++ b/elk_metrics_6x/templates/10-mysql.conf @@ -0,0 +1,33 @@ +filter { + if "mysql" in [tags] { + grok { + match => { "message" => "# User@Host: %{WORD:user}\[%{WORD}\] @ (%{HOSTNAME:client_hostname}|) \[(%{IP:client_ip}|)\]" } + } + + grok { + match => { "message" => "# Thread_id: %{NUMBER:thread_id:int} \s*Schema: (%{WORD:schema}| ) \s*QC_hit: %{WORD:qc_hit}" } + } + + grok { + match => { "message" => "# Query_time: %{NUMBER:query_time:float} \s*Lock_time: %{NUMBER:lock_time:float} \s*Rows_sent: %{NUMBER:rows_sent:int} \s*Rows_examined: %{NUMBER:rows_examined:int}" } + } + + grok { + match => { "message" => "(?m)SET timestamp=%{NUMBER:timestamp};%{GREEDYDATA:logmessage}" } + } + + date { + match => [ "timestamp", "UNIX" ] + } + + mutate { + remove_field => "timestamp" + } + + mutate { + gsub => [ "logmessage", "^\n", "" ] + add_field => { "module" => "mysql" } + add_field => { "loglevel" => "WARNING" } + } + } +} diff --git a/elk_metrics_6x/templates/11-auth.conf b/elk_metrics_6x/templates/11-auth.conf new file mode 100644 index 00000000..d9d1ee45 --- /dev/null +++ b/elk_metrics_6x/templates/11-auth.conf @@ -0,0 +1,10 @@ +filter { + if "auth" in [tags] { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" } + } + mutate { + add_field => { "module" => "auth" } + } + } +} diff --git a/elk_metrics_6x/templates/12-logstash.conf b/elk_metrics_6x/templates/12-logstash.conf new file mode 100644 index 00000000..e8ff8d3e --- /dev/null +++ b/elk_metrics_6x/templates/12-logstash.conf @@ -0,0 +1,25 @@ +filter { + if "logstash" in [tags] { + grok { + match => { + "message" => "\{\:timestamp=>\"%{TIMESTAMP_ISO8601:timestamp}\", \:message=>\"%{DATA:logmessage}\"(;|)(, \:address=>\"%{URIHOST:address}\", \:exception=>#<%{DATA:exception}>, \:backtrace=>\[%{DATA:backtrace}\]|)(, \:level=>:%{LOGLEVEL:loglevel}|)\}" + } + } + + mutate { + add_field => { "module" => "logstash" } + uppercase => [ "loglevel" ] + } + + if [loglevel] == "WARN" { + mutate { + replace => { "loglevel" => "WARNING" } + } + } else if ![loglevel] { + mutate { + add_field => { "loglevel" => "ERROR" } + } + } + + } +} diff --git a/elk_metrics_6x/templates/13-swift.conf b/elk_metrics_6x/templates/13-swift.conf new file mode 100644 index 00000000..aaa577d5 --- /dev/null +++ b/elk_metrics_6x/templates/13-swift.conf @@ -0,0 +1,56 @@ +filter { + if "swift-container" in [tags] { + grok { + match => { + "message" => "%{CISCOTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{CISCOTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{USER}%{SPACE}%{USERNAME}%{SPACE}%{NOTSPACE}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{HTTPDUSER}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{INT}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SECOND}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}" + } + } + } + if "swift-account" in [tags] { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP}%{SPACE}%{HOSTNAME}%{SPACE}%{PROG}%{SPACE}%{SYSLOGTIMESTAMP}%{SPACE}%{S3_REQUEST_LINE}%{SPACE}%{IP}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE}%{SPACE}%{SYSLOG5424SD}%{SPACE}%{QS}%{SPACE}%{POSINT}%{SPACE}%{NOTSPACE}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{QS}%{SPACE}%{SECOND}%{SPACE}%{QS}%{SPACE}%{NUMBER}%{SPACE}%{NOTSPACE}" + } + } + } + if "swift" in [tags] { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{NOTSPACE:logsource} %{SYSLOGPROG:module}: (?:%{SPACE})?%{GREEDYDATA:logmessage}" + } + } + + grok { + patterns_dir => ['/opt/logstash/patterns'] + match => { + "logmessage" => [ + "%{COMBINEDAPACHELOG}", + "%{SWIFTPROXY_ACCESS}", + "%{GREEDYDATA:logmessage} \(txn\: %{DATA:swift_txn}\)" + ] + } + tag_on_failure => [] + overwrite => [ "logmessage" ] + } + + if [request] { + mutate { + replace => { "logmessage" => "%{request}" } + } + } + + mutate { + replace => { "module" => "swift.%{module}" } + } + + if [file] =~ "error.log$" { + mutate { + add_field => { "loglevel" => "NOTICE" } + } + } else { + mutate { + add_field => { "loglevel" => "INFO" } + } + } + } +} diff --git a/elk_metrics_6x/templates/14-keystone.conf b/elk_metrics_6x/templates/14-keystone.conf new file mode 100644 index 00000000..e8cc6eca --- /dev/null +++ b/elk_metrics_6x/templates/14-keystone.conf @@ -0,0 +1,28 @@ +filter { + if "keystone-access" in [tags] { + grok { + match => { "message" => "%{CISCOTIMESTAMP:keystone_access_timestamp}%{SPACE}%{SYSLOGHOST:log_host}%{SPACE}%{SYSLOGPROG:prog}%{SPACE}%{TIMESTAMP_ISO8601:keystone_timestmp}%{SPACE}%{NUMBER:pid}%{SPACE}%{NOTSPACE:loglevel}%{SPACE}%{NOTSPACE:module}%{SPACE}%{SYSLOG5424SD:requestid}%{SPACE}%{WORD:verb}%{SPACE}%{NOTSPACE:request}" } + } + } + if "keystone" in [tags] { + if "apache-access" in [tags] { + grok { + match => { "message" => "%{COMMONAPACHELOG}" } + } + mutate { + add_field => { "logmessage" => "%{request}" } + add_field => { "module" => "keystone.access" } + add_field => { "loglevel" => "INFO" } + } + } else if "apache-error" in [tags] { + grok { + patterns_dir => ["/opt/logstash/patterns"] + match => { "message" => "%{KEYSTONE_SUBSECOND_TIMESTAMP:keystone_subsecond_timestamp} %{STANDARD_TIMESTAMP:standard_timestamp} %{NUMBER:pid} %{DATA:loglevel} %{DATA:module} \[%{DATA:requestid}\] %{WORD:verb} %{NOTSPACE:request}" } + } + mutate { + replace => { "module" => "keystone.error.%{module}" } + uppercase => [ "loglevel" ] + } + } + } +} diff --git a/elk_metrics_6x/templates/16-elasticsearch.conf b/elk_metrics_6x/templates/16-elasticsearch.conf new file mode 100644 index 00000000..8ca5890d --- /dev/null +++ b/elk_metrics_6x/templates/16-elasticsearch.conf @@ -0,0 +1,10 @@ +filter { + if "elasticsearch" in [tags] { + grok { + match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{LOGLEVEL:loglevel}\s*\]\[%{NOTSPACE:module}\s*\] %{GREEDYDATA:logmessage}" } + } + mutate { + replace => { "module" => "elasticsearch.%{module}" } + } + } +} diff --git a/elk_metrics_6x/templates/17-rabbitmq.conf b/elk_metrics_6x/templates/17-rabbitmq.conf new file mode 100644 index 00000000..c46f2551 --- /dev/null +++ b/elk_metrics_6x/templates/17-rabbitmq.conf @@ -0,0 +1,20 @@ +filter { + if "rabbitmq" in [tags] { + if [message] == "" { + drop { } + } + grok { + match => { "message" => "^\=%{LOGLEVEL:loglevel} REPORT\=\=\=\= %{MONTHDAY:event_day}\-%{MONTH:event_month}\-%{YEAR:event_year}\:\:%{TIME:event_time} \=\=\=\n%{GREEDYDATA:logmessage}" } + } + + mutate { + replace => { "module" => "rabbitmq" } + add_field => { "timestamp" => "%{event_day} %{event_month} %{event_year} %{event_time}" } + } + + date { + match => [ "timestamp", "dd MMM YYYY HH:mm:ss" ] + remove_field => [ "event_day", "event_month", "event_year", "event_time", "timestamp" ] + } + } +} diff --git a/elk_metrics_6x/templates/18-ceph.conf b/elk_metrics_6x/templates/18-ceph.conf new file mode 100644 index 00000000..5eb0d700 --- /dev/null +++ b/elk_metrics_6x/templates/18-ceph.conf @@ -0,0 +1,12 @@ +filter { + if "ceph" in [tags] { + grok { + match => { "message" => "%{TIMESTAMP_ISO8601:date} %{NOTSPACE:osd_epoch} ?%{SPACE}?%{NOTSPACE:error_bool} %{GREEDYDATA:logmessage}" } + } + } + if "ceph-osd" in [tags] { + grok { + match => { "message" => "-- (?(%{IPORHOST}\:%{POSINT}/%{POSINT})) (?:[<|>]){1,2} (?(%{IPORHOST}\:%{POSINT}/%{POSINT}))" } + } + } +} diff --git a/elk_metrics_6x/templates/19-nginx.conf b/elk_metrics_6x/templates/19-nginx.conf new file mode 100644 index 00000000..0c3f5ba9 --- /dev/null +++ b/elk_metrics_6x/templates/19-nginx.conf @@ -0,0 +1,20 @@ +filter { + if "nginx" in [tags] { + if "nginx-access" in [tags] { + grok { + patterns_dir => ['/opt/logstash/patterns'] + match => { + "message" => "%{IP:client_ip} - %{USER:client_user} \[%{NGINX_TIMESTAMP:timestamp}\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{INT:response_code} %{INT:bytes} %{QUOTEDSTRING:referer} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:gzip_ratio}" + } + } + } + if "nginx-error" in [tags] { + grok { + patterns_dir => ['/opt/logstash/patterns'] + match => { + "message" => "%{NGINX_ERROR_TIMESTAMP:timestamp} \[%{LOGLEVEL:loglevel}\] %{GREEDYDATA:error_msg}" + } + } + } + } +} diff --git a/elk_metrics_6x/templates/20-magnum.conf b/elk_metrics_6x/templates/20-magnum.conf new file mode 100644 index 00000000..0b827b0a --- /dev/null +++ b/elk_metrics_6x/templates/20-magnum.conf @@ -0,0 +1,16 @@ +filter { + if "magnum" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "magnum.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/21-octavia.conf b/elk_metrics_6x/templates/21-octavia.conf new file mode 100644 index 00000000..e797f6e1 --- /dev/null +++ b/elk_metrics_6x/templates/21-octavia.conf @@ -0,0 +1,16 @@ +filter { + if "octavia" in [tags] { + if [module] == "eventlet.wsgi.server" { + mutate { + gsub => ['logmessage',"\"",""] + } + grok { + match => { "logmessage" => "\[(%{NOTSPACE:requestid} %{NOTSPACE:user_id} %{NOTSPACE:tenant} \- \- \-|\-)\] %{NOTSPACE:requesterip} \- \- \[%{NOTSPACE:req_date} %{NOTSPACE:req_time}\] %{NOTSPACE:verb} %{NOTSPACE:url_path} %{NOTSPACE:http_ver} %{NUMBER:response} %{NUMBER:bytes:int} %{BASE10NUM:httptime:float}" } + add_tag => ["apimetrics"] + } + mutate { + replace => { "module" => "octavia.%{module}" } + } + } + } +} diff --git a/elk_metrics_6x/templates/98-traceback.conf b/elk_metrics_6x/templates/98-traceback.conf new file mode 100644 index 00000000..102dd3b6 --- /dev/null +++ b/elk_metrics_6x/templates/98-traceback.conf @@ -0,0 +1,8 @@ +filter { + if "Traceback" in [message] { + mutate { + add_tag => ["traceback"] + remove_tag => ["_grokparsefailure"] + } + } +} diff --git a/elk_metrics_6x/templates/30-elasticsearch-output.conf.j2 b/elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 similarity index 100% rename from elk_metrics_6x/templates/30-elasticsearch-output.conf.j2 rename to elk_metrics_6x/templates/99-elasticsearch-output.conf.j2 diff --git a/elk_metrics_6x/vars/variables.yml b/elk_metrics_6x/vars/variables.yml index 46b8dca9..25b759e3 100644 --- a/elk_metrics_6x/vars/variables.yml +++ b/elk_metrics_6x/vars/variables.yml @@ -35,7 +35,7 @@ kibana_server_name: "{{ ansible_hostname }}" # logstash vars logstash_beat_input_port: 5044 - +logstash_deploy_filters: true # Beat options heartbeat_services: