input{ file{ path => "/tmp/access.1w.log" sincedb_path => "/dev/null" start_position => "beginning" } } filter{ if [@metadata][debug] { mutate{ remove_field => ["headers"] } } grok{ match => { "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:[@metadata][timestamp]}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NOTSPACE:response_status_code} (?:%{NUMBER:bytes}|-) %{QS:hostname} %{QS:referrer} (?:-|%{DATA:params}) %{QS:agent} %{QS:xforwardedfor} (?:-|%{MY_URI:upstream_host}) (?:-|%{MY_RESP:upstream_response_status_code}) (?:-|%{MY_RESP_TIME:upstream_response_time}) %{BASE10NUM:response_time:float}' } pattern_definitions=>{ "MY_URI" => '%{URIHOST}(, %{URIHOST})*' "MY_RESP" => '%{NUMBER}(, %{NUMBER})*' "MY_RESP_TIME" => '%{BASE10NUM}(, %{BASE10NUM})*' } } date{ match => ["[@metadata][timestamp]","dd/MMM/yyyy:HH:mm:ss Z"] } mutate{ split => {"upstream_host" => ", "} split => {"upstream_response_status_code" => ", "} split => {"upstream_response_time" => ", "} gsub => ["hostname",'"',''] } mutate{ convert => {"upstream_response_time"=>"float"} } geoip{ source => "clientip" } useragent{ source => "agent" target => "useragent" } mutate{ add_field => { "[@metadata][index]" => "nginx_logs_%{+YYYY.MM.dd}" } } if [referrer] =~ /^"http/ { grok{ match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' } } if "imooc.com" in [referrer_host] { grok{ match => { "referrer" => ['%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:imooc_type}/%{NOTSPACE:imooc_res_id})?"','%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:imooc_type})?"'] } } } } mutate{ gsub => ["referrer",'"',''] } if "_grokparsefailure" in [tags] { mutate{ replace => { "[@metadata][index]" => "nginx_logs_parsefailure_%{+YYYY.MM.dd}" } } }else{ mutate{remove_field=>["message"]} } } output{ if [@metadata][debug]{ stdout{ codec => rubydebug{ metadata => true } } } else { stdout{codec=>dots} elasticsearch{ index => "%{[@metadata][index]}" } } }