nginx-es.conf
input { file { path => "/opt/logtest/nginx_access.log.1" start_position => "beginning" sincedb_path => "/opt/logstash-2.3.4/sincedb/" } }
filter { ruby{ init => "@kname=['remote_addr','time_local','http_host','request','status','body_bytes_sent','http_referer','http_user_agent','upstream_response_time','request_time']" code => "event.append(Hash[@kname.zip(event['message'].split(' | '))])" } if [http_user_agent] =="-" { drop { } } if [request] { ruby { init => "@kname = ['method','uri','verb']" code => "event.append(Hash[@kname.zip(event['request'].split(' '))])" } if [uri] { ruby { init => "@kname = ['url_path', 'url_args']" code => "event.append(Hash[@kname.zip(event['uri'].split('?'))])" } } } geoip { source => "remote_addr" } mutate { convert => [ "body_bytes_sent", "integer", "status", "integer", "upstream_response_time", "float", "request_time", "float" ] add_field => {"type" => "nginx"} remove_field => ["uri","request"] } date { match => ["time_local", "dd/MMM/yyyy:HH:mm:ss +0800", "ISO8601"] target => "@timestamp" remove_field => ["time_local", "message"] } }
output { stdout{ codec => rubydebug} elasticsearch { hosts => ["192.168.0.135:9200"] index => "logstash-%{type}-%{+YYYY.MM.dd}" workers => 2 template_overwrite => true } }
log content sample
192.168.100.10 | 29/Jul/2016:17:19:36 +0800 | api2.unichat.cn | GET / HTTP/1.1 | 200 | 17 | - | - | 0.052 | 0.052 118.212.135.122 | 29/Jul/2016:17:19:44 +0800 | api2.unichat.cn | GET /api/uchat?page=2&recommend_sort=1469783728640 HTTP/1.1 | 200 | 3499 | - | PhotoFlow/1.0.9 (iPhone; iOS 9.2.1; Scale/2.00) | 0.200 | 0.201
log_format uchat "$remote_addr | $time_local | $http_host | $request | $status | $body_bytes_sent | $http_referer | $http_user_agent | $upstream_response_time | $request_time";
output
{ "@version" => "1", "@timestamp" => "2016-08-03T02:18:21.000Z", "path" => "/opt/logtest/nginx_access.log", "host" => "inok-c0", "remote_addr" => "125.71.215.46", "http_host" => "api2.unichat.cn", "status" => 200, "body_bytes_sent" => 17, "http_referer" => "-", "http_user_agent" => "check_http/v1.4.16 (nagios-plugins 1.4.16)", "upstream_response_time" => 0.042, "request_time" => 0.043, "method" => "GET", "verb" => "HTTP/1.1", "url_path" => "/", "url_args" => nil, "geoip" => { "ip" => "125.71.215.46", "country_code2" => "CN", "country_code3" => "CHN", "country_name" => "China", "continent_code" => "AS", "region_name" => "32", "city_name" => "Chengdu", "latitude" => 30.66669999999999, "longitude" => 104.06670000000003, "timezone" => "Asia/Chongqing", "real_region_name" => "Sichuan", "location" => [ [0] 104.06670000000003, [1] 30.66669999999999 ] }, "type" => "nginx" }