Logstash filter{} 插件 grok，收集apache访问日志

Logstash filter{} 插件 grok,收集apache访问日志

1. grok非常影响性能
2. 不够灵活，除非很懂ruby

标准输入标准输出执行

[root@node1 conf.d]# cat grok.conf
input{
    stdin {}
}

filter{
    grok{
    match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request}%{NUMBER:bytes} %{NUMBER:duration}" }
    }
}

output{
    stdout{
    codec => rubydebug
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f grok.conf

• 演示结果

[root@node1 conf.d]# /opt/logstash/bin/logstash -f grok.conf
Settings: Default pipeline workers: 2
Pipeline main started
192.168.79.103 GET /index.html 15000 0.041
{
       "message" => "192.168.79.103 GET /index.html 15000 0.041",
      "@version" => "1",
    "@timestamp" => "2018-08-24T07:55:10.386Z",
          "host" => "node1",
        "client" => "192.168.79.103",
        "method" => "GET",
       "request" => "/index.html",
         "bytes" => "15000",
      "duration" => "0.041"
}

标准输出，收集apache日志，使用grok插件

[root@node1 conf.d]# cat apache_grok.conf
input{
    file{
    path => "/var/log/httpd/access_log"
    start_position => "beginning"
    type => "apache_log"
    }
}

filter{
    grok {
    match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
}

output{
    stdout {
        codec => rubydebug
    }
}

• 实验结果

[root@node1 conf.d]# /opt/logstash/bin/logstash -f apache_grok.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
        "message" => "192.168.79.1 - - [24/Aug/2018:16:14:59 +0800] \"GET /noindex/css/fonts/Bold/OpenSans-Bold.ttf HTTP/1.1\" 404 238 \"http://192.168.79.103/noindex/css/open-sans.css\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36\"",
       "@version" => "1",
     "@timestamp" => "2018-08-24T08:14:59.771Z",
           "path" => "/var/log/httpd/access_log",
           "host" => "node1",
           "type" => "apache_log",
       "clientip" => "192.168.79.1",
          "ident" => "-",
           "auth" => "-",
      "timestamp" => "24/Aug/2018:16:14:59 +0800",
           "verb" => "GET",
        "request" => "/noindex/css/fonts/Bold/OpenSans-Bold.ttf",
    "httpversion" => "1.1",
       "response" => "404",
          "bytes" => "238",
       "referrer" => "\"http://192.168.79.103/noindex/css/open-sans.css\"",
          "agent" => "\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36\""
}

收集至es

[root@node1 conf.d]# cat apache_grok.conf
input{
    file{
    path => "/var/log/httpd/access_log"
    start_position => "beginning"
    type => "apache_log"
    }
}

filter{
    grok {
    match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
}

output{
    elasticsearch{
    hosts => ["192.168.79.103:9200"]
    index => "apache_accesslog-%{+YYYY.MM.dd}"
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f apache_grok.conf