logstash获取nginx日志 两种方法

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/ywmack/article/details/83819058

获取nginx日志要写grok 还有很多正则来做

那么很多像我一样的新手不知道该如何操作

下面我们来个简单的

第一种 ：

重点是： 把nginx的access.log日志格式改成json类型

更重要的是下面两行

log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"request":"$request",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';
    access_log /data/nginx/logs/access_json.log json;

上面字体 颜色  一种颜色是一行

把这两行加到nginx.conf的http里面

如下代码：

http {
    include       mime.types;
    default_type  application/octet-stream;
    log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"request":"$request",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';
    access_log /data/nginx/logs/access_json.log json;
    server_names_hash_bucket_size  128;
    client_header_buffer_size   32K;
    large_client_header_buffers  4 32k;
--------------------------以下省略

重启nginx 则在/data/nginx/logs/看到access_json.log的日志文件

下面我们写logstash的配置

我们配置文件是输出到redis里面，如果是直接写到es里面。需要改动

input {
    file {
        path => ['/data/nginx/logs/access_json.log']
        start_position => "beginning"
        codec => "json"
        tags => ['user']
        type => "nginx"
    }
}
output {
    if [type] == "nginx" {
        redis {
            host => "172.17.0.90"
            port => "6379"
            key => "nginx"
            db => "10"
            data_type => "list"
        }
    }
}

上面的配置文件就不做多解释 其它文章里面会介绍到

接下来就可以操作kibana了加索引了。会看到更多的列了

第二种 是后面发现的

https://grafana.com/dashboards/2292

参考grafana.com的

这个比第一种的全面

定义日志类型

log_format main   '{"@timestamp":"$time_iso8601",'
                        '"@source":"$server_addr",'
                        '"hostname":"$hostname",'
                        '"ip":"$http_x_forwarded_for",'
                        '"client":"$remote_addr",'
                        '"request_method":"$request_method",'
                        '"scheme":"$scheme",'
                        '"domain":"$server_name",'
                        '"referer":"$http_referer",'
                        '"request":"$request_uri",'
                        '"args":"$args",'
                        '"size":$body_bytes_sent,'
                        '"status": $status,'
                        '"responsetime":$request_time,'
                        '"upstreamtime":"$upstream_response_time",'
                        '"upstreamaddr":"$upstream_addr",'
                        '"http_user_agent":"$http_user_agent",'
                        '"https":"$https"'
                        '}';

 logstash的配置文件

input {
    file {
#这里根据自己日志命名使用正则匹配所有域名访问日志
        #path => [ "/usr/local/nginx/logs/*_access.log" ]
        path => ['/data/nginx/logs/access_json.log']
        start_position => "beginning"
        codec => "json"
        tags => ['user']
        type => "nginx"
    }
}
filter {
    mutate {
      convert => [ "status","integer" ]
      convert => [ "size","integer" ]
      convert => [ "upstreatime","float" ]
      remove_field => "message"
    }
    geoip {
        source => "ip"
    }


}
output {
    if [type] == "nginx" {
        redis {
            host => "172.17.0.90"
            port => "6379"
            key => "nginx"
            db => "10"
            data_type => "list"
        }
    }
}