filebeat get the nginx access log configuration

acquiring the access log filebeat nginx arranged 

to generate a log server nginx server configuration i.e. Manufacturer: 
Take omp.chinasoft.com Example: 
. 1 .nginx.conf main configuration file adding log format 

log_format log_json   ' { "REMOTE_ADDR": "$ REMOTE_ADDR", ' 
                          ' "REMOTE_USER": "$ REMOTE_USER", ' 
                          ' "time_local": "$ time_local", ' 
                          ' "request_url": "$ scheme: // $ Host $ REQUEST_URI", ' 
                          ' "Status": "$ Status" , ' 
                          ' "body_bytes_sent": "$ body_bytes_sent", ' 
                          ' "HTTP_REFERER": "$ HTTP_REFERER",'
                          '"http_user_agent": "$http_user_agent", '
                          '"http_x_forwarded_for": "$http_x_forwarded_for", '
                          '"request_time": "$request_time", '
                          '"timestamp": "$msec", '
                          '"request_method": "$request_method", '
                          '"root_id": "$root_id", '
                          '"trace_id": "$http_trace_id", '
                          '"span_id": "$http_span_id" '
                        '}';



2.vhosts虚拟主机配置

a.server section add tracking parameters IF
.. the SET root_id $ $ $ REMOTE_ADDR $ pid $ msec. . Connection $ connection_requests;;"omp.chinasoft.com"
the SET $ app_name
####### add link tracking parameters #########
(! $ http_root_id = "" ) 
{ 
    SET $ $ root_id http_root_id; 
} 
2 was added portion .location 
####### add tracking link parameter ######### 


fastcgi_param ROOT_ID $ root_id; 
fastcgi_param $ APP_NAME APP_NAME; 

 
. 3 to adjust the log path. 
the error_log / Data / WWW / logs / nginx_log / error / omp.chinasoft.com_error.log; 
access_log / Data / WWW / logs / nginx_log / Access / omp.chinasoft.com_access.log log_json; 


# specific configuration 
# CAT /usr/local/nginx/config/vhost.d/ omp.chinasoft.com 
Server { 
    the listen 80 ; 
    server_name omp.chinasoft.com; 
    return301 https://$server_name$request_uri;
}

server {
    listen 443 ssl;
    ssl_certificate /usr/local/nginx/config/cert2016/chinasoft_com.crt;
    ssl_certificate_key /usr/local/nginx/config/cert2016/chinasoft_com.key;
    ssl_session_cache shared:SSL1:20m;
    ssl_session_timeout 60m;
    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;

    server_name omp.chinasoft.com;
    index index.html index.php;
    root /var/www/vhosts/cloud_oa/public;

    client_max_body_size 800M;

    add_header Ws-From 'http/1.1 2.1.1.1 stable';

    set $app_name "omp.chinasoft.com";
    set $root_id $pid.$msec.$remote_addr.$connection.$connection_requests;
    if ($http_root_id != "")
    {
        set $root_id $http_root_id;
    }

    location / {
        try_files $uri $uri/ /index.php?$query_string;
    }

    location ~ /*\.php {
        fastcgi_param ROOT_ID $root_id;
        fastcgi_param APP_NAME $app_name;
        include fastcgi_params;
        fastcgi_param SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        fastcgi_param APP_ENV "production";
        fastcgi_pass unix:/tmp/php5.6-fpm.sock;
        fastcgi_index index.php;
    }

    location ~ .*\.(xml|gif|jpg|jpeg|png|bmp|swf|woff|woff2|ttf|js|css)$ {
        expires 30d;
    }

    #error_log /var/log/nginx/omp.chinasoft.com_error.log;
    #access_log /var/log/nginx/omp.chinasoft.com_access.log;
    error_log /data/www/logs/nginx_log/error/omp.chinasoft.com_error.log;
    access_log /data/www/logs/nginx_log/access/omp.chinasoft.com_access.log log_json;
}


3.filebeat的配置

# cat /usr/local/filebeat-7.3.1-linux-x86_64/modules.d/nginx.yml
# Module: nginx
# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.3/filebeat-module-nginx.html
filebeat.inputs:
# 采集nginx错误日志
- type: log
  enabled: true
  paths:
    - /data/www/logs/nginx_log/error/*_error.log
  fields:
    log_type: ngx_error
  fields_under_root: true
  tail_files: true
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: after

# 采集nginx访问日志
- type: log
  enabled: true
  paths:
    - /data/www/logs/nginx_log/access/*_access.log
  fields:
    log_type: ngx_access 
  fields_under_root: to true 
  tail_files: to true 
  exclude_lines: [ '. \ (xml | GIF | JPG | jpeg | PNG | BMP | SWF | WOFF | woff2 | ttf | JS | CSS | SVG | ico)'] 

# output 
output. Kafka used to live: 
  hosts: [ "1.1.1.1:19092"] 
  Topic: filebeats-log 

# filebeat start command 
./filebeat -c modules.d / nginx.yml 
/usr/local/filebeat-7.3.1-linux-x86_64/filebeat /usr/local/filebeat-7.3.1-linux-x86_64/modules.d/nginx.yml & -c 

: View kafka log details of the command 
path kafka configuration path, you can view the log storage: 
config / server.config 

/ usr / local / elk / kafka / bin / kafka-run-class.sh kafka.tools.DumpLogSegments --files /data/kafka/logs/filebeats-log-omp-0/00000000000000000000.log --print-data-log 


filebeat udp monitor configuration 
filebeat.inputs:

# 监听udp
- type: udp
  enabled: true
  host: "0.0.0.0:9999"

output.kafka:
  hosts: ["10.10.18.242:9092"]
  topic: filebeats-log

# 采集者
processors:
- drop_fields:
    fields: ["ecs",  "input", "agent", "@metadata", "tags"]