logstash 6.3.2 配置与模板

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/wanglei_storage/article/details/82662862

服务端

logstash.conf

input {
    redis {
        host => "10.168.216.1"
        port => "6379"
        key => "redis_auth"
        data_type => "list"
        password => "2RMYFdlZNSSsqKYi"
    }
}

output {

############################# DEV ###########################
    # dev-chuiyi-site-landing-1
    if [type] == "dev-chuiyi-site-landing-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

############################# TEST  ############################
    # test-chuiyi-site-landing-1
    if [type] == "test-chuiyi-site-landing-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # test-chuiyi-site-landing-mcode-1
    if [type] == "test-chuiyi-site-landing-mcode-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # test-chuiyi-site-landing-request-1
    if [type] == "test-chuiyi-site-landing-request-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # test-chuiyi-site-landing-2
    if [type] == "test-chuiyi-site-landing-2" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # test-chuiyi-site-landing-mcode-2
    if [type] == "test-chuiyi-site-landing-mcode-2" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # test-chuiyi-site-landing-request-2
    if [type] == "test-chuiyi-site-landing-request-2" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # test-chuiyi-crm-1
    if [type] == "test-chuiyi-crm-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # test-chuiyi-crm-2
    if [type] == "test-chuiyi-crm-2" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }



############################# RELEASE ############################
    # release-chuiyi-site-landing-1
    if [type] == "release-chuiyi-site-landing-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # release-chuiyi-site-landing-mcode-1
    if [type] == "release-chuiyi-site-landing-mcode-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # release-chuiyi-site-landing-request-1
    if [type] == "release-chuiyi-site-landing-request-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # release-chuiyi-site-landing-2
    if [type] == "release-chuiyi-site-landing-2" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # release-chuiyi-site-landing-mcode-2
    if [type] == "release-chuiyi-site-landing-mcode-2" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # release-chuiyi-site-landing-request-2
    if [type] == "release-chuiyi-site-landing-request-2" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # release-chuiyi-crm-1
    if [type] == "release-chuiyi-crm-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

############################# PRODUCT ############################
    # product-chuiyi-site-landing-1
    if [type] == "product-chuiyi-site-landing-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # product-chuiyi-site-landing-mcode-1
    if [type] == "product-chuiyi-site-landing-mcode-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # product-chuiyi-site-landing-request-1
    if [type] == "product-chuiyi-site-landing-request-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }

    # product-chuiyi-crm-1
    if [type] == "product-chuiyi-crm-1" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash.json"
            template_name => "logstash.json"
            template_overwrite => true
        }
    }



#################################### nginx ############################
    if [type] == "product-chuiyi-site-nginx" {
        elasticsearch {
            hosts => ["10.24.254.146:9200", "10.24.155.230:9200", "10.24.153.128:9200"]
            index => "%{[type]}-%{+YYYY.MM.dd}"
            codec => "json"
            template => "/usr/local/logstash-6.3.2/config/logstash_web.json"
            template_name => "logstash_web.json"
            template_overwrite => true
        }
    }


}

logstash.json

{
  "template": "*",
  "order": 10,
  "settings": {
     "index.mapping.total_fields.limit": 1000,
     "index.mapping.depth.limit": 20,
     "index.mapping.nested_fields.limit": 50,
     "index": {
        "number_of_replicas": "1",
        "number_of_shards": "5",
        "refresh_interval": "5s"
      },
     "analysis": {
       "analyzer": {
         "default": {
           "tokenizer":"standard",
           "filter": ["standard"] }
       }
     }
   }
}

logstash_web.json

{
  "template": "*",
  "order": 10,

  "settings": {
    "index.mapping.total_fields.limit": 1000,
    "index.mapping.depth.limit": 20,
    "index.mapping.nested_fields.limit": 50,
    "index": {
      "number_of_replicas": "1",
      "number_of_shards": "5",
      "refresh_interval": "5s"
    }
  },

  "mappings" : {
    "_default_" : {
      "dynamic_templates" : [
      {
        "message_field" : {
          "path_match" : "message",
          "match_mapping_type" : "string",
          "mapping" : { "type" : "text", "norms" : false } }
      },
      {
        "string_fields" : {
          "match" : "*",
          "match_mapping_type" : "string",
          "mapping" : { "type" : "text", "norms" : false, "fields" : { "keyword" : { "type": "keyword", "ignore_above": 256 } } } }
      }],
      "properties" : {
        "@timestamp": {
          "type": "date" },
        "@version": {
          "type": "keyword" },
        "geoip": {
          "dynamic": true,
          "properties" : { "http_x_forwarded_for_geoip": { "type": "ip" }, "location" : { "type" : "geo_point" }, "latitude" : { "type" : "half_float" }, "longitude" : { "type" : "half_float" } } }
      }
    }
  }
}

客户端

logstash 收集nginx及业务日志

input {
    file {
        path => "/data/logs/project_logs/product-chuiyi-site-landing/laravel.log"
        type => "product-chuiyi-site-landing-1"
        start_position => "end"
    }

    file {
        path => "/data/logs/project_logs/product-chuiyi-site-landing/mcode.log"
        type => "product-chuiyi-site-landing-mcode-1"
        codec => json
        start_position => "end"
    }

    file {
        path => "/data/logs/project_logs/product-chuiyi-site-landing/request.log"
        type => "product-chuiyi-site-landing-request-1"
        codec => json
        start_position => "end"
    }

    file {
        path => "/data/logs/nginx_logs/access.log"
        type => "product-chuiyi-site-nginx"
        codec => json
        start_position => "end"
    }
}

filter {
    if [type] == "product-chuiyi-site-nginx"  {
        if [request] == "HEAD / HTTP/1.0" { drop {} }
        if [http_x_forwarded_for] == "120.55.75.236" { drop {} }

        mutate {
            gsub => [ "http_user_agent", "\\x", "\\\x" ]
            split => [ "http_x_forwarded_for", "," ]
        }

        geoip {
            database => "/usr/local/logstash-6.3.2/GeoLite2-City_20180807/GeoLite2-City.mmdb"
            source => "http_x_forwarded_for[0]"
            target => "geoip"
            add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
            add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
        }
        mutate {
            convert => [ "[geoip][coordinates]", "float"]
        }
    }
}

output {
    redis {
        host => "10.168.216.1"
        port => "6379"
        key => "redis_auth"
        data_type => "list"
        password => "2RMYFdlZNSSsqKYi"
    }
}

nginx log format

log_format main     '{"remote_addr": "$remote_addr",'
                    '"time_local": "$time_local",'
                    '"request": "$request",'
                    '"status": "$status",'
                    '"body_bytes_sent": "$body_bytes_sent",'
                    '"http_referer": "$http_referer",'
                    '"http_user_agent": "$http_user_agent",'
                    '"http_x_forwarded_for": "$http_x_forwarded_for",'
                    '"upstream_addr": "$upstream_addr",'
                    '"upstream_status": "$upstream_status",'
                    '"request_time": "$request_time",'
                    '"upstream_response_time": "$upstream_response_time"}';