logstash filter && output 简介

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/Gekkoou/article/details/80986994

input 详解参考之前的文章
https://blog.csdn.net/gekkoou/article/details/80986017

input 官方详解
https://www.elastic.co/guide/en/logstash/current/input-plugins.html

filter 官方详解
https://www.elastic.co/guide/en/logstash/current/filter-plugins.html

output 官方详解
https://www.elastic.co/guide/en/logstash/current/output-plugins.html

---

贴上收集 apache 日志的代码做简介

input {
    # access日志
    file {
        type => "apache_access"
        tag => "apache_access"
        path => ["/var/log/apache/access.log"]
        start_position => beginning
    }
    # error日志
    file {
        type => "apache_error"
        tag => "apache_error"
        path => ["/var/log/apache/error.log"]
        start_position => beginning
    }
}

filter {
    # 根据 input 添加的 type 来区分, 实现同时读取两种日志, 也可以用 tag 来区分 (例如 if [tag] in "apache_access")
    if [type] == "apache_access"{
        # 文本片段切分的方式来切分日志事件
        # 推荐使用grokdebugger来写匹配模式: http://grokdebug.herokuapp.com/
        # grok官方详解: https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
        grok {
            match => { "message" => "%{COMBINEDAPACHELOG}"}
        }
        # data插件可以用来转换你的日志记录中的时间字符串, 然后转存到 @timestamp 字段里
        date {
            match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        # 通过geoip能获取到很多的信息，包括经纬度，国家，城市，地区等信息
        geoip {
            # 来源于apache日志中的clientip
            source => "clientip"
        }
        # useragent插件可以帮助我们过滤出浏览器版本、型号以及系统版本
        useragent {
            source => "agent"
            target => "useragent"
        }
    } else if [type] == "apache_error"{
        grok {
            match => { "message" => "\[(?<mytimestamp>%{DAY:day} %{MONTH:month} %{MONTHDAY} %{TIME} %{YEAR})\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{NUMBER:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}" }
        }
        date {
            match => [ "mytimestamp" , "EEE MMM dd HH:mm:ss.SSSSSS yyyy" ]
        }
    }

    #转换类型 (integer, float, integer_eu, float_eu, string, boolean)
    #mutate {
    #    convert => ["ctime", "integer"]
    #    convert => ["lat", "float"]
    #}

    #当某条日志信息符合if规则时
    #if [field_name] == "value" {
    #    #drop可以跳过某些不想统计的日志信息
    #    drop {}
    #}

    #create_at为时间戳时需要转换为0时区(UTC), 然后放入@timestamp字段里
    #date {
    #   match => ["create_at", "yyyy-MM-dd HH:mm:ss,SSS", "UNIX"]
    #   #match => ["create_at", "UNIX"]
    #   target => "@timestamp"
    #   locale => "cn"
    #   #remove_field => 'create_at' #删除字段
    #}

    # 执行ruby代码
    #ruby {
    #    code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
    #}
    #ruby {
    #    code => "event.set('@timestamp',event.get('timestamp'))"
    #}
}

# 输出插件将数据发送到一个特定的目的地, 除了elasticsearch还有好多可输出的地方, 例如file, csv, mongodb, redis, syslog等
output {
    if [type] == "apache_access"{
        elasticsearch {
            hosts => [ "localhost:9200" ]
            # 记录的index索引名称格式
            index => "apache-access-log-%{+YYYY.MM}"
        }
    } else if [type] == "apache_error"{
        elasticsearch {
            hosts => [ "localhost:9200" ]
            index => "apache-error-log"
        }
    }
}

---