【ELK】Logstash配置小记

Filter

grok：进行正则匹配

grok {
    match => [ "message" , "%{NGINXLOG} %{QS:upstream_addr} %{QS:upstream_response_time} %{QS:request_time} %{QS:body}"]
}

# 自定义正则匹配
grok {
    break_on_match => "false"
    patterns_dir => "./logstash/pat"
    match => [ "request" , "%{URIMATCH}" ]
}

ruby：使用ruby代码调整输出内容

ruby {
    code => "event.set('timelocal', event.get('@timestamp').time.localtime + 8*60*60)"
}
ruby {
    code => "event.set('@timestamp',event.get('timelocal'))"
}
ruby {
    code => "event.set('timeunix',event.get('@timestamp').to_i)"
}

kv：进行基于单一分隔符的键值匹配

kv {
    source => "message"
    field_split => "#"
    value_split => "~"
    target => "msg_json"

}

date：时间匹配到·timestamp

date {
    match => ["timestamp", "UNIX"]
    remove_field => ["message","timestamp", "count", "fields", "input_type", "offset"]
}

mutate：调整输出

mutate {
    lowercase => [ "loglevel" ]
    remove_field => [ "ident", "auth", "source", "beat", "message", "@version", "count", "fields", "input_type", "offset" ]
}

Output

if "_grokparsefailure" not in [tags]

过滤掉匹配失败的日志

Pattern

例如
LOGHEAD1 ^[
LOGHEAD2 ]
LOGHEAD [A-Z]{4,5}

LOGMSG %{LOGHEAD1}%{LOGHEAD:loghead}%{LOGHEAD2}

待续