ELK日志处理详解
[root@kibana ~]# cat /var/log/messages | tail
[root@kibana ~]# vim /etc/rsyslog.conf
local0.info /var/log/info.log
[root@kibana ~]# systemctl restart rsyslog
[root@kibana ~]# cd /var/log/
[root@kibana log]# ls info.log
Info.log
[root@kibana log]# man logger
EXAMPLES
logger System rebooted
logger -p local0.notice -t HOSTIDM -f /dev/idmc
logger -n loghost.example.com System rebooted
[root@kibana log]# logger -p local0.notice -t "testlog" "a b c d"
[root@kibana log]# cat info.log
Jul 30 17:01:57 kibana testlog: a b c d
[root@kibana log]# vim /etc/rsyslog.conf
#*.* @@remote-host:514
local0.info @@192.168.6.16:514
[root@kibana log]# systemctl restart rsyslog
[root@kibana log]# logger -p local0.notice -t "testlog" "hello world"
[root@kibana log]# cat info.log
Jul 31 11:38:32 kibana testlog: hello world
[root@logstash logstash]# cat logstash.conf
input{
stdin{ codec => "json" }
file {
path => [ "/tmp/a.log", "/tmp/b.log" ]
sincedb_path => "/var/lib/logstash/sincedb.log"
start_position => "beginning"
type => "filelog"
}
tcp {
mode => "server"
host => "0.0.0.0"
port => 8888
type => "tcplog"
}
udp {
port => 9999
type => "udplog"
}
syslog {
host => "0.0.0.0"
port => 514
type => "syslog"
}
}
filter{
}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "a b c d",
"@version" => "1",
"@timestamp" => "2018-07-31T03:34:14.000Z",
"type" => "syslog",
"host" => "192.168.6.10",
"priority" => 133,
"timestamp" => "Jul 31 11:34:14",
"logsource" => "kibana",
"program" => "testlog",
"severity" => 5,
"facility" => 16,
"facility_label" => "local0",
"severity_label" => "Notice"
}
{
"message" => "hello world\n",
"@version" => "1",
"@timestamp" => "2018-07-31T03:38:32.000Z",
"type" => "syslog",
"host" => "192.168.6.10",
"priority" => 133,
"timestamp" => "Jul 31 11:38:32",
"logsource" => "kibana",
"program" => "testlog",
"severity" => 5,
"facility" => 16,
"facility_label" => "local0",
"severity_label" => "Notice"
}
[root@kibana log]# vim /etc/rsyslog.conf
local0.info @192.168.6.16:514
[root@kibana log]# systemctl restart rsyslog
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
[root@kibana log]# logger -p local0.notice -t "testlog" "hello world"
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "hello world",
"@version" => "1",
"@timestamp" => "2018-07-31T03:46:15.000Z",
"type" => "syslog",
"host" => "192.168.6.10",
"priority" => 133,
"timestamp" => "Jul 31 11:46:15",
"logsource" => "kibana",
"program" => "testlog",
"severity" => 5,
"facility" => 16,
"facility_label" => "local0",
"severity_label" => "Notice"
}
echo "test udp log" >/dev/udp/192.168.6.16/9999
{
"message" => "echo \"test udp log\" >/dev/udp/192.168.6.16/9999",
"tags" => [
[0] "_jsonparsefailure"
],
"@version" => "1",
"@timestamp" => "2018-07-31T03:48:21.235Z",
"host" => "logstash"
}
echo "test tcp log" >/dev/tcp/192.168.6.16/8888
{
"message" => "echo \"test tcp log\" >/dev/tcp/192.168.6.16/8888",
"tags" => [
[0] "_jsonparsefailure"
],
"@version" => "1",
"@timestamp" => "2018-07-31T03:49:02.033Z",
"host" => "logstash"
}
[root@logstash logstash]# yum -y install tcpdump
[root@logstash logstash]# vim logstash.conf
filter{
grok{
match => ["message","%{IP:ip}, (?<key>reg)"]
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
[root@kibana log]# cat /var/log/httpd/access_log
192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] "GET /favicon.ico HTTP/1.1" 404 209 "http://192.168.6.10/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"
[root@logstash logstash]# vim /tmp/a.log
192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] "GET /favicon.ico HTTP/1.1" 404 209 "http://192.168.6.10/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T06:14:11.929Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
{
"message" => "",
"@version" => "1",
"@timestamp" => "2018-07-31T06:14:11.931Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
//从头开始记录日志文件
[root@logstash logstash]# vim logstash.conf
#sincedb_path => "/var/lib/logstash/sincedb.log"
sincedb_path => "/dev/null"
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T06:18:30.413Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
{
"message" => "",
"@version" => "1",
"@timestamp" => "2018-07-31T06:18:30.572Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
{
"message" => "b1",
"@version" => "1",
"@timestamp" => "2018-07-31T06:18:30.573Z",
"path" => "/tmp/b.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
[root@logstash logstash]# vim logstash.conf
file {
path => [ "/tmp/a.log" ]
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T06:20:08.237Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
{
"message" => "",
"@version" => "1",
"@timestamp" => "2018-07-31T06:20:08.372Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
使用正则表达式匹配日志
//匹配ip地址
[root@logstash logstash]# vim logstash.conf
filter{
grok{
match => ["message", "(?<client_ip>([12]?\d?\d\.){3}[12]?\d?\d)"]
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T06:27:07.235Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"client_ip" => "192.168.6.254"
}
{
"message" => "",
"@version" => "1",
"@timestamp" => "2018-07-31T06:27:07.378Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
[root@logstash logstash]# vim logstash.conf
filter{
grok{
match => ["message", "(?<client_ip>([12]?\d?\d\.){3}[12]?\d?\d).+\[(?<time>.+)\] \"(?<method>[A-Z]+) (?<url>\S+) (?<porto>[^\"]+)\" (?<code>\d+) "]
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T06:36:04.496Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"client_ip" => "192.168.6.254",
"time" => "31/Jul/2018:10:15:32 +0800",
"method" => "GET",
"url" => "/favicon.ico",
"porto" => "HTTP/1.1",
"code" => "404"
}
{
"message" => "",
"@version" => "1",
"@timestamp" => "2018-07-31T06:36:04.653Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
[root@logstash logstash]# vim logstash.conf
filter{
grok{
match => ["message", "%{IP:client_ip}"]
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T06:39:42.620Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"client_ip" => "192.168.6.254"
}
{
"message" => "",
"@version" => "1",
"@timestamp" => "2018-07-31T06:39:42.809Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
[root@logstash ~]# cd /opt/logstash/vendor/bundle/jruby/
[root@logstash jruby]# find ./ -type f | grep grok
[root@logstash patterns]# pwd
/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns
[root@logstash patterns]#
[root@logstash patterns]# vim grok-patterns
IP (?:%{IPV6}|%{IPV4})
[root@kibana ~]# vim /etc/httpd/conf/httpd.conf
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
[root@logstash patterns]# vim grok-patterns
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
[root@logstash logstash]# vim logstash.conf
filter{
grok{
match => ["message", "%{COMBINEDAPACHELOG}"]
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T07:09:56.310Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"clientip" => "192.168.6.254",
"ident" => "-",
"auth" => "-",
"timestamp" => "31/Jul/2018:10:15:32 +0800",
"verb" => "GET",
"request" => "/favicon.ico",
"httpversion" => "1.1",
"response" => "404",
"bytes" => "209",
"referrer" => "\"http://192.168.6.10/\"",
"agent" => "\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\""
}
{
"message" => "",
"@version" => "1",
"@timestamp" => "2018-07-31T07:09:56.444Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog",
"tags" => [
[0] "_grokparsefailure"
]
}
[root@logstash logstash]# vim logstash.conf
input{
stdin{ codec => "json" }
beats{
port => 5044
}
output{
stdout{ codec => "rubydebug" }
if [type] == "filelog"{
elasticsearch {
hosts => ["192.168.6.15:9200", "192.168.6.11:9200"]
index => "weblog"
flush_size => 2000
idle_flush_time => 10
}}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
[root@logstash patterns]# ss -tunlp
Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port
udp UNCONN 0 0 127.0.0.1:323 *:* users:(("chronyd",pid=478,fd=1))
udp UNCONN 0 0 :::514 :::* users:(("java",pid=2619,fd=52))
udp UNCONN 0 0 :::9999 :::* users:(("java",pid=2619,fd=46))
udp UNCONN 0 0 ::1:323 :::* users:(("chronyd",pid=478,fd=2))
tcp LISTEN 0 128 *:22 *:* users:(("sshd",pid=679,fd=3))
tcp LISTEN 0 100 127.0.0.1:25 *:* users:(("master",pid=784,fd=13))
tcp LISTEN 0 50 :::5044 :::* users:(("java",pid=2619,fd=7))
tcp LISTEN 0 128 :::22 :::* users:(("sshd",pid=679,fd=4))
tcp LISTEN 0 50 :::8888 :::* users:(("java",pid=2619,fd=17))
tcp LISTEN 0 100 ::1:25 :::* users:(("master",pid=784,fd=14))
tcp LISTEN 0 50 :::514 :::* users:(("java",pid=2619,fd=50))
[root@logstash patterns]#
//客户端
[root@kibana ~]# yum -y install filebeat
[root@kibana ~]# vim /etc/filebeat/filebeat.yml
[root@kibana ~]# cd /etc/filebeat/
[root@kibana filebeat]# grep -Pv "^\s*(#|$)" filebeat.yml
[root@kibana filebeat]# ll /var/log/httpd/access_log
-rw-r--r-- 1 root root 3121 7月 31 10:32 /var/log/httpd/access_log
#elasticsearch:
#hosts: ["localhost:9200"]
logstash:
# The Logstash hosts
hosts: ["192.168.6.16:5044"]
[root@kibana filebeat]# grep -Pv "^\s*(#|$)" filebeat.yml
filebeat:
prospectors:
-
paths:
- /var/log/httpd/access_log
input_type: log
document_type: apachelog
registry_file: /var/lib/filebeat/registry
output:
logstash:
hosts: ["192.168.6.16:5044"]
shipper:
logging:
files:
rotateeverybytes: 10485760 # = 10MB
[root@kibana filebeat]# systemctl restart filebeat
{
"message" => "192.168.6.254 - - [31/Jul/2018:16:00:15 +0800] \"GET / HTTP/1.1\" 304 - \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T08:00:16.608Z",
"input_type" => "log",
"source" => "/var/log/httpd/access_log",
"offset" => 3121,
"type" => "apachelog",
"count" => 1,
"fields" => nil,
"beat" => {
"hostname" => "kibana",
"name" => "kibana"
},
"host" => "kibana",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"clientip" => "192.168.6.254",
"ident" => "-",
"auth" => "-",
"timestamp" => "31/Jul/2018:16:00:15 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "304",
"referrer" => "\"-\"",
"agent" => "\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\""
}
[root@logstash logstash]# vim logstash.conf
output{
stdout{ codec => "rubydebug" }
if [type] == "apachelog"{
elasticsearch {
hosts => ["192.168.6.15:9200", "192.168.6.11:9200"]
index => "apachelog"
flush_size => 2000
idle_flush_time => 10
}}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.6.254 - - [31/Jul/2018:10:15:32 +0800] \"GET /favicon.ico HTTP/1.1\" 404 209 \"http://192.168.6.10/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-07-31T08:06:16.515Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "filelog"
}
[root@logstash logstash]# vim logstash.conf
file {
path => [ "/tmp/a.log" ]
sincedb_path => "/var/lib/logstash/sincedb.log"
start_position => "beginning"
type => "filelog"
}
[root@logstash logstash]# /opt/logstash/bin/logstash -f logstash.conf
[root@kibana filebeat]# curl -XDELETE http://192.168.6.11:9200/*
{"acknowledged":true}
[root@kibana filebeat]# systemctl restart kibana
访问页面，记录apache日志
