Logstash配置与使用

Logstash配置与使用

host	ip
node1	192.168.79.103
node2	192.168.79.101

1、在命令行执行logstash

• 标准输入和标准输出，输入什么显示什么

[root@node1 ~]# cd /opt/logstash/bin
[root@node1 bin]# ./logstash -e 'input { stdin{} } output { stdout{} }'
Settings: Default pipeline workers: 2
Pipeline main started
hello world
2018-08-23T03:27:42.798Z node1 hello world

• 以rubydebug格式输出

[root@node1 bin]# ./logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug }}'
Settings: Default pipeline workers: 2
Pipeline main started
hello world
{
       "message" => "hello world",
      "@version" => "1",
    "@timestamp" => "2018-08-23T03:31:04.556Z",
          "host" => "node1"
}

• 从stdin输入，输出值elasticsearch

[root@node1 bin]# ./logstash -e 'input { stdin{} } output { elasticsearch { hosts => ["192.168.79.103"] index => "logstash-%{+YYYY.MM.dd}" } }'

• 即输出至elasticsearch也输出至stdout

[root@node1 bin]# ./logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug }elasticsearch { hosts => ["192.168.79.103"] index => "logstash-%{+YYYY.MM.dd}" } }'
Settings: Default pipeline workers: 2
Pipeline main started
hello world
{
       "message" => "hello world",
      "@version" => "1",
    "@timestamp" => "2018-08-23T03:46:28.131Z",
          "host" => "node1"
}

2、通过配置文件输入、输出

==logstash配置文件目录/etc/logstash/conf.d==

[root@node1 etc]# cat demo.conf
input{
    stdin{}
}

filter{

}

output{
    elasticsearch {
        hosts => ["192.168.79.103:9200"]
        index => "logstash-%{+YYYY.MM.dd}"
    }
    stdout {
        codec => rubydebug
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/demo.conf

• 收集系统日志

[root@node1 conf.d]# cat file.conf
input{
    file{
    path => ["/var/log/messages","/var/log/secure"]
    type => "system-log"
    start_position => "beginning"
    }
}

filter{
}

output{
        elasticsearch {
        hosts => ["192.168.79.103:9200"]
        index => "system-log-%{+YYYY.MM}"
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/file.conf

• 收集Java日志

[root@node1 conf.d]# cat file.conf
input{
    file{
    path => ["/var/log/messages","/var/log/secure"]
    type => "system-log"
    start_position => "beginning"
    }
    file{
    path => "/var/log/elasticsearch/myes.log"
    type => "es-log"
    start_position => "beginning"
    }
}

filter{
}

output{
    if [type] == "system-log" {
            elasticsearch {
        hosts => ["192.168.79.103:9200"]
        index => "system-log-%{+YYYY.MM}"
        }
    }
    if [type] == "es-log" {
            elasticsearch {
        hosts => ["192.168.79.103:9200"]
        index => "es-log-%{+YYYY.MM}"
        }
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/file.conf

==上述配置文件浏览器访问，发现java日志按行分隔，不好阅读，所以加入多行codec==

[root@node1 conf.d]# cat file.conf
input{
    file{
    path => ["/var/log/messages","/var/log/secure"]
    type => "system-log"
    start_position => "beginning"
    }
    file{
    path => "/var/log/elasticsearch/myes.log"
    type => "es-log"
    start_position => "beginning"
    codec => multiline {
            pattern => "^\["
            negate => true
            what => "previous"
    }
    }
}

filter{
}

output{
    if [type] == "system-log" {
            elasticsearch {
        hosts => ["192.168.79.103:9200"]
        index => "system-log-%{+YYYY.MM}"
        }
    }
    if [type] == "es-log" {
            elasticsearch {
        hosts => ["192.168.79.103:9200"]
        index => "es-log-%{+YYYY.MM}"
        }
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f file.conf

• 收集nginx Jason格式日志

  • 方法1：nginx日志改成json输出
  • 方法2：文件直接收取redis，python脚本读取redis，写成json，写入es

方法1：

# nginx日志配置
log_format  access_log_json  '{"user_ip":"$http_x_real_ip","lan_ip":"$remote_addr","log_time":"$time_iso8601","user_req":"$request","http_code":"$status","body_bytes_sent":"$body_bytes_sent","req_time":"$request_time","user_ua":"$http_user_agent"}';

# 收集日志输出至es
[root@node1 conf.d]# cat nginx.conf
input{
    file{
        path => "/var/log/nginx/access_json.log"
        codec => "json"
    }
}

filter{
}

output{
    elasticsearch {
        hosts => ["192.168.79.103:9200"]
        index => "nginx-access-log-%{+YYYY.MM.dd}"
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f nginx.conf

方法2：

• input插件rsyslog

输出至屏幕

[root@node1 conf.d]# cat syslog.conf
input{
    syslog{
        type => "system-syslog"
    port => 514
    }
}

filter{

}

output{
    stdout{
    codec => rubydebug
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f syslog.conf

修改rsyslog配置文件

[root@node1 ~]# vim /etc/rsyslog.conf
*.* @@192.168.79.103:514
[root@node1 ~]# systemctl restart rsyslog

输出至es

[root@node1 conf.d]# cat syslog.conf
input{
    syslog{
        type => "system-syslog"
    port => 514
    }
}

filter{

}

output{
    elasticsearch{
    hosts => ["192.168.79.103:9200"]
    index => "system-syslog-%{+YYYY.MM}"
    }
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f syslog.conf