logstash安装使用

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_36666651/article/details/83617858

下载logstash(最好下载与es相同的版本，这里为了测试下载的低版本)

wget https://download.elastic.co/logstash/logstash/logstash-2.3.4.tar.gz

解压

tar -zxvf logstash-2.3.4.tar.gz

运行测试

# 使用标准输入输出
./logstash-2.3.4/bin/logstash -e 'input { stdin { } } output { stdout {} }'

# 使用标准输入输出，输出格式化为json
./logstash-2.3.4/bin/logstash -e 'input { stdin { } } output { stdout {codec => json} }'

# 加载配置文件启动
./logstash-2.3.4/bin/logstash -f logstash-simple.conf
# 加载多个配置文件启动
./logstash-2.3.4/bin/logstash -f .conf/*

logstash模式

logstash做的事情分三个阶段依次执行：输入——》处理filter（不是必须）——》输出

logstash配置文件

宏观配置文件格式

#　输入
input {
  ...
}

# 过滤器
filter {
  ...
}

# 输出
output {
  ...
}

配置文件示例：

# 参考 https://www.jianshu.com/p/25ed5ed46682
# https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/filter/kv.html
# https://www.cnblogs.com/qq27271609/p/4762562.html

# 日志格式
# [log_time=2018-11-01 15:47:03] [level=ERROR] [app_name=logback_test] [version=1.0.0] [class=com.wk.logbackdemo.LogbackTest] test error


input {
        file {
                #add_field => {"project_name" => "battleship"}
                #tags => "tag1"
                path => ["/home/es-wk/logstash/logs/info/*.log","/home/es-wk/logstash/logs/debug/*.log","/home/es-wk/logstash/logs/warn/*.log","/home/es-wk/logstash/logs/error/*.log"]
                start_position => beginning
				sincedb_path => "/dev/null" #从头读  第一读取时就加上才会生效  后面再加需要用新的日志文件 测试用
				# 多行合并
#				codec => multiline {
#					pattern => "^%{TIMESTAMP_ISO8601} "
#					negate => true
#					what => previous
#               }
        }
}

# 可写多个  内部顺序执行
filter {
		mutate {
                        # 替换掉开头的[
                        gsub => ["message", "\[", ""]
                        # 根据]分割字段
                        split => ["message", "] "]

                        add_field => { "log_time" => "%{[message][0]}"}
                        add_field => { "level" => "%{[message][1]}"}
                        add_field => { "app_name" => "%{[message][2]}"}
                        add_field => { "version" => "%{[message][3]}"}
                        add_field => { "class" => "%{[message][4]}"}
                        rename => ["host", "host_name"]
                }

                kv {
                    #include_keys => ["log_time", "level", "version", "class"]
                        field_split => "="
                }

                mutate {
                        replace => {"message" => "%{[message][5]}"}
                }

}

output {
        elasticsearch{
			hosts => ["127.0.0.1:19200"]
			index => "%{app_name}"
			user => "elastic" 
			password => "changme"
		}
		# 调试用
		stdout { codec => rubydebug }
}