199. ELK架构接入Redis

接上一篇博文整理：

ES接入Redis

1.安装redis
[root@redis ~]# yum install redis -y

2.配置redis密码
[root@redis ~]# grep "^[a-Z]" /etc/redis.conf |head -4
bind 127.0.0.1 10.0.0.50
requirepass 123456

3.启动redis
[root@redis ~]# systemctl restart redis
[root@redis ~]# systemctl enable redis


4.修改filebeat输出数据至Redis
[root@web01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.redis:
  hosts: ["10.0.0.50:6379"]		#redis地址
  password: 123456				#redis密码
  timeout: 5					#连接超时时间
  db: 0							#写入db0库中
  keys:							#存储的key名称
    - key: "nginx_access"
      when.contains:
        tags: "access"
    - key: "nginx_error"
      when.contains:
        tags: "error"

启动filebeat并测试是否能存到redis里
systectl restart filebeat

[root@redis ~]# redis-cli 
127.0.0.1:6379> keys *
1) "nginx_access"
127.0.0.1:6379> TYPE nginx_access 
list
127.0.0.1:6379> LLEN nginx_access 
(integer) 112875
127.0.0.1:6379> LRANGE nginx_access 1 2 
1) "{\"@timestamp\":\"2020-02-17T13:54:58.449Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"input\":{\"type\":\"log\"},\"ecs\":{\"version\":\"1.1.0\"},\"host\":{\"name\":\"web01\"},\"agent\":{\"hostname\":\"web01\",\"id\":\"405d2c76-4639-4263-88dc-87dac3a8f313\",\"version\":\"7.4.0\",\"type\":\"filebeat\",\"ephemeral_id\":\"1a77a34e-0e5e-4ba4-8d4c-0eb21acf5dc0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"},\"offset\":197},\"message\":\"192.168.37.59 - - [16/Jan/2020:10:21:14 +0800] \\\"GET /favicon.ico HTTP/1.1\\\" 404 187 \\\"http://192.168.37.22/\\\" \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36\\\" \\\"-\\\"\",\"tags\":[\"access\"]}"
2) "{\"@timestamp\":\"2020-02-17T13:54:58.449Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"log\":{\"offset\":426,\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"message\":\"192.168.37.59 - - [16/Jan/2020:10:21:14 +0800] \\\"GET / HTTP/1.1\\\" 200 13 \\\"-\\\" \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36\\\" \\\"-\\\"\",\"tags\":[\"access\"],\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"web01\"},\"agent\":{\"ephemeral_id\":\"1a77a34e-0e5e-4ba4-8d4c-0eb21acf5dc0\",\"hostname\":\"web01\",\"id\":\"405d2c76-4639-4263-88dc-87dac3a8f313\",\"version\":\"7.4.0\",\"type\":\"filebeat\"},\"ecs\":{\"version\":\"1.1.0\"}}"
127.0.0.1:6379> 

5.修改logstash从Redis中读取数据
[root@logstash-node1 ~]# cat /etc/logstash/conf.d/input_redis_nginx_1w_output_es.conf 
input {
	redis {
		host => ["10.0.0.50"]		#redis地址
		port => "6379"
		password => "123456"
		data_type => "list"
		key => "nginx_access"	
		db => "0"		#从哪个库读
	}

	redis {
		host => ["10.0.0.50"]
		port => "6379"
		password => "123456"
		data_type => "list"
		key => "nginx_error"
		db => "0"
	}
}

filter {
    if "access" in [tags][0] {
        grok {
            match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:hostname} (?:%{QS:referrer}|-) (?:%{NOTSPACE:post_args}|-) %{QS:useragent} (?:%{QS:x_forward_for}|-) (?:%{URIHOST:upstream_host}|-) (?:%{NUMBER:upstream_response_code}|-) (?:%{NUMBER:upstream_response_time}|-) (?:%{NUMBER:response_time}|-)" }
        }

        useragent {
            source => "useragent"
            target => "useragent"
        }
        geoip {
            source => "clientip"
        }
        date {
            match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
            target => "@timestamp"
            timezone => "Asia/Shanghai"
        }
        mutate {
            convert => ["bytes","integer"]
            convert => ["response_time", "float"]
            remove_field => ["message","agent","tags"]
            add_field => { "target_index" => "logstash-redis-nginx-access-%{+YYYY.MM.dd}" }
    }
           #提取referrer具体的域名/^"http/
        if [referrer] =~ /^"http/ {
            grok {
                match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' }
            }
        }

        #提取用户请求资源类型以及资源ID编号
        if "oldxu.com" in [referrer_host] {
            grok {
                match => { "referrer" => '%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:oldxu_type}/%{NOTSPACE:oldxu_res_id})?"' }
            }
        }
    } 
    else if "error" in [tags][0] {
        date {
            match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
                target => "@timestamp"
                timezone => "Asia/Shanghai"
        }
        mutate {
            add_field => { "target_index" => "logstash-redis-nginx-error-%{+YYYY.MM.dd}" }
        }
    }
}

output {
    elasticsearch {
        hosts => ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
        index => "%{[target_index]}"
        template_overwrite => true
    }

    stdout {
	codec => rubydebug
    }

}


测试：
[root@redis ~]# systemctl restart redis
[root@redis ~]# redis-cli
127.0.0.1:6379> auth 123456
127.0.0.1:6379> keys *
127.0.0.1:6379> type nginx_access
list
127.0.0.1:6379> llen nginx_access