ELK configuration

Environment:
Centos7.4

软件：
jdk-1.8.0_211；
elasticsearch-7.9.2；
search-guard-7
logstash-7.9.2；
kibana-7.9.2；
filebeat-7.7.1

Scenario: The
company's self-developed java application has three log files, appname_info.log, appname_warn.log, and appname_error.log. The format of info and warn logs are the same, and the error log is a java application error log.

Difficulty 1: Due to the configuration of search-guard-7, the authentication error is reported when logstash connects to elasticsearch. The final configuration is as follows:

output {
    elasticsearch {
        hosts => ["192.168.20.39:9200", "192.168.20.40:9200", "192.168.20.41:9200"]
        ssl => true
        ssl_certificate_verification => false
        cacert => "/export/server/logstash-7.9.2/config/root-ca.pem"
        index => "ngms-exchange-%{+YYYY.MM.dd}"
        user => "admin"
        password => "admin"
    }
}

Difficulty 2: logstash matches logs in two formats, the configuration is as follows:

filter {
    grok {
        match => {
            "message" => [
                '(?<recod-time>\d+-\d+-\d+\s+\d+:\d+:\d+\.\d+)(\|)(?<log-level>\w+)(\s+\|)(?<log-message>.*)',  #匹配单行日志
                '(?m)^%{TIMESTAMP_ISO8601:recod-time}\|%{LOGLEVEL:log-level}\|%{GREEDYDATA:log-message}'  #匹配多行日志
            ]
        }
        remove_field => ["message","agent","flags","ecs","os","path","@version"]
    }

    date {
        match => ["recod-time","yyyy-MM-dd HH:mm:ss.SSS"]
        target => "@timestamp"  #用日志的时间替换logstash原来的timestamp
    }
}

Difficulty 3: Use filebeat to output logs of different applications to logstash, add tags in the filebeat configuration, and then judge in logstash:

#filebeat配置如下：

- type: log
  paths:
    - /export/logs/appname1/appname1*.log
  fields:
    source: appname1
  scan_frequency: 10s
    #filebeat多行匹配
  multiline.pattern: '^\d{4}-\d{2}-\d{2}'
  multiline.negate: true
  multiline.match: after
    #添加tags
  tags: ['appname1']

- type: log
  paths:
    - /export/logs/appname2/appname2*.log
  fields:
    source: appname2
  scan_frequency: 10s
    #filebeat多行匹配
  multiline.pattern: '^\d{4}-\d{2}-\d{2}'
  multiline.negate: true
  multiline.match: after
    #添加tags
  tags: ['appname2']

#logstash配置

output {
    if "ngms-exchange" in [tags] {
        elasticsearch {
            hosts => ["192.168.20.39:9200", "192.168.20.40:9200", "192.168.20.41:9200"]
            ssl => true
            ssl_certificate_verification => false
            cacert => "/export/server/logstash-7.9.2/config/root-ca.pem"
            index => "ngms-exchange-%{+YYYY.MM.dd}"
            user => "admin"
            password => "admin"
        }
    }
}