elk analyzes nginx logs and tomcat logs

1. Introduction

Elasticsearch + Logstash + Kibana (ELK) is an open source log management solution.

ElasticsearchIt is an open source distributed search engine. Its features are: distributed, zero configuration, automatic discovery, automatic index fragmentation, index replication mechanism, restful style interface, multiple data sources, automatic search load, etc.

Logstashis a completely open source tool that collects, analyzes, and stores your logs for later use

kibana is an open source and free tool that provides a log analysis-friendly web interface for Logstash and ElasticSearch, which can help you aggregate, analyze and search important data logs.

ELK official website: https://www.elastic.co/

ELK official website documentation: https://www.elastic.co/guide/index.html

ELK Chinese manual: http://kibana.logstash.es/content/elasticsearch/monitor/logging.html

2. Description of the test environment

System: centos6.5_x86_64

Software: elasticsearch-6.1.2, kibana-6.1.2-linux-x86_64, logstash-6.1.2, redis-3.2.6, jdk1.8

1. Server (all software installed)

ip：10.10.123.201

Public network ip: 123.206.57.23

hostname：VM_123_201_centos

2. Client (install jdk and logstash )

ip：10.10.30.86

hostname：VM_30_86_centos

3. Server installation and configuration

1. Install redis

#!/bin/bash
yum -y install make gcc gcc-c++ zlib-devel openssl openssl-devel pcre-devel patch perl tcl 
cd / var / ftp /
tar xf redis-3.2.6.tar.gz
mv redis-3.2.6 /usr/local/redis
cd /usr/local/redis
make && make test && make install
if [ ! -d  "/usr/local/bin" ];   
then 
    mkdir -p /usr/local/bin
be
ln -s /usr/local/redis/redis.conf /etc/redis.conf 
sed -i '/^daemonize no/cdaemonize yes' /etc/redis.conf
redis-server /etc/redis.conf #Start redis service
echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf
sysctl -p
cat> /etc/init.d/redis <<'EOF'
#!/bin/sh
# chkconfig:   2345 90 10
# description:  Redis is a persistent key-value database
# redis    Startup script for redis processes
# processname: redis
redis_path="/usr/local/bin/redis-server"
redis_conf="/etc/redis.conf"
redis_pid="/var/run/redis.pid"
# Source function library.
. /etc/rc.d/init.d/functions
[ -x $redis_path ] || exit 0
RETVAL=0
prog="redis"
# Start daemons.
start() {
if [ -e $redis_pid -a ! -z $redis_pid ];then
echo $prog" already running...."
exit 1
be
echo -n $"Starting $prog "
# Single instance for all caches
$redis_path $redis_conf
RETVAL=$?
[ $RETVAL -eq 0 ] && {
touch /var/lock/subsys/$prog
success $"$prog"
}
echo
return $RETVAL
}
# Stop daemons.
stop() {
echo -n $"Stopping $prog "
killproc -d 10 $redis_path
echo
[ $RETVAL = 0 ] &&rm -f $redis_pid /var/lock/subsys/$prog
RETVAL=$?
return $RETVAL
}
# See how we were called.
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status $prog
RETVAL=$?
;;
restart)
stop
start
;;
condrestart)
if test "x`pidofredis`" != x; then
stop
start
be
;;
*)
echo $"Usage: $0 {start|stop|status|restart|condrestart}"
exit 1
esac
exit $RETVAL
EOF
sleep 3
chmod 755 /etc/init.d/redis
chkconfig --add redis
chkconfig --level 2345 redis on
chkconfig redis on
service redis restart

2. Install elasticsearch

# vim /etc/sysctl.conf

vm.overcommit_memory=1

vm.overcommit_memory = 1

vm.max_map_count=262144

kernel.msgmax = 65536

kernel.msgmnb = 65536

# sysctl -p #Make the configuration take effect

# vim /etc/security/limits.conf

* hard nofile 65536

* soft nofile 65536

# vim /etc/security/limits.d/90-nproc.conf

* soft nproc 4096

root soft nproc unlimited

# groupadd elk
# useradd any -g any
# cd /data/each/
# tar zxvf elasticsearch-6.1.2.tar.gzvim elasticsearch.yml

# vim /data/elk/elasticsearch-6.1.2/config/elasticsearch.yml

cluster.name: my-application

node.name: node-201

bootstrap.memory_lock: false

bootstrap.system_call_filter: false

network.host: 10.10.123.201

http.port: 9200

http.cors.enabled: true

http.cors.allow-origin: "*"

path.data: /usr/deploy/elk/elasticsearch-6.1.2/data

path.logs: /usr/deploy/elk/elasticsearch-6.1.2/logs

# chown -R any:elk /data/elk/elasticsearch-6.1.2/*
# su - each
$ /data/elk/elasticsearch-6.1.2/bin/elasticsearch -d #Start elasticsearch service

3. Install logstash+jdk

# cd /data/each/
# tar zxf jdk-8u162-linux-x64.tar.gz
# mv  jdk-8u162-linux-x64  /opt/jdk1.8

# vim /etc/profile

export JAVA_HOME=/opt/jdk1.8

export PATH=$JAVA_HOME/bin:$PATH

# source /etc/profile

# tar zxvf logstash-6.1.2.tar.gz

# vim /data/elk/logstash-6.1.2/config/input.conf

input {

say again {

type => "tomcat-10.10.30.86"

host => "123.206.57.23"

key => "tomcat"

data_type => 'list'

port => "6379"

db => "6"

}

say again {

type => "nginx-10.10.30.86"

host => "123.206.57.23"

key => "nginx"

data_type => 'list'

port => "6379"

db => "6"

}

filter {

if [type] == "nginx-10.10.30.86"{

geoip {

source => "clientip"

target => "geoip"

database => "/usr/deploy/elk/GeoLite2-City.mmdb"

add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]

add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]

}

output {

if [type] == "tomcat-10.10.30.86" {

elasticsearch {

hosts => ["123.206.57.23:9200"]

index => "logstash-tomcat-10.10.30.86-%{+YYYY.MM.dd}"

}

if [type] == "nginx-10.10.30.86" {

elasticsearch {

hosts => ["123.206.57.23:9200"]

index => "logstash-nginx-10.10.30.86-%{+YYYY.MM.dd}"

}

# cd /usr/deploy/elk/
# wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz          #地图的库
# gzip -d GeoLite2-City.mmdb.gz
# logstash-plugin install logstash-filter-geoip
# /data/elk/logstash-6.1.2/bin/logstash -f  /data/elk/logstash-6.1.2/config/input.conf       #启动logstash服务

4、安装kibana

# cd /data/elk/

# tar zxvf kibana-6.1.2-linux-x86_64.tar.gz

# vim /usr/deploy/elk/kibana-6.1.2-linux-x86_64/config/kibana.yml

server.port: 5601

server.host: "0.0.0.0"

elasticsearch.url: http://10.10.123.201:9200

kibana.index: ".kibana"

tilemap.url: http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z} #地图显示链接

# /data/elk/kibana-6.1.2-linux-x86_64/bin/kibana & #后台运行kibana服务

四、客户端安装配置

# cd /data/elk/

# tar zxf jdk-8u162-linux-x64.tar.gz

# mv jdk-8u162-linux-x64 /opt/jdk1.8

# vim /etc/profile

export JAVA_HOME=/opt/jdk1.8

export PATH=$JAVA_HOME/bin:$PATH

# source /etc/profile

# tar zxvf logstash-6.1.2.tar.gz

# vim /data/elk/logstash-6.1.2/config/output.conf

input {

file {

path => "/usr/deploy/server/tomcat/tomcat1/logs/catalina*"

type => "tomcat-10.10.30.86"

start_position => "beginning"

codec => multiline {

pattern => "^\["

negate => true

what => "previous"

}

file {

path => "/usr/deploy/server/openresty/nginx/logs/access_json.log"

codec => json

type => "nginx-10.10.30.86"

start_position => "beginning"

}

output {

if [type] == "tomcat-10.10.30.86" {

redis {

host => "123.206.57.23"

key => "tomcat"

data_type => 'list'

port => "6379"

db => "6"

}

if [type] == "nginx-10.10.30.86" {

redis {

host => "123.206.57.23"

key => "nginx"

data_type => 'list'

port => "6379"

db => "6"

}

客户端nginx日志设置为json格式的日志，方便显示地图分布图

log_format json '{"@timestamp":"$time_iso8601",'

'"host":"$server_addr",'

'"clientip":"$remote_addr",'

'"size":$body_bytes_sent,'

'"responsetime":$request_time,'

'"upstreamtime":"$upstream_response_time",'

'"upstreamhost":"$upstream_addr",'

'"http_host":"$host",'

'"url":"$uri",'

'"domain":"$host",'

'"xff":"$http_x_forwarded_for",'

'"referer":"$http_referer",'

'"agent":"$http_user_agent",'

'"status":"$status"}';

access_log /usr/deploy/server/openresty/nginx/logs/access_json.log json;

# /data/elk/logstash-6.1.2/bin/logstash -f /data/elk/logstash-6.1.2/config/input.conf #启动logstash服务

在浏览器访问：

http://123.206.57.23:5601

五、常用浏览器分析设置

1、显示top10 的ip地址条形统计图

2、在地图上显示访问ip的分布

3、饼状图显示各个时间段的访问数量

并.png

4、可以下载到本地的ip统计数据

表格.png

图形定义完成后保存，在Dashboard面板添加定义好的图形，就显示一组我们需要的图形了。

Dashboard显示如下图：

elk analyzes nginx logs and tomcat logs

Guess you like