1.
[root@server1 ~]# yum install -y elasticsearch-2.3.3.rpm
[root@server1 ~]# cd /etc/elasticsearch/
[root@server1 elasticsearch]# ls
elasticsearch.yml #主配置文件
logging.yml
scripts
[root@server1 elasticsearch]# vim elasticsearch.yml
17 cluster.name: my-es #集群名称
23 node.name: server1 #节点名称
33 path.data: /var/lib/elasticsearch #数据存放位置
43 bootstrap.mlockall: true #锁定内存
54 network.host: 172.25.44.1
58 http.port: 9200 #网络的监听端口
68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"] # 设置集群的节点个数
[root@server1 elasticsearch]# /etc/init.d/elasticsearch start #依赖Java
which: no java in (/sbin:/usr/sbin:/bin:/usr/bin)
Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME
[root@server1 ~]# rpm -ivh jdk-8u121-linux-x64.rpm # 解决依赖性
[root@server1 ~]# /etc/init.d/elasticsearch start
[root@server1 ~]# netstat -natlp
tcp 0 0 ::ffff:172.25.44.1:9200 :::* LISTEN 1371/java
#
# 访问端口
# 在线安装
[root@server1 ~]# cd /usr/share/elasticsearch/bin
[root@server1 bin]# ./plugin install mobz/elasticsearch-head
#
# 我有这个压缩包,不用在线安装,直接安装存在的压缩包即可,注意路径
[root@server1 ~]# /usr/share/elasticsearch/bin/plugin install file:/root/elasticsearch-head-master.zip # 安装
[root@server1 ~]# cd /usr/share/elasticsearch/plugins/
[root@server1 plugins]# ls
head
#查看健康状况
[root@server1 conf.d]# curl -XGET 'http://172.25.44.1:9200/_cluster/health?pretty=true'
在实际生产环境中,master尽量多一点
改配置文件的时候,注意空格,如果重启报错,一定是文件内容写错了,空格不要多也不要少,本人已经踩过很多这样的坑了
[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml
24 node.master: true
25 node.data: false # 不做数据节点,存储数据
26 http.enable: true # 查询
[root@server1 ~]# /etc/init.d/elasticsearch reload
Stopping elasticsearch: [ OK ]
Starting elasticsearch: [ OK ]
[root@server2 ~]# ls
elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server2 ~]# rpm -ivh elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml
17 cluster.name: my-es
23 node.name: server2
24 node.master: false # 不做master
25 node.data: true # 存储数据
26 http.enable: true # 可以查询,如果关闭查询功能,则端口查看不到
33 path.data: /var/lib/elasticsearch
43 bootstrap.mlockall: true
54 network.host: 172.25.44.2
58 http.port: 9200
68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]
[root@server2 ~]# /etc/init.d/elasticsearch start
Starting elasticsearch: [ OK ]
[root@server3 ~]# ls
elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server3 ~]# rpm -ivh elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml
17 cluster.name: my-es
23 node.name: server3
25 node.master: false
26 node.data: true
27 http.enabled: true
33 path.data: /var/lib/elasticsearch
43 bootstrap.mlockall: true
54 network.host: 172.25.44.3
58 http.port: 9200
68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]
[root@server3 ~]# /etc/init.d/elasticsearch start
Starting elasticsearch: [ OK ]
3.数据采集(logstash)
[root@server1 ~]# rpm -ivh logstash-2.3.3-1.noarch.rpm
[root@server1 ~]# cd /usr/share/elasticsearch/bin
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin{ } } output { stdout {} }' #调用input模块,stdin 是终端目录,stdout是终端输出
Settings: Default pipeline workers: 1
Pipeline main started
hello
2018-08-25T02:42:14.700Z server1 hello
westos
2018-08-25T02:42:46.580Z server1 westos
lalalala
2018-08-25T02:42:51.582Z server1 lalalala
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin{ } } output { stdout { codec => rubydebug } }' #codec=>rubydebug ,控制输出,格式转换
Settings: Default pipeline workers: 1
Pipeline main started
hello
{
"message" => "hello",
"@version" => "1",
"@timestamp" => "2018-08-25T02:47:57.213Z",
"host" => "server1"
}
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin{ } } output { elasticsearch { hosts => ["172.25.44.1"] index => "logstash-%{+YYYY.MM.dd}" }stdout { codec => rubydebug } }' #添加elasticsearch模块:hosts,指定主机,index,指定索引
Settings: Default pipeline workers: 1
Pipeline main started
hello
{
"message" => "hello ",
"@version" => "1",
"@timestamp" => "2018-08-25T03:28:41.315Z",
"host" => "server1"
}
lalala
{
"message" => "lalala",
"@version" => "1",
"@timestamp" => "2018-08-25T03:29:36.377Z",
"host" => "server1"
}
hahahaha
{
"message" => "hahahaha",
"@version" => "1",
"@timestamp" => "2018-08-25T03:29:41.547Z",
"host" => "server1"
}
#
# 在终端上的输入会记录在浏览器中
[root@server1 elasticsearch]# cd /etc/logstash/
[root@server1 logstash]# cd conf.d/
[root@server1 conf.d]# ls
[root@server1 conf.d]# pwd
/etc/logstash/conf.d
[root@server1 conf.d]# vim es.conf
1 input {
2 stdin {}
3 }
4
5 output {
6 elasticsearch {
7 hosts => ["172.25.44.1"]
8 index => "logstash-%{+YYYY.MM.dd}"
9 }
10 stdout {
11 codec => rubydebug
12 }
13 }
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf #
Settings: Default pipeline workers: 1
Pipeline main started
Settings: Default pipeline workers: 1
Pipeline main started
say hi
{
"message" => "say hi",
"@version" => "1",
"@timestamp" => "2018-08-25T03:38:15.980Z",
"host" => "server1"
}
扫描二维码关注公众号,回复:
3428400 查看本文章
4.文件模块的使用
logstash # 以logstash身份在运行
终端运行,以root用户运行
打入后台,脚本运行,不是root用户
[root@server1 conf.d]# cp es.conf message.conf
[root@server1 conf.d]# vim message.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf # 开启之后,网页会生成新的索引文件
Settings: Default pipeline workers: 1
Pipeline main started
# 重新打开一个shell,录入数据,日志就会有记录,网页也会有记录
[root@server1 ~]# logger test
[root@server1 ~]# logger hello
[root@server1 ~]# logger hello
[root@server1 ~]# logger hello
[root@server1 ~]# logger hello
[root@server1 ~]# cat /var/log/messages
# 中断之后再开启,不会从头开始,有隐藏文件来记录文件信息,来确保文件内容是否变化,若修改文件内容,会有相应变化,避免文件重复加载
[root@server1 ~]# l.
. .bash_profile .oracle_jre_usage .tcshrc
.. .bashrc .sincedb_452905a167cf4509fd08acb964fdb20c .viminfo
.bash_logout .cshrc .ssh
[root@server1 ~]# cat .sincedb_452905a167cf4509fd08acb964fdb20c
1044503 0 64768 32668
# 想把文件重新加载,必须删掉隐藏文件
5. 在master端把lodstash 作为日志收集器
[root@server1 conf.d]# vim message.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf #打开
[root@server1 ~]# netstat -antlp | grep :514 # 查看端口
tcp 0 0 :::514 :::* LISTEN 2273/java
[root@server2 ~]# vim /etc/rsyslog.conf
82 *.* @@172.25.44.1:514
[root@server2 ~]# /etc/init.d/rsyslog restart
Shutting down system logger: [ OK ]
Starting system logger: [ OK ]
# 录入数据,master上就会有记录
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos
# 自动记录远程数据,生成日志
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "imklog 5.8.10, log source = /proc/kmsg started.\n",
"@version" => "1",
"@timestamp" => "2018-08-25T03:59:16.000Z",
"host" => "172.25.44.2",
"priority" => 6,
"timestamp" => "Aug 25 11:59:16",
"logsource" => "server2",
"program" => "kernel",
"severity" => 6,
"facility" => 0,
"facility_label" => "kernel",
"severity_label" => "Informational"
}
{
"message" => "[origin software=\"rsyslogd\" swVersion=\"5.8.10\" x-pid=\"1946\" x-info=\"http://www.rsyslog.com\"] start\n",
"@version" => "1",
"@timestamp" => "2018-08-25T03:59:16.000Z",
"host" => "172.25.44.2",
"priority" => 46,
"timestamp" => "Aug 25 11:59:16",
"logsource" => "server2",
"program" => "rsyslogd",
"severity" => 6,
"facility" => 5,
"facility_label" => "syslogd",
"severity_label" => "Informational"
}
{
"message" => "(root) CMD (run-parts /etc/cron.hourly)\n",
"@version" => "1",
"@timestamp" => "2018-08-25T04:01:01.000Z",
"host" => "172.25.44.2",
"priority" => 78,
"timestamp" => "Aug 25 12:01:01",
"logsource" => "server2",
"program" => "CROND",
"pid" => "1952",
"severity" => 6,
"facility" => 9,
"facility_label" => "clock",
"severity_label" => "Informational"
}
{
"message" => "run-parts(/etc/cron.hourly)[1952 starting 0anacron\n",
"@version" => "1",
"@timestamp" => "2018-08-25T04:01:01.000Z",
"host" => "172.25.44.2",
"priority" => 77,
"timestamp" => "Aug 25 12:01:01",
"logsource" => "server2",
"severity" => 5,
"facility" => 9,
"facility_label" => "clock",
"severity_label" => "Notice"
}
6.过滤
类型
关键字
向上匹配
[root@server1 conf.d]# pwd
/etc/logstash/conf.d
[root@server1 conf.d]# vim message.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf
7.软件的数据采集
(1)httpd
[root@server1 ~]# yum intall -y httpd
[root@server1 ~]# /etc/init.d/httpd start
[root@server1 ~]# vim /var/www/html/index.html
1 www.westos server1
[root@server1 conf.d]# vim message.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf
# 分类
[root@server1 conf.d]# vim test.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf
[root@server1 ~]# cd /var/log/
[root@server1 log]# ls
audit cron elasticsearch lastlog maillog rhsm spooler yum.log
boot.log dmesg httpd logstash messages secure wtmp
[root@server1 log]# cd httpd/
[root@server1 httpd]# ls
access_log error_log
[root@server1 httpd]# ls -i
1050095 access_log 1045219 error_log
[root@server1 httpd]# ls -i error_log
1045219 error_log
[root@server1 httpd]# ls -i access_log
1050095 access_log
[root@server1 httpd]# cd
[root@server1 ~]# cat .sincedb_ef0edb00900aaa8dcb520b280cb2fb7d
1050095 0 64768 304
1045219 0 64768 439
[root@server1 ~]# rm -fr .sincedb_ef0edb00900aaa8dcb520b280cb2fb7d
[root@server1 conf.d]# vim message.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf
(2)数据可视化
[root@server3 ~]# ls
elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm kibana-4.5.1-1.x86_64.rpm
[root@server3 ~]# rpm -ivh kibana-4.5.1-1.x86_64.rpm
[root@server3 ~]# cd /opt/kibana/config/
[root@server3 config]# ls
kibana.yml
[root@server3 config]# vim kibana.yml
15 elasticsearch.url: "http://172.25.44.1:9200"
23 kibana.index: ".kiban
[root@server3 config]# /etc/init.d/kibana start
[root@server3 config]# netstat -antlp
tcp 0 0 0.0.0.0:5601 0.0.0.0:* LISTEN 1996/node
数据传输过程
logstash input { nginx } output { redis } -> logstash input { redis } output { redis } -> es kibana
#
[root@server2 ~]# ls
elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm redis-3.0.6.tar.gz
[root@server2 ~]# tar zxf redis-3.0.6.tar.gz
[root@server2 ~]# ls
elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm redis-3.0.6 redis-3.0.6.tar.gz
[root@server2 ~]# cd redis-3.0.6
[root@server2 redis-3.0.6]# make && make install
[root@server2 redis-3.0.6]# cd utils/
[root@server2 utils]# ./install_server.sh
[root@server2 redis-3.0.6]# netstat -antlp
[root@server2 utils]# netstat -antulp | grep :6379
tcp 0 0 0.0.0.0:6379 0.0.0.0:* LISTEN 4975/redis-server *
tcp 0 0 :::6379 :::* LISTEN 4975/redis-server *
[root@server1 ~]# /etc/init.d/httpd stop
[root@server1 ~]# rpm -ivh nginx-1.8.0-1.el6.ngx.x86_64.rpm
[root@server1 ~]# /etc/init.d/nginx start
[root@server1 ~]# cd /etc/logstash/conf.d/
[root@server1 conf.d]# ls
es.conf message.conf test.conf
[root@server1 conf.d]# cp message.conf nginx.conf
[root@server1 conf.d]# vim nginx.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf #打开
# 在真机做压侧
[root@foundation44 ~]# ab -c 1 -n 10 http://172.25.44.1/index.html
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "172.25.44.250 - - [25/Aug/2018:16:47:06 +0800] \"GET /index.html HTTP/1.0\" 200 612 \"-\" \"ApacheBench/2.3\" \"-\"",
"@version" => "1",
"@timestamp" => "2018-08-25T08:47:07.691Z",
"path" => "/var/log/nginx/access.log",
"host" => "server1",
"clientip" => "172.25.44.250",
"ident" => "-",
"auth" => "-",
"timestamp" => "25/Aug/2018:16:47:06 +0800",
"verb" => "GET",
"request" => "/index.html",
"httpversion" => "1.0",
"response" => "200",
"bytes" => "612",
"referrer" => "\"-\"",
"agent" => "\"ApacheBench/2.3\"",
"x_forworded_for" => "\"-\""
}
{
"message" => "172.25.44.250 - - [25/Aug/2018:16:47:06 +0800] \"GET /index.html HTTP/1.0\" 200 612 \"-\" \"ApacheBench/2.3\" \"-\"",
"@version" => "1",
"@timestamp" => "2018-08-25T08:47:07.694Z",
"path" => "/var/log/nginx/access.log",
"host" => "server1",
"clientip" => "172.25.44.250",
"ident" => "-",
"auth" => "-",
"timestamp" => "25/Aug/2018:16:47:06 +0800",
"verb" => "GET",
"request" => "/index.html",
"httpversion" => "1.0",
"response" => "200",
"bytes" => "612",
"referrer" => "\"-\"",
"agent" => "\"ApacheBench/2.3\"",
"x_forworded_for" => "\"-\""
}
[root@server2 ~]# rpm -ivh logstash-2.3.3-1.noarch.rpm
[root@server2 ~]# cd /etc/logstash/conf.d/
[root@server2 conf.d]# ls
es.conf
[root@server2 conf.d]# vim es.conf
[root@server2 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
[root@foundation44 elk]# ab -c 1 -n 10 http://172.25.44.1/index.html
[root@server2 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Settings: Default pipeline workers: 1
Pipeline main started
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf
# 做压侧