Elasticsearch架构

其他 2018-10-05 21:20:07 阅读次数: 0

1.

[root@server1 ~]# yum install -y elasticsearch-2.3.3.rpm
[root@server1 ~]# cd /etc/elasticsearch/
[root@server1 elasticsearch]# ls  
        elasticsearch.yml     #主配置文件
        logging.yml           
         scripts
[root@server1 elasticsearch]# vim elasticsearch.yml  
        17 cluster.name: my-es     #集群名称
        23 node.name: server1      #节点名称
        33 path.data: /var/lib/elasticsearch   #数据存放位置
        43 bootstrap.mlockall: true    #锁定内存
        54 network.host: 172.25.44.1
        58 http.port: 9200   #网络的监听端口
        68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]      # 设置集群的节点个数
[root@server1 elasticsearch]# /etc/init.d/elasticsearch start   #依赖Java
which: no java in (/sbin:/usr/sbin:/bin:/usr/bin)
Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME
[root@server1 ~]# rpm -ivh jdk-8u121-linux-x64.rpm  # 解决依赖性
[root@server1 ~]# /etc/init.d/elasticsearch start
[root@server1 ~]# netstat -natlp
        tcp        0      0 ::ffff:172.25.44.1:9200     :::*                        LISTEN      1371/java

#

# 访问端口

这里写图片描述

# 在线安装
[root@server1 ~]# cd /usr/share/elasticsearch/bin
[root@server1 bin]# ./plugin   install mobz/elasticsearch-head

#

# 我有这个压缩包，不用在线安装，直接安装存在的压缩包即可，注意路径
[root@server1 ~]# /usr/share/elasticsearch/bin/plugin install file:/root/elasticsearch-head-master.zip        #  安装
[root@server1 ~]# cd /usr/share/elasticsearch/plugins/
[root@server1 plugins]# ls
        head

这里写图片描述
这里写图片描述
这里写图片描述
这里写图片描述
这里写图片描述

#查看健康状况
[root@server1 conf.d]# curl -XGET 'http://172.25.44.1:9200/_cluster/health?pretty=true'

这里写图片描述

在实际生产环境中，master尽量多一点

改配置文件的时候，注意空格，如果重启报错，一定是文件内容写错了，空格不要多也不要少，本人已经踩过很多这样的坑了

[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml  
        24 node.master: true  
        25 node.data: false     # 不做数据节点,存储数据
        26 http.enable: true    # 查询
[root@server1 ~]# /etc/init.d/elasticsearch reload
        Stopping elasticsearch:                                    [  OK  ]
        Starting elasticsearch:                                    [  OK  ]




[root@server2 ~]# ls
        elasticsearch-2.3.3.rpm  jdk-8u121-linux-x64.rpm
[root@server2 ~]# rpm -ivh  elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml 
        17 cluster.name: my-es
        23 node.name: server2
        24 node.master: false    # 不做master
        25 node.data: true       # 存储数据
        26 http.enable: true     # 可以查询，如果关闭查询功能，则端口查看不到
        33 path.data: /var/lib/elasticsearch
        43 bootstrap.mlockall: true
        54 network.host: 172.25.44.2
        58 http.port: 9200
        68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]           
[root@server2 ~]# /etc/init.d/elasticsearch start
        Starting elasticsearch:                                    [  OK  ]





[root@server3 ~]# ls
        elasticsearch-2.3.3.rpm  jdk-8u121-linux-x64.rpm
[root@server3 ~]# rpm -ivh  elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml 
        17 cluster.name: my-es
        23 node.name: server3
        25 node.master: false
        26 node.data: true
        27 http.enabled: true
        33 path.data: /var/lib/elasticsearch
        43 bootstrap.mlockall: true
        54 network.host: 172.25.44.3
        58 http.port: 9200
        68 discovery.zen.ping.unicast.hosts: ["server1", "server2","server3"]
[root@server3 ~]# /etc/init.d/elasticsearch start
        Starting elasticsearch:                                    [  OK  ]

这里写图片描述

3.数据采集（logstash）

[root@server1 ~]# rpm -ivh logstash-2.3.3-1.noarch.rpm 
[root@server1 ~]# cd /usr/share/elasticsearch/bin
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin{ } } output { stdout {} }'  #调用input模块,stdin 是终端目录,stdout是终端输出
            Settings: Default pipeline workers: 1
            Pipeline main started
            hello
            2018-08-25T02:42:14.700Z server1 hello
            westos
            2018-08-25T02:42:46.580Z server1 westos
            lalalala
            2018-08-25T02:42:51.582Z server1 lalalala
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin{ } } output { stdout { codec => rubydebug } }'   #codec=>rubydebug ,控制输出,格式转换
        Settings: Default pipeline workers: 1
        Pipeline main started
        hello
        {
               "message" => "hello",
              "@version" => "1",
            "@timestamp" => "2018-08-25T02:47:57.213Z",
                  "host" => "server1"
        }
[root@server1 bin]# /opt/logstash/bin/logstash -e 'input { stdin{ } } output { elasticsearch { hosts => ["172.25.44.1"] index => "logstash-%{+YYYY.MM.dd}" }stdout { codec => rubydebug } }'  #添加elasticsearch模块：hosts，指定主机，index，指定索引
Settings: Default pipeline workers: 1
Pipeline main started
hello 
{
       "message" => "hello ",
      "@version" => "1",
    "@timestamp" => "2018-08-25T03:28:41.315Z",
          "host" => "server1"
}
lalala
{
       "message" => "lalala",
      "@version" => "1",
    "@timestamp" => "2018-08-25T03:29:36.377Z",
          "host" => "server1"
}
hahahaha
{
       "message" => "hahahaha",
      "@version" => "1",
    "@timestamp" => "2018-08-25T03:29:41.547Z",
          "host" => "server1"
}

#

# 在终端上的输入会记录在浏览器中

这里写图片描述

[root@server1 elasticsearch]# cd /etc/logstash/
[root@server1 logstash]# cd conf.d/
[root@server1 conf.d]# ls
[root@server1 conf.d]# pwd
            /etc/logstash/conf.d
[root@server1 conf.d]# vim es.conf
              1 input {
              2         stdin {}
              3 }
              4 
              5 output {
              6         elasticsearch {
              7                 hosts => ["172.25.44.1"]
              8                 index => "logstash-%{+YYYY.MM.dd}"
              9         }
             10         stdout {
             11                 codec => rubydebug
             12         }
             13 }

这里写图片描述

    [root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf   #
            Settings: Default pipeline workers: 1
            Pipeline main started
            Settings: Default pipeline workers: 1
            Pipeline main started
            say hi
            {
                   "message" => "say hi",
                  "@version" => "1",
                "@timestamp" => "2018-08-25T03:38:15.980Z",
                      "host" => "server1"
            }

这里写图片描述

4.文件模块的使用

logstash       # 以logstash身份在运行

终端运行，以root用户运行  
打入后台，脚本运行，不是root用户

[root@server1 conf.d]# cp es.conf message.conf
[root@server1 conf.d]# vim message.conf

这里写图片描述

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf   # 开启之后，网页会生成新的索引文件
Settings: Default pipeline workers: 1
Pipeline main started

这里写图片描述

# 重新打开一个shell,录入数据，日志就会有记录,网页也会有记录
[root@server1 ~]# logger test
[root@server1 ~]# logger hello
[root@server1 ~]# logger hello
[root@server1 ~]# logger hello
[root@server1 ~]# logger hello
[root@server1 ~]# cat /var/log/messages

这里写图片描述

这里写图片描述

# 中断之后再开启，不会从头开始,有隐藏文件来记录文件信息，来确保文件内容是否变化，若修改文件内容，会有相应变化，避免文件重复加载

[root@server1 ~]# l.
        .             .bash_profile  .oracle_jre_usage                          .tcshrc
        ..            .bashrc        .sincedb_452905a167cf4509fd08acb964fdb20c  .viminfo
        .bash_logout  .cshrc         .ssh

[root@server1 ~]# cat .sincedb_452905a167cf4509fd08acb964fdb20c
        1044503 0 64768 32668     

# 想把文件重新加载，必须删掉隐藏文件

5. 在master端把lodstash 作为日志收集器

[root@server1 conf.d]# vim message.conf

这里写图片描述

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf       #打开
[root@server1 ~]# netstat -antlp | grep :514   # 查看端口
        tcp        0      0 :::514                      :::*                        LISTEN      2273/java           




[root@server2 ~]# vim /etc/rsyslog.conf 
82 *.*   @@172.25.44.1:514
[root@server2 ~]# /etc/init.d/rsyslog restart
        Shutting down system logger:                               [  OK  ]
        Starting system logger:                                    [  OK  ]



# 录入数据，master上就会有记录
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos
[root@server2 ~]# logger westos


# 自动记录远程数据，生成日志
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf 
    Settings: Default pipeline workers: 1
    Pipeline main started
    {
               "message" => "imklog 5.8.10, log source = /proc/kmsg started.\n",
              "@version" => "1",
            "@timestamp" => "2018-08-25T03:59:16.000Z",
                  "host" => "172.25.44.2",
              "priority" => 6,
             "timestamp" => "Aug 25 11:59:16",
             "logsource" => "server2",
               "program" => "kernel",
              "severity" => 6,
              "facility" => 0,
        "facility_label" => "kernel",
        "severity_label" => "Informational"
    }
    {
               "message" => "[origin software=\"rsyslogd\" swVersion=\"5.8.10\" x-pid=\"1946\" x-info=\"http://www.rsyslog.com\"] start\n",
              "@version" => "1",
            "@timestamp" => "2018-08-25T03:59:16.000Z",
                  "host" => "172.25.44.2",
              "priority" => 46,
             "timestamp" => "Aug 25 11:59:16",
             "logsource" => "server2",
               "program" => "rsyslogd",
              "severity" => 6,
              "facility" => 5,
        "facility_label" => "syslogd",
        "severity_label" => "Informational"
    }
    {
               "message" => "(root) CMD (run-parts /etc/cron.hourly)\n",
              "@version" => "1",
            "@timestamp" => "2018-08-25T04:01:01.000Z",
                  "host" => "172.25.44.2",
              "priority" => 78,
             "timestamp" => "Aug 25 12:01:01",
             "logsource" => "server2",
               "program" => "CROND",
                   "pid" => "1952",
              "severity" => 6,
              "facility" => 9,
        "facility_label" => "clock",
        "severity_label" => "Informational"
    }
    {
               "message" => "run-parts(/etc/cron.hourly)[1952 starting 0anacron\n",
              "@version" => "1",
            "@timestamp" => "2018-08-25T04:01:01.000Z",
                  "host" => "172.25.44.2",
              "priority" => 77,
             "timestamp" => "Aug 25 12:01:01",
             "logsource" => "server2",
              "severity" => 5,
              "facility" => 9,
        "facility_label" => "clock",
        "severity_label" => "Notice"
    }

6.过滤

类型
关键字
向上匹配

[root@server1 conf.d]# pwd
        /etc/logstash/conf.d
[root@server1 conf.d]# vim message.conf

这里写图片描述
这里写图片描述

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf

这里写图片描述

7.软件的数据采集

(1)httpd
[root@server1 ~]# yum intall -y httpd
[root@server1 ~]# /etc/init.d/httpd start
[root@server1 ~]# vim /var/www/html/index.html
         1 www.westos  server1

这里写图片描述

[root@server1 conf.d]# vim message.conf

这里写图片描述

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf

这里写图片描述
这里写图片描述

# 分类
[root@server1 conf.d]# vim test.conf

这里写图片描述

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf

这里写图片描述

[root@server1 ~]# cd /var/log/
[root@server1 log]# ls
audit     cron   elasticsearch  lastlog   maillog   rhsm    spooler  yum.log
boot.log  dmesg  httpd          logstash  messages  secure  wtmp
[root@server1 log]# cd httpd/
[root@server1 httpd]# ls
access_log  error_log
[root@server1 httpd]# ls -i 
1050095 access_log  1045219 error_log
[root@server1 httpd]# ls -i error_log
1045219 error_log
[root@server1 httpd]# ls -i access_log
1050095 access_log
[root@server1 httpd]# cd
[root@server1 ~]# cat .sincedb_ef0edb00900aaa8dcb520b280cb2fb7d
1050095 0 64768 304
1045219 0 64768 439
[root@server1 ~]# rm -fr .sincedb_ef0edb00900aaa8dcb520b280cb2fb7d


[root@server1 conf.d]# vim message.conf

这里写图片描述

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf

这里写图片描述
这里写图片描述
这里写图片描述
这里写图片描述

(2)数据可视化
[root@server3 ~]# ls
        elasticsearch-2.3.3.rpm  jdk-8u121-linux-x64.rpm  kibana-4.5.1-1.x86_64.rpm
[root@server3 ~]# rpm -ivh kibana-4.5.1-1.x86_64.rpm
[root@server3 ~]# cd /opt/kibana/config/
[root@server3 config]# ls
        kibana.yml
[root@server3 config]# vim kibana.yml 
        15 elasticsearch.url: "http://172.25.44.1:9200"
        23 kibana.index: ".kiban
[root@server3 config]# /etc/init.d/kibana start
[root@server3 config]# netstat -antlp
        tcp        0      0 0.0.0.0:5601                0.0.0.0:*                   LISTEN      1996/node

这里写图片描述
这里写图片描述
这里写图片描述

数据传输过程
logstash  input { nginx }  output { redis } -> logstash input { redis } output { redis }  -> es kibana

#

[root@server2 ~]# ls
elasticsearch-2.3.3.rpm  jdk-8u121-linux-x64.rpm  redis-3.0.6.tar.gz
[root@server2 ~]# tar zxf redis-3.0.6.tar.gz 
[root@server2 ~]# ls
elasticsearch-2.3.3.rpm  jdk-8u121-linux-x64.rpm  redis-3.0.6  redis-3.0.6.tar.gz
[root@server2 ~]# cd redis-3.0.6
[root@server2 redis-3.0.6]# make && make install
[root@server2 redis-3.0.6]# cd utils/
[root@server2 utils]# ./install_server.sh 
[root@server2 redis-3.0.6]# netstat -antlp
[root@server2 utils]# netstat -antulp | grep  :6379
        tcp        0      0 0.0.0.0:6379                0.0.0.0:*                   LISTEN      4975/redis-server * 
        tcp        0      0 :::6379                     :::*                        LISTEN      4975/redis-server * 




[root@server1 ~]# /etc/init.d/httpd stop
[root@server1 ~]# rpm -ivh nginx-1.8.0-1.el6.ngx.x86_64.rpm 
[root@server1 ~]# /etc/init.d/nginx start
[root@server1 ~]# cd /etc/logstash/conf.d/
[root@server1 conf.d]# ls
        es.conf  message.conf  test.conf
[root@server1 conf.d]# cp message.conf  nginx.conf
[root@server1 conf.d]# vim nginx.conf

这里写图片描述

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf   #打开

# 在真机做压侧
[root@foundation44 ~]# ab -c 1 -n 10 http://172.25.44.1/index.html
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf
Settings: Default pipeline workers: 1
Pipeline main started
{
            "message" => "172.25.44.250 - - [25/Aug/2018:16:47:06 +0800] \"GET /index.html HTTP/1.0\" 200 612 \"-\" \"ApacheBench/2.3\" \"-\"",
           "@version" => "1",
         "@timestamp" => "2018-08-25T08:47:07.691Z",
               "path" => "/var/log/nginx/access.log",
               "host" => "server1",
           "clientip" => "172.25.44.250",
              "ident" => "-",
               "auth" => "-",
          "timestamp" => "25/Aug/2018:16:47:06 +0800",
               "verb" => "GET",
            "request" => "/index.html",
        "httpversion" => "1.0",
           "response" => "200",
              "bytes" => "612",
           "referrer" => "\"-\"",
              "agent" => "\"ApacheBench/2.3\"",
    "x_forworded_for" => "\"-\""
}
{
            "message" => "172.25.44.250 - - [25/Aug/2018:16:47:06 +0800] \"GET /index.html HTTP/1.0\" 200 612 \"-\" \"ApacheBench/2.3\" \"-\"",
           "@version" => "1",
         "@timestamp" => "2018-08-25T08:47:07.694Z",
               "path" => "/var/log/nginx/access.log",
               "host" => "server1",
           "clientip" => "172.25.44.250",
              "ident" => "-",
               "auth" => "-",
          "timestamp" => "25/Aug/2018:16:47:06 +0800",
               "verb" => "GET",
            "request" => "/index.html",
        "httpversion" => "1.0",
           "response" => "200",
              "bytes" => "612",
           "referrer" => "\"-\"",
              "agent" => "\"ApacheBench/2.3\"",
    "x_forworded_for" => "\"-\""
}




[root@server2 ~]# rpm -ivh logstash-2.3.3-1.noarch.rpm 
[root@server2 ~]# cd /etc/logstash/conf.d/
[root@server2 conf.d]# ls
        es.conf
[root@server2 conf.d]# vim es.conf

这里写图片描述
这里写图片描述

[root@server2 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

这里写图片描述
这里写图片描述
这里写图片描述
这里写图片描述
这里写图片描述

这里写图片描述
这里写图片描述

[root@foundation44 elk]# ab -c 1 -n 10 http://172.25.44.1/index.html
[root@server2 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf 
Settings: Default pipeline workers: 1
Pipeline main started
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf

这里写图片描述

# 做压侧

这里写图片描述

猜你喜欢

转载自blog.csdn.net/wzt888_/article/details/82050067
今日推荐
开源日报 | Chrome内置Gemini的意义不在于Gemini；中国AI追随之路的五大误区；ECharts创始人“下海”养鱼；谷歌I/O开发者大会什么都有，只是没有惊喜
微软回应中国区AI团队“打包赴美”传闻
基于大语言模型的开源知识库问答系统 MaxKB GitHub Star 数量突破 5,000 个！
美国拟限制 AI 大模型出口中国和俄罗斯
苹果将与 OpenAI 达成协议，将 ChatGPT 应用于 iPhone
openKylin 社区生态委员会第六次会议圆满召开
阿里云正式发布通义千问 2.5
Python 3.13 发布首个 Beta：实验性自由线程模式和 JIT、改进交互式解释器
Stack Overflow 拿我的代码去训练 AI 大模型，还封了我的账号
Pop!_OS 的 COSMIC 桌面完成 App Store 上架工作
《2024 年一季度互联网投融资运行情况》研究报告
报告：Django 仍然是 74% 开发者的首选
周排行
laravle中orm简单的增删改查
文本分类 特征选取之CHI开方检验
Spark核心编程-WordCount
大数据开发实战系列之电信客服(1)
读书笔记 - 把时间当作朋友 by 李笑来
python 笔记--if else
SpringBoot/Mybatis/Druid, 多数据源MultiDataSource配置思路
排序三个整数
redis集群搭建【2】-Windows中Redis集群搭建
STM32F030驱动TM1650点亮4联数码管
每日归档
更多
2024-05-16(6)
2024-05-15(24)
2024-05-14(0)
2024-05-13(18)
2024-05-12(0)
2024-05-11(38)
2024-05-10(38)
2024-05-09(35)
2024-05-08(42)
2024-05-07(14)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022