ELK installation and deployment

ELK installation and deployment

### CentOS 7.2

yum install -y vim wget

#rpm -ivh http://indoor.meilele.com/download/centos/jdk-8u101-linux-x64.rpm
#java -version


###安装java jdk
yum -y install java-1.8.0-openjdk
java -version

echo "export JAVA_HOME=$(rpm -ql  java-1.8.0-openjdk |grep jvm |head -n1 |awk -F '/jre' '{print $1}')" >>/etc/profile
echo '
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
' >>/etc/profile


source /etc/profile

echo $JAVA_HOME
#################



######卸载命令
userdel elasticsearch
userdel logstash
userdel kibana
yum remove -y elasticsearch logstash kibana
rm -rf /etc/{elasticsearch,logstash,kibana} /etc/yum.repos.d/elk.repo
find / \( -name elasticsearch -o -name logstash \) -exec rm -rf {} \;

##########

wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/yum/elastic-6.x/6.7.0/elasticsearch-6.7.0.rpm
wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/yum/elastic-6.x/6.7.0/logstash-6.7.0.rpm
wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/yum/elastic-6.x/6.7.0/kibana-6.7.0-x86_64.rpm

rpm -ivh elasticsearch-6.7.0.rpm
rpm -ivh logstash-6.7.0.rpm
rpm -ivh kibana-6.7.0-x86_64.rpm


#############
wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/yum/elastic-6.x/6.1.0/elasticsearch-6.1.0.rpm
rpm -ivh elasticsearch-6.1.0.rpm

wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/yum/elastic-6.x/6.1.0/logstash-6.1.0.rpm
rpm -ivh logstash-6.1.0.rpm

wget https://mirrors.tuna.tsinghua.edu.cn/elasticstack/yum/elastic-6.x/6.1.0/kibana-6.1.0-x86_64.rpm
rpm -ivh kibana-6.1.0-x86_64.rpm
########


systemctl enable elasticsearch
systemctl restart elasticsearch

systemctl status elasticsearch

\cp -f /etc/elasticsearch/elasticsearch.yml{,.bak}

grep -Ev "^#|^$" /etc/elasticsearch/elasticsearch.yml.bak >/etc/elasticsearch/elasticsearch.yml

echo "
network.host: 0.0.0.0
http.port: 9200
bootstrap.memory_lock: false
bootstrap.system_call_filter: false

cluster.name: log_online
node.name: $(hostname)
node.master: true
node.data: true
#path.data: /data/esdata
discovery.zen.ping.unicast.hosts: [ ' 172.16.3.232:9300 ' , ' 172.16.3.233:9300 ' , ' 172.16.3.234:9300 ' , ' 172.16.3.235:9300 ' ]
 " >> / etc / elasticsearch / elasticsearch.yml 


systemctl elasticsearch restart 

systemctl Status elasticsearch 

PS -ef | grep the Java 

netstat -lntp | grep the Java 

#### health check- 
curl ' localhost:? 9200 / _cluster / health Pretty '  


##### cluster details 
curl ' localhost: 9200 / _cluster / state? pretty'     

######################################## elasticsearch - head 
yum  the install - Y Git NPM 

# command-line tool to install grunt grunt - cli 
npm install -g grunt- cli 

# grunt and install the plug- 
npm install grunt --save- dev 

# View installed versions circumstances 
grunt - version 
 
#### download and install elasticsearch - head 
# choice execute the following command in a directory, do not plug clone after head into the plugins directory elasticsearch, otherwise it will error 

#git download 
git clone git: // github.com/mobz/elasticsearch-head.git 

# enter the directory 
cd elasticsearch - head

# If npm use a lot of mistakes, try to use CNPM 
npm install   

####### npm ERR PhantomJS-prebuilt @! 2.1 . 16  install : `the Node install .js` 
### npm -g install PhantomJS-prebuilt @ 2.1 . 16 --ignore- Script 

# and mounted to the module stored grunt 
NPM the install grunt - save   

#### elasticsearch modified config installation directory under / elasticsearch.yml
 echo  '
 http.cors.enabled: to true 
http.cors.allow - Origin: " * " 
' >>/etc/elasticsearch/elasticsearch.yml


##npm run start
##sed -i '/port:/ i hostname : "*",' Gruntfile.js
#echo "cd $(pwd);grunt server &" >>/etc/rc.local


cd elasticsearch-head ;grunt server &
systemctl restart elasticsearch




################################## logstash
#日志收集节点

systemctl enable logstash

systemctl restart logstash

###172.16.3.64 - - [15/Oct/2019:13:00:00 +0800] "GET /favicon.ico HTTP/1.1" 404 555 "http://172.16.3.232/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36" "-"

echo '
input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    type => "nginx"
  }
}
filter {
    grok {
        match => { "message" => "%{IP:clientip} - %{USER:user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" %{NUMBER:response} (?:%{NUMBER:bytes_read}|-) \"%{DATA:referrer}\" \"%{DATA:agent}\" \"%{DATA:xforwardedfor}\""}
    }
    geoip {
        source => "clientip"
    }
}
output {
    stdout { codec => rubydebug }
    elasticsearch {
        hosts => ["172.16.3.232:9200"]
        index => "nginx-test-%{+YYYY.MM.dd}"
  }
}
' >/etc/logstash/conf.d/nginx.conf


systemctl restart logstash

systemctl status logstash


###########################kibana



\cp /etc/kibana/kibana.yml{,.bak}


echo '
server.host: 0.0.0.0
elasticsearch.url: "http://localhost:9200"
logging.dest: /var/log/kibana.log
' >/etc/kibana/kibana.yml

grep -Ev '# ^ | ^ $ '/ etc / kibana / kibana.yml 

Touch / var / log / kibana.log
 chmod  777 / var / log / kibana.log 


systemctl enable kibana 
systemctl restart kibana 
systemctl Status kibana 

netstat -lntp | grep the Node 


#### HTTP: // 172.16.3.232:5601 access the browser http://172.16.3.232 : 5601 / App / kibana 

############################ nginx ### 


RPM -Uvh HTTP: // nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm 
yum  install - the y-nginx 

systemctl Start nginx .service 
systemctl enable nginx.service 

Sed -i " S / available for purchase to Nginx / Nginx available for purchase to $ (hostname) / G " / usr / Share / Nginx / HTML / index.html 



#### kibana . 6 .7 Speaking above the following command can be used
 echo  ' i18n.locale: "the CN-ZH" ' >> / etc / kibana / kibana.yml 
systemctl the restart kibana 
systemctl Status kibana