CentOS下部署kakfa集群
zookeeper版本：zookeeper-3.4.13
kakfa版本：kafka_2.11-2.0.0
1 环境准备
在已有zookeeper集群上，部署kakfa集群。
zookeeper集群主机信息如下：
| hostname | ip | 端口 |
|---|---|---|
| master.hadoop.com | 192.168.162.180 | 2181/2881/3881 |
| slave1.hadoop.com | 192.168.162.181 | 2181/2881/3881 |
| slave2.hadoop.com | 192.168.162.182 | 2181/2881/3881 |
2 安装kafka
2.1 下载kafka
下载地址：http://mirrors.hust.edu.cn/apache/kafka/2.0.0/kafka_2.11-2.0.0.tgz
wget http://mirrors.hust.edu.cn/apache/kafka/2.0.0/kafka_2.11-2.0.0.tgz解压kafka
tar -zxvf kafka_2.11-2.0.0.tgz移动到/root/apps下并重命名kafka
mv kafka_2.11-2.0.0 /root/apps/kafka2.2 修改配置文件
以下操作只在master主机上进行，完成后将kafka目录scp到其它两台机子
2.2.1 创建日志目录
进入kafka目录
cd /root/apps/kafka在此目录创建kafkalogs目录用来存放kafka的日志数据
mkdir kafkalogs2.2.2 修改配置文件
vi /config/server.properties修改配置说明
broker.id=0 # 设置broker.id为0,其它两个节点需修改为1，2
listeners=PLAINTEXT://:9092 #监听端口为9092
advertised.listeners=PLAINTEXT://master.hadoop.com:9092 # 设置hostname
log.dirs=/root/apps/kafka/kafkalogs # 日志数据储存位置
zookeeper.connect=master.hadoop.com:2181,slave1.hadoop.com:2181,slave2.hadoop.com:2181 # zookeeper的ip和端口2.3 主机分发
在master主机上将修改后的kafka文件夹分发到其它两台主机上,并修改配置文件中的broker.id 和hostname
scp -r /root/apps/kafka/ slave1.hadoop.com:/root/apps/
scp -r /root/apps/kafka/ slave2.hadoop.com:/root/apps/2.4 启动kafka
启动kakfa之前，先启动zookeeper。然后再启动kafka
sh /root/apps/kafka/bin/kafka-server-start.sh -daemon /root/apps/kafka/config/server.properties启动脚本
#!/bin/bash
#master
echo 'start master zookeeper'
sh /root/apps/zookeeper/bin/zkServer.sh start
echo 'start slave1 zookeeper'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/zookeeper/bin/zkServer.sh start
exit
eeooff
echo 'start slave2 zookeeper'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/zookeeper/bin/zkServer.sh start
exit
eeooff
echo 'start mastet kafka'
sh /root/apps/kafka/bin/kafka-server-start.sh -daemon /root/apps/kafka/config/server.properties
echo 'start slave1 kafka'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/kafka/bin/kafka-server-start.sh -daemon /root/apps/kafka/config/server.properties
exit
eeooff
echo 'start slave2 kafka'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/kafka/bin/kafka-server-start.sh -daemon /root/apps/kafka/config/server.properties
exit
eeooff停止脚本
#!/bin/bash
# stop kafka
echo 'stop master kafka'
sh /root/apps/kafka/bin/kafka-server-stop.sh
echo 'stop slave1 kafka'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/kafka/bin/kafka-server-stop.sh
exit
eeooff
echo 'stop slave2 kafka'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/kafka/bin/kafka-server-stop.sh
exit
eeooff
# stop zookeeper
echo 'stop master zookeeper'
sh /root/apps/zookeeper/bin/zkServer.sh stop
echo 'stop slave1 zookeeper'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/zookeeper/bin/zkServer.sh stop
exit
eeooff
echo 'stop slave2 zookeeper'
ssh [email protected] > /dev/null 2>&1 << eeooff
sh /root/apps/zookeeper/bin/zkServer.sh stop
exit
eeooff