[kafka]kafka集群实践

环境

ip	hostname	server_id
192.168.1.111	UAT04	2
192.168.1.112	UAT03	1
192.168.1.102	UAT05	3

配置hosts:

192.168.1.112 UAT03
192.168.1.111 UAT04
192.168.1.102 UAT05

安装jdk

# rpm -ivh jdk-8u65-linux-x64.rpm
# java -version
java version "1.8.0_65"
Java(TM) SE Runtime Environment (build 1.8.0_65-b17)
Java HotSpot(TM) 64-Bit Server VM (build 25.65-b01, mixed mode)

下载软件包

# wget http://apache.fayea.com/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz
# wget http://apache.fayea.com/kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz

安装kafka+zookeeper

# tar -xf zookeeper-3.4.8.tar.gz -C /usr/local/
# tar -xf kafka_2.11-0.10.0.1.tgz -C /usr/local/

三台机器都这样操作

配置zookeeper

# cd /usr/local/zookeeper/conf/
# cp zoo_sample.cfg zoo.cfg
# vim zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/local/zookeeper/data
clientPort=2181
dataLogDir=/usr/local/zookeeper/logs
server.1 = 192.168.1.112:3888:4888
server.2 = 192.168.1.111:3888:4888
server.3 = 192.168.1.102:3888:4888
# mkdir /usr/local/zookeeper/{data,logs}
# echo 1 > /usr/local/zookeeper/data/myid
将整个目录拷贝到另外两台机器(注意修改myid、server_id、hostname)

配置kafka

# vim /usr/local/kafka/config/server.properties
broker.id=1
port=9092
host.name=UAT03
advertised.host.name=UAT03
num.network.threads=9
num.io.threads=16
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=16
default.replication.factor=2
num.recovery.threads.per.data.dir=1
log.flush.interval.messages=10000
log.flush.interval.ms=1000
log.retention.hours=720
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
log.cleaner.enable=false
zookeeper.connect=192.168.1.112:2181,192.168.1.111:2181,192.168.1.102:2181
zookeeper.connection.timeout.ms=6000
同时将整个kafka目录拷到另外两台机器，修改broker.id、hostname、

拷贝zookeeper+kafka目录至另外机器

# scp -r zookeeper/ kafka/ 192.168.1.111:/usr/local/
# scp -r zookeeper/ kafka/ 192.168.1.102:/usr/local/
# echo 2 > /usr/local/zookeeper/data/myid
# echo 3 > /usr/local/zookeeper/data/myid
修改192.168.1.111上配置：
/usr/local/kafka/config/server.properties
broker.id=2
host.name=UAT04
advertised.host.name=UAT04
修改192.168.1.111上配置：
/usr/local/kafka/config/server.properties
broker.id=3
host.name=UAT05
advertised.host.name=UAT05

启动zookeeper+kafka集群

# /usr/local/zookeeper/bin/zkServer.sh start & 3台机器都执行
查看zookeeper集群状态：
[root@UAT-03 ~]# /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: follower
[root@UAT-04 ~]# /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: leader
[root@UAT-05 ~]# /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mode: follower
# /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties & 3台机器都执行
[2016-08-16 18:34:04,139] INFO [Kafka Server 1], started (kafka.server.KafkaServer)
[2016-08-16 18:34:36,712] INFO [Kafka Server 2], started (kafka.server.KafkaServer)
[2016-08-16 18:34:42,738] INFO [Kafka Server 3], started (kafka.server.KafkaServer)

上面的启动方式结束会话退出shell时，进程会被杀掉，依赖当前会话，使用下面方式启动

# nohup /usr/local/zookeeper/bin/zkServer.sh start &
# nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties&

简单使用测试

创建topic:

# /usr/local/kafka/bin/kafka-topics.sh --create --zookeeper 192.168.1.111:2181 --replication-factor 1 --partitions 1 --topic test &

列出topic:

# /usr/local/kafka/bin/kafka-topics.sh --list --zookeeper 192.168.1.111:2181
test

使用web监控

KafkaOffsetMonitor

# mkdir /usr/local/kafka_monitor
# cd /usr/local/kafka_monitor 将下载好的jar包上传到此处
# vim mobile_start_en.sh
#!/bin/bash
java -Xms512M -Xmx512M -Xss1024K -XX:PermSize=256m -XX:MaxPermSize=512m -cp KafkaOffsetMonitor-assembly-0.2.0.jar com.quantifind.kafka.offsetapp.OffsetGetterWeb --zk 192.168.1.1
12:2181,192.168.1.111:2182,192.168.1.102:2181 --port 8086 --refresh 10.seconds --retain 7.days 1>mobile-logs/stdout.log 2>mobile-logs/stderr.log &
# mkdir mobile-logs
# chmod +x mobile_start_en.sh
# ./mobile_start_en.sh
访问：http://192.168.1.112:8086/

Zookeeper for systemd service

[Unit]
Description=zookeeper.service
After=network.target
[Service]
User=zk
Type=forking
Environment=ZOO_LOG_DIR=/data/zk/zookeeper_logs
Environment=JAVA_HOME=/usr/java/jdk1.8.0_121
ExecStart=/usr/local/zookeeper/bin/zkServer.sh start
ExecStop=/usr/local/zookeeper/bin/zkServer.sh stop
ExecReload=/usr/local/zookeeper/bin/zkServer.sh restart
[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl restart zookeeper
systemctl enable zookeeper

Kafka for systemd service

[Unit]
Description=kafka.service
After=network.target
[Service]
User=zk
Type=simple
PIDFile=/var/run/kafka.pid
Environment=KAFKA_LOG_DIR=/data/zk/kafka_logs
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
ExecStop=/usr/local/kafka/bin/kafka-server-stop.sh
ExecReload=$ExecStop;$ExecStart
[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl restart kafka
systemctl enable kafka

测试生产消费

/usr/local/kafka/bin/kafka-console-producer.sh --broker-list zk1:9092,zk2:9092,zk3:9092 --topic test1 生产
/usr/local/kafka/bin/kafka-console-consumer.sh --zookeeper zk1:2181,zk2:2181,zk3:2181 --from-beginning --topic test1 消费

kafka消息监控

可以使用kafka-manager来管理集群