手把手教你实现kafka集群搭建

选择搭建主机
选择node1,node2,node3搭建集群

1.解压

[root@node1 apps]# tar -zxvf kafka_2.11-0.11.0.3.tgz -C /opt

2.修改配置

[root@node1 config]# vim server.properties

2.1将log目录修改为/kafaka-logs

2.2修改zookeeper

2.3编写启动脚本

nohup bin/kafka-server-start.sh   config/server.properties > kafka.log 2>&1 &

放到根目录下

授权脚本
将软件发送到node2和node3上

[root@node1 opt]# scp -r kafka_2.11-0.11.0.3/ node3:`pwd`
[root@node1 opt]# scp -r kafka_2.11-0.11.0.3/ node2:`pwd`

2.4修改node2和node3的service.properties

[root@node2 ~]# cd /opt/kafka_2.11-0.11.0.3/config/
[root@node2 config]# vim server.properties

node2的改为

node3改为2

3.启动

node1,node2,node3

[root@node1 kafka_2.11-0.11.0.3]# ./startkafka.sh
[root@node2 kafka_2.11-0.11.0.3]# ./startkafka.sh
[root@node3 kafka_2.11-0.11.0.3]# ./startkafka.sh

启动集群成功

4.测试

[root@node1 bin]# ./kafka-topics.sh --zookeeper node2,node3,node4 --create --topic t0210 --partitions 3 --replication-factor 3
Created topic "t0210".
[root@node1 bin]# ./kafka-topics.sh --zookeeper node2,node3,node4 --list
t0210

让node1作为生产者生产数据
node2作为消费者消费数据

[root@node1 bin]# ./kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic t0210
>

[root@node2 bin]# ./kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --topic t0210

查看数据

[root@node3 bin]# ./kafka-consumer-groups.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --list
Note: This will only show information about consumers that use the Java consumer API (non-ZooKeeper-based consumers).

console-consumer-91847
[root@node3 bin]# ./kafka-consumer-groups.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --describe --group console-consumer-91847
Note: This will only show information about consumers that use the Java consumer API (non-ZooKeeper-based consumers).


TOPIC                          PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG        CONSUMER-ID                                       HOST                           CLIENT-ID
t0210                          0          0               0               0          consumer-1-c0e7bb36-59aa-40ad-a832-e79648def3d4   /192.168.40.62                 consumer-1
t0210                          1          1               1               0          consumer-1-c0e7bb36-59aa-40ad-a832-e79648def3d4   /192.168.40.62                 consumer-1
t0210                          2          1               1               0          consumer-1-c0e7bb36-59aa-40ad-a832-e79648def3d4   /192.168.40.62                 consumer-1
[root@node3 bin]#