kafka学习笔记六可靠的数据传输机制

Reliable Data Delivery

转自：https://blog.csdn.net/u013256816/article/details/71091774

kafka自带测试工具的使用

org.apache.kafka.tools.VerifiableProducer

org.apache.kafka.tools.VerifiableConsumer

创建topic test22

unclean.leader.election.enable=true

min.insync.replicas=1

[root@iZbp14ouog5ocoeakj39prZ kafka_2.11-0.11.0.1]# bin/kafka-topics.sh --create --topic test22 --partitions 1 --replication-factor 3 --zookeeper 10.174.32.122:2181,10.117.15.224:2181,10.168.96.198:2181 --config unclean.leader.election.enable=true --config min.insync.replicas=1 
Created topic "test22".
[root@iZbp14ouog5ocoeakj39prZ kafka_2.11-0.11.0.1]# bin/kafka-topics.sh --describe --topic test22 --zookeeper 10.174.32.122:2181
Topic:test22    PartitionCount:1        ReplicationFactor:3     Configs:unclean.leader.election.enable=true,min.insync.replicas=1
        Topic: test22   Partition: 0    Leader: 2       Replicas: 2,0,1 Isr: 2,0,1
[root@iZbp14ouog5ocoeakj39prZ kafka_2.11-0.11.0.1]# 

运行org.apache.kafka.tools.VerifiableConsumer 等待Producer生产者工作，并打印接收的数据日志

[root@iZbp14ouog5ocoeakj39prZ kafka_2.11-0.11.0.1]# bin/kafka-run-class.sh org.apache.kafka.tools.VerifiableConsumer --topic test22 --broker-list 10.174.32.122:9092,10.117.15.224:9092,10.168.96.198:9092 --group-id hw_test --enable-autocommit
{"timestamp":1535014906925,"name":"startup_complete"}
{"timestamp":1535014907119,"partitions":[],"name":"partitions_revoked"}
{"timestamp":1535014907154,"partitions":[{"topic":"test22","partition":0}],"name":"partitions_assigned"}
{"timestamp":1535014912525,"count":9,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":9,"minOffset":0,"maxOffset":8}]}
{"timestamp":1535014912543,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":9,"maxOffset":9}]}
{"timestamp":1535014912575,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":10,"maxOffset":10}]}
{"timestamp":1535014912624,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":11,"maxOffset":11}]}
{"timestamp":1535014912680,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":12,"maxOffset":12}]}
{"timestamp":1535014912725,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":13,"maxOffset":13}]}
{"timestamp":1535014912776,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":14,"maxOffset":14}]}
{"timestamp":1535014912826,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":15,"maxOffset":15}]}
{"timestamp":1535014912877,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":16,"maxOffset":16}]}
{"timestamp":1535014912928,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":17,"maxOffset":17}]}
{"timestamp":1535014912978,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":18,"maxOffset":18}]}
{"timestamp":1535014913030,"count":1,"name":"records_consumed","partitions":[{"topic":"test22","partition":0,"count":1,"minOffset":19,"maxOffset":19}]}

运行org.apache.kafka.tools.VerifiableConsumer配置参数生产数据

[root@iZbp14ouog5ocoeakj39prZ kafka_2.11-0.11.0.1]# bin/kafka-run-class.sh org.apache.kafka.tools.VerifiableProducer --topic test22 --broker-list 10.174.32.122:9092,10.117.15.224:9092,10.168.96.198:9092 --max-messages 20 --throughput 20 --acks 1 --value-prefix 123
{"name":"startup_complete"}
{"partition":0,"offset":0,"time_ms":1535014912496,"name":"producer_send_success","topic":"test22","value":"123.0","key":null}
{"partition":0,"offset":1,"time_ms":1535014912518,"name":"producer_send_success","topic":"test22","value":"123.1","key":null}
{"partition":0,"offset":2,"time_ms":1535014912521,"name":"producer_send_success","topic":"test22","value":"123.2","key":null}
{"partition":0,"offset":3,"time_ms":1535014912524,"name":"producer_send_success","topic":"test22","value":"123.3","key":null}
{"partition":0,"offset":4,"time_ms":1535014912526,"name":"producer_send_success","topic":"test22","value":"123.4","key":null}
{"partition":0,"offset":5,"time_ms":1535014912528,"name":"producer_send_success","topic":"test22","value":"123.5","key":null}
{"partition":0,"offset":6,"time_ms":1535014912530,"name":"producer_send_success","topic":"test22","value":"123.6","key":null}
{"partition":0,"offset":7,"time_ms":1535014912534,"name":"producer_send_success","topic":"test22","value":"123.7","key":null}
{"partition":0,"offset":8,"time_ms":1535014912536,"name":"producer_send_success","topic":"test22","value":"123.8","key":null}
{"partition":0,"offset":9,"time_ms":1535014912541,"name":"producer_send_success","topic":"test22","value":"123.9","key":null}
{"partition":0,"offset":10,"time_ms":1535014912571,"name":"producer_send_success","topic":"test22","value":"123.10","key":null}
{"partition":0,"offset":11,"time_ms":1535014912623,"name":"producer_send_success","topic":"test22","value":"123.11","key":null}
{"partition":0,"offset":12,"time_ms":1535014912678,"name":"producer_send_success","topic":"test22","value":"123.12","key":null}
{"partition":0,"offset":13,"time_ms":1535014912724,"name":"producer_send_success","topic":"test22","value":"123.13","key":null}
{"partition":0,"offset":14,"time_ms":1535014912774,"name":"producer_send_success","topic":"test22","value":"123.14","key":null}
{"partition":0,"offset":15,"time_ms":1535014912824,"name":"producer_send_success","topic":"test22","value":"123.15","key":null}
{"partition":0,"offset":16,"time_ms":1535014912875,"name":"producer_send_success","topic":"test22","value":"123.16","key":null}
{"partition":0,"offset":17,"time_ms":1535014912925,"name":"producer_send_success","topic":"test22","value":"123.17","key":null}
{"partition":0,"offset":18,"time_ms":1535014912976,"name":"producer_send_success","topic":"test22","value":"123.18","key":null}
{"partition":0,"offset":19,"time_ms":1535014913027,"name":"producer_send_success","topic":"test22","value":"123.19","key":null}
{"name":"shutdown_complete"}
{"avg_throughput":19.98001998001998,"target_throughput":20,"name":"tool_data","sent":20,"acked":20}
[root@iZbp14ouog5ocoeakj39prZ kafka_2.11-0.11.0.1]#