topic的创建见 https://blog.csdn.net/qq_26437925/article/details/97611693
发数据到kafka
void send(){
Properties props = new Properties();
props.put("bootstrap.servers", "172.16.227.250:9092,172.16.227.129:9092,172.16.227.130:9092");
props.put("acks", "all");
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
Producer<String, String> producer = new KafkaProducer<>(props);
int n = 100;
for (int i = 0; i < n; i++) {
producer.send(new ProducerRecord<>("chat", "" + Integer.toString(i), Integer.toString(i)));
}
producer.close();
System.out.println("producer closed");
}
多线程消费
void multiConsumer(){
int numConsumers = 4;
String groupId = "topic-chat-group-test";
List<String> topics = Arrays.asList("chat");
ExecutorService executor = Executors.newFixedThreadPool(numConsumers);
final List<KafkaConsumerRunner> consumers = new ArrayList<>(4);
for (int i = 0; i < numConsumers; i++) {
KafkaConsumerRunner consumer = new KafkaConsumerRunner(i, groupId, topics);
consumers.add(consumer);
executor.submit(consumer);
}
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
for (KafkaConsumerRunner consumer : consumers) {
consumer.shutdown();
}
executor.shutdown();
try {
executor.awaitTermination(1000, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
});
}
class KafkaConsumerRunner implements Runnable {
KafkaConsumer<String, String> consumer;
List<String> topics = null;
int id = 0;
public KafkaConsumerRunner(){
Properties props = new Properties();
props.setProperty("bootstrap.servers", "172.16.227.250:9092,172.16.227.129:9092,172.16.227.130:9092");
// props.setProperty("zookeeper.connect", "172.16.227.250:2181,172.16.227.129:2181,172.16.227.130:2181");
props.setProperty("group.id", "chat-room-7");//消费组id
props.setProperty("enable.auto.commit", "true");
// props.setProperty("auto.commit.interval.ms", "1000");
props.put("auto.offset.reset", "earliest");
props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
this.consumer = new KafkaConsumer<>(props);
}
public KafkaConsumerRunner(int id, String groupId, List<String> topics) {
this.id = id;
this.topics = topics;
Properties props = new Properties();
props.setProperty("bootstrap.servers", "172.16.227.250:9092,172.16.227.129:9092,172.16.227.130:9092");
props.put("group.id", groupId);
props.put("auto.offset.reset", "earliest");
props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
this.consumer = new KafkaConsumer<>(props);
}
@Override
public void run() {
try {
consumer.subscribe(topics);
while (true) {
ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(Long.MAX_VALUE));
for (ConsumerRecord<String, String> record : records) {
Map<String, Object> data = new HashMap<>(16);
data.put("partition", record.partition());
data.put("offset", record.offset());
data.put("value", record.value());
System.out.println(this.id + ": " + data);
}
}
} catch (WakeupException e) {
// ignore for shutdown
} finally {
consumer.close();
}
}
public void shutdown() {
consumer.wakeup();
}
}
output
- 4个线程
2: {partition=2, offset=0, value=0}
0: {partition=0, offset=0, value=1}
0: {partition=0, offset=1, value=5}
0: {partition=0, offset=2, value=7}
0: {partition=0, offset=3, value=8}
0: {partition=0, offset=4, value=11}
0: {partition=0, offset=5, value=15}
0: {partition=0, offset=6, value=17}
0: {partition=0, offset=7, value=21}
0: {partition=0, offset=8, value=22}
0: {partition=0, offset=9, value=23}
0: {partition=0, offset=10, value=25}
0: {partition=0, offset=11, value=27}
0: {partition=0, offset=12, value=28}
0: {partition=0, offset=13, value=30}
0: {partition=0, offset=14, value=33}
2: {partition=2, offset=1, value=2}
2: {partition=2, offset=2, value=3}
2: {partition=2, offset=3, value=9}
2: {partition=2, offset=4, value=16}
2: {partition=2, offset=5, value=29}
2: {partition=2, offset=6, value=32}
2: {partition=2, offset=7, value=36}
2: {partition=2, offset=8, value=40}
2: {partition=2, offset=9, value=41}
2: {partition=2, offset=10, value=49}
2: {partition=2, offset=11, value=50}
2: {partition=2, offset=12, value=57}
2: {partition=2, offset=13, value=69}
2: {partition=2, offset=14, value=73}
2: {partition=2, offset=15, value=74}
2: {partition=2, offset=16, value=78}
2: {partition=2, offset=17, value=83}
2: {partition=2, offset=18, value=85}
2: {partition=2, offset=19, value=90}
2: {partition=2, offset=20, value=91}
2: {partition=2, offset=21, value=97}
0: {partition=0, offset=15, value=34}
0: {partition=0, offset=16, value=37}
0: {partition=0, offset=17, value=43}
0: {partition=0, offset=18, value=44}
0: {partition=0, offset=19, value=45}
0: {partition=0, offset=20, value=47}
1: {partition=1, offset=0, value=4}
1: {partition=1, offset=1, value=6}
1: {partition=1, offset=2, value=10}
1: {partition=1, offset=3, value=12}
1: {partition=1, offset=4, value=13}
1: {partition=1, offset=5, value=14}
1: {partition=1, offset=6, value=18}
1: {partition=1, offset=7, value=19}
1: {partition=1, offset=8, value=20}
1: {partition=1, offset=9, value=24}
1: {partition=1, offset=10, value=26}
1: {partition=1, offset=11, value=31}
1: {partition=1, offset=12, value=35}
1: {partition=1, offset=13, value=38}
1: {partition=1, offset=14, value=39}
1: {partition=1, offset=15, value=42}
1: {partition=1, offset=16, value=46}
1: {partition=1, offset=17, value=51}
1: {partition=1, offset=18, value=53}
1: {partition=1, offset=19, value=58}
1: {partition=1, offset=20, value=62}
1: {partition=1, offset=21, value=65}
1: {partition=1, offset=22, value=66}
1: {partition=1, offset=23, value=67}
1: {partition=1, offset=24, value=68}
1: {partition=1, offset=25, value=75}
1: {partition=1, offset=26, value=77}
1: {partition=1, offset=27, value=81}
1: {partition=1, offset=28, value=84}
1: {partition=1, offset=29, value=86}
1: {partition=1, offset=30, value=87}
1: {partition=1, offset=31, value=94}
1: {partition=1, offset=32, value=95}
0: {partition=0, offset=21, value=48}
0: {partition=0, offset=22, value=52}
0: {partition=0, offset=23, value=54}
0: {partition=0, offset=24, value=55}
0: {partition=0, offset=25, value=56}
0: {partition=0, offset=26, value=59}
0: {partition=0, offset=27, value=60}
0: {partition=0, offset=28, value=61}
0: {partition=0, offset=29, value=63}
0: {partition=0, offset=30, value=64}
0: {partition=0, offset=31, value=70}
0: {partition=0, offset=32, value=71}
0: {partition=0, offset=33, value=72}
0: {partition=0, offset=34, value=76}
0: {partition=0, offset=35, value=79}
0: {partition=0, offset=36, value=80}
0: {partition=0, offset=37, value=82}
0: {partition=0, offset=38, value=88}
0: {partition=0, offset=39, value=89}
0: {partition=0, offset=40, value=92}
0: {partition=0, offset=41, value=93}
0: {partition=0, offset=42, value=96}
0: {partition=0, offset=43, value=98}
0: {partition=0, offset=44, value=99}
- 2个线程
1: {partition=2, offset=0, value=0}
1: {partition=2, offset=1, value=2}
0: {partition=0, offset=0, value=1}
0: {partition=0, offset=1, value=5}
0: {partition=0, offset=2, value=7}
0: {partition=0, offset=3, value=8}
0: {partition=0, offset=4, value=11}
0: {partition=0, offset=5, value=15}
0: {partition=0, offset=6, value=17}
1: {partition=2, offset=2, value=3}
1: {partition=2, offset=3, value=9}
0: {partition=0, offset=7, value=21}
1: {partition=2, offset=4, value=16}
1: {partition=2, offset=5, value=29}
1: {partition=2, offset=6, value=32}
1: {partition=2, offset=7, value=36}
1: {partition=2, offset=8, value=40}
1: {partition=2, offset=9, value=41}
1: {partition=2, offset=10, value=49}
0: {partition=0, offset=8, value=22}
1: {partition=2, offset=11, value=50}
0: {partition=0, offset=9, value=23}
1: {partition=2, offset=12, value=57}
0: {partition=0, offset=10, value=25}
1: {partition=2, offset=13, value=69}
0: {partition=0, offset=11, value=27}
1: {partition=2, offset=14, value=73}
0: {partition=0, offset=12, value=28}
1: {partition=2, offset=15, value=74}
0: {partition=0, offset=13, value=30}
1: {partition=2, offset=16, value=78}
0: {partition=0, offset=14, value=33}
1: {partition=2, offset=17, value=83}
0: {partition=0, offset=15, value=34}
1: {partition=2, offset=18, value=85}
1: {partition=2, offset=19, value=90}
1: {partition=2, offset=20, value=91}
1: {partition=2, offset=21, value=97}
0: {partition=0, offset=16, value=37}
0: {partition=0, offset=17, value=43}
0: {partition=0, offset=18, value=44}
0: {partition=0, offset=19, value=45}
0: {partition=0, offset=20, value=47}
0: {partition=0, offset=21, value=48}
0: {partition=0, offset=22, value=52}
0: {partition=0, offset=23, value=54}
0: {partition=0, offset=24, value=55}
0: {partition=0, offset=25, value=56}
0: {partition=0, offset=26, value=59}
0: {partition=0, offset=27, value=60}
0: {partition=0, offset=28, value=61}
0: {partition=0, offset=29, value=63}
0: {partition=0, offset=30, value=64}
0: {partition=0, offset=31, value=70}
0: {partition=0, offset=32, value=71}
0: {partition=0, offset=33, value=72}
0: {partition=0, offset=34, value=76}
0: {partition=0, offset=35, value=79}
0: {partition=0, offset=36, value=80}
0: {partition=0, offset=37, value=82}
0: {partition=0, offset=38, value=88}
0: {partition=0, offset=39, value=89}
0: {partition=0, offset=40, value=92}
0: {partition=0, offset=41, value=93}
0: {partition=0, offset=42, value=96}
0: {partition=0, offset=43, value=98}
0: {partition=0, offset=44, value=99}
0: {partition=1, offset=0, value=4}
0: {partition=1, offset=1, value=6}
0: {partition=1, offset=2, value=10}
0: {partition=1, offset=3, value=12}
0: {partition=1, offset=4, value=13}
0: {partition=1, offset=5, value=14}
0: {partition=1, offset=6, value=18}
0: {partition=1, offset=7, value=19}
0: {partition=1, offset=8, value=20}
0: {partition=1, offset=9, value=24}
0: {partition=1, offset=10, value=26}
0: {partition=1, offset=11, value=31}
0: {partition=1, offset=12, value=35}
0: {partition=1, offset=13, value=38}
0: {partition=1, offset=14, value=39}
0: {partition=1, offset=15, value=42}
0: {partition=1, offset=16, value=46}
0: {partition=1, offset=17, value=51}
0: {partition=1, offset=18, value=53}
0: {partition=1, offset=19, value=58}
0: {partition=1, offset=20, value=62}
0: {partition=1, offset=21, value=65}
0: {partition=1, offset=22, value=66}
0: {partition=1, offset=23, value=67}
0: {partition=1, offset=24, value=68}
0: {partition=1, offset=25, value=75}
0: {partition=1, offset=26, value=77}
0: {partition=1, offset=27, value=81}
0: {partition=1, offset=28, value=84}
0: {partition=1, offset=29, value=86}
0: {partition=1, offset=30, value=87}
0: {partition=1, offset=31, value=94}
0: {partition=1, offset=32, value=95}
- 线程0 消费了partition 1; 线程 1消费了 partition 0, partition 2