KafkaProducer & KafkaConsumer API 使用- 多线程消费多patition

topic的创建见 https://blog.csdn.net/qq_26437925/article/details/97611693

发数据到kafka

void send(){
        Properties props = new Properties();
        props.put("bootstrap.servers", "172.16.227.250:9092,172.16.227.129:9092,172.16.227.130:9092");
        props.put("acks", "all");
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        Producer<String, String> producer = new KafkaProducer<>(props);
        int n = 100;
        for (int i = 0; i < n; i++) {
            producer.send(new ProducerRecord<>("chat", "" + Integer.toString(i), Integer.toString(i)));
        }
        producer.close();
        System.out.println("producer closed");
    }

多线程消费

 void multiConsumer(){
        int numConsumers = 4;
        String groupId = "topic-chat-group-test";
        List<String> topics = Arrays.asList("chat");
        ExecutorService executor = Executors.newFixedThreadPool(numConsumers);

        final List<KafkaConsumerRunner> consumers = new ArrayList<>(4);
        for (int i = 0; i < numConsumers; i++) {
            KafkaConsumerRunner consumer = new KafkaConsumerRunner(i, groupId, topics);
            consumers.add(consumer);
            executor.submit(consumer);
        }

        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                for (KafkaConsumerRunner consumer : consumers) {
                    consumer.shutdown();
                }
                executor.shutdown();
                try {
                    executor.awaitTermination(1000, TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        });
    }

class KafkaConsumerRunner implements Runnable {
    KafkaConsumer<String, String> consumer;
    List<String> topics = null;
    int id = 0;

    public KafkaConsumerRunner(){
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "172.16.227.250:9092,172.16.227.129:9092,172.16.227.130:9092");
//        props.setProperty("zookeeper.connect", "172.16.227.250:2181,172.16.227.129:2181,172.16.227.130:2181");
        props.setProperty("group.id", "chat-room-7");//消费组id
        props.setProperty("enable.auto.commit", "true");
//        props.setProperty("auto.commit.interval.ms", "1000");
        props.put("auto.offset.reset", "earliest");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        this.consumer = new KafkaConsumer<>(props);
    }
    public KafkaConsumerRunner(int id, String groupId, List<String> topics) {
        this.id = id;
        this.topics = topics;
        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "172.16.227.250:9092,172.16.227.129:9092,172.16.227.130:9092");
        props.put("group.id", groupId);
        props.put("auto.offset.reset", "earliest");
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        this.consumer = new KafkaConsumer<>(props);
    }

    @Override
    public void run() {
        try {
            consumer.subscribe(topics);
            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(Long.MAX_VALUE));
                for (ConsumerRecord<String, String> record : records) {
                    Map<String, Object> data = new HashMap<>(16);
                    data.put("partition", record.partition());
                    data.put("offset", record.offset());
                    data.put("value", record.value());
                    System.out.println(this.id + ": " + data);
                }
            }
        } catch (WakeupException e) {
            // ignore for shutdown
        } finally {
            consumer.close();
        }
    }

    public void shutdown() {
        consumer.wakeup();
    }
}

output

• 4个线程

2: {partition=2, offset=0, value=0}
0: {partition=0, offset=0, value=1}
0: {partition=0, offset=1, value=5}
0: {partition=0, offset=2, value=7}
0: {partition=0, offset=3, value=8}
0: {partition=0, offset=4, value=11}
0: {partition=0, offset=5, value=15}
0: {partition=0, offset=6, value=17}
0: {partition=0, offset=7, value=21}
0: {partition=0, offset=8, value=22}
0: {partition=0, offset=9, value=23}
0: {partition=0, offset=10, value=25}
0: {partition=0, offset=11, value=27}
0: {partition=0, offset=12, value=28}
0: {partition=0, offset=13, value=30}
0: {partition=0, offset=14, value=33}
2: {partition=2, offset=1, value=2}
2: {partition=2, offset=2, value=3}
2: {partition=2, offset=3, value=9}
2: {partition=2, offset=4, value=16}
2: {partition=2, offset=5, value=29}
2: {partition=2, offset=6, value=32}
2: {partition=2, offset=7, value=36}
2: {partition=2, offset=8, value=40}
2: {partition=2, offset=9, value=41}
2: {partition=2, offset=10, value=49}
2: {partition=2, offset=11, value=50}
2: {partition=2, offset=12, value=57}
2: {partition=2, offset=13, value=69}
2: {partition=2, offset=14, value=73}
2: {partition=2, offset=15, value=74}
2: {partition=2, offset=16, value=78}
2: {partition=2, offset=17, value=83}
2: {partition=2, offset=18, value=85}
2: {partition=2, offset=19, value=90}
2: {partition=2, offset=20, value=91}
2: {partition=2, offset=21, value=97}
0: {partition=0, offset=15, value=34}
0: {partition=0, offset=16, value=37}
0: {partition=0, offset=17, value=43}
0: {partition=0, offset=18, value=44}
0: {partition=0, offset=19, value=45}
0: {partition=0, offset=20, value=47}
1: {partition=1, offset=0, value=4}
1: {partition=1, offset=1, value=6}
1: {partition=1, offset=2, value=10}
1: {partition=1, offset=3, value=12}
1: {partition=1, offset=4, value=13}
1: {partition=1, offset=5, value=14}
1: {partition=1, offset=6, value=18}
1: {partition=1, offset=7, value=19}
1: {partition=1, offset=8, value=20}
1: {partition=1, offset=9, value=24}
1: {partition=1, offset=10, value=26}
1: {partition=1, offset=11, value=31}
1: {partition=1, offset=12, value=35}
1: {partition=1, offset=13, value=38}
1: {partition=1, offset=14, value=39}
1: {partition=1, offset=15, value=42}
1: {partition=1, offset=16, value=46}
1: {partition=1, offset=17, value=51}
1: {partition=1, offset=18, value=53}
1: {partition=1, offset=19, value=58}
1: {partition=1, offset=20, value=62}
1: {partition=1, offset=21, value=65}
1: {partition=1, offset=22, value=66}
1: {partition=1, offset=23, value=67}
1: {partition=1, offset=24, value=68}
1: {partition=1, offset=25, value=75}
1: {partition=1, offset=26, value=77}
1: {partition=1, offset=27, value=81}
1: {partition=1, offset=28, value=84}
1: {partition=1, offset=29, value=86}
1: {partition=1, offset=30, value=87}
1: {partition=1, offset=31, value=94}
1: {partition=1, offset=32, value=95}
0: {partition=0, offset=21, value=48}
0: {partition=0, offset=22, value=52}
0: {partition=0, offset=23, value=54}
0: {partition=0, offset=24, value=55}
0: {partition=0, offset=25, value=56}
0: {partition=0, offset=26, value=59}
0: {partition=0, offset=27, value=60}
0: {partition=0, offset=28, value=61}
0: {partition=0, offset=29, value=63}
0: {partition=0, offset=30, value=64}
0: {partition=0, offset=31, value=70}
0: {partition=0, offset=32, value=71}
0: {partition=0, offset=33, value=72}
0: {partition=0, offset=34, value=76}
0: {partition=0, offset=35, value=79}
0: {partition=0, offset=36, value=80}
0: {partition=0, offset=37, value=82}
0: {partition=0, offset=38, value=88}
0: {partition=0, offset=39, value=89}
0: {partition=0, offset=40, value=92}
0: {partition=0, offset=41, value=93}
0: {partition=0, offset=42, value=96}
0: {partition=0, offset=43, value=98}
0: {partition=0, offset=44, value=99}

• 2个线程

1: {partition=2, offset=0, value=0}
1: {partition=2, offset=1, value=2}
0: {partition=0, offset=0, value=1}
0: {partition=0, offset=1, value=5}
0: {partition=0, offset=2, value=7}
0: {partition=0, offset=3, value=8}
0: {partition=0, offset=4, value=11}
0: {partition=0, offset=5, value=15}
0: {partition=0, offset=6, value=17}
1: {partition=2, offset=2, value=3}
1: {partition=2, offset=3, value=9}
0: {partition=0, offset=7, value=21}
1: {partition=2, offset=4, value=16}
1: {partition=2, offset=5, value=29}
1: {partition=2, offset=6, value=32}
1: {partition=2, offset=7, value=36}
1: {partition=2, offset=8, value=40}
1: {partition=2, offset=9, value=41}
1: {partition=2, offset=10, value=49}
0: {partition=0, offset=8, value=22}
1: {partition=2, offset=11, value=50}
0: {partition=0, offset=9, value=23}
1: {partition=2, offset=12, value=57}
0: {partition=0, offset=10, value=25}
1: {partition=2, offset=13, value=69}
0: {partition=0, offset=11, value=27}
1: {partition=2, offset=14, value=73}
0: {partition=0, offset=12, value=28}
1: {partition=2, offset=15, value=74}
0: {partition=0, offset=13, value=30}
1: {partition=2, offset=16, value=78}
0: {partition=0, offset=14, value=33}
1: {partition=2, offset=17, value=83}
0: {partition=0, offset=15, value=34}
1: {partition=2, offset=18, value=85}
1: {partition=2, offset=19, value=90}
1: {partition=2, offset=20, value=91}
1: {partition=2, offset=21, value=97}
0: {partition=0, offset=16, value=37}
0: {partition=0, offset=17, value=43}
0: {partition=0, offset=18, value=44}
0: {partition=0, offset=19, value=45}
0: {partition=0, offset=20, value=47}
0: {partition=0, offset=21, value=48}
0: {partition=0, offset=22, value=52}
0: {partition=0, offset=23, value=54}
0: {partition=0, offset=24, value=55}
0: {partition=0, offset=25, value=56}
0: {partition=0, offset=26, value=59}
0: {partition=0, offset=27, value=60}
0: {partition=0, offset=28, value=61}
0: {partition=0, offset=29, value=63}
0: {partition=0, offset=30, value=64}
0: {partition=0, offset=31, value=70}
0: {partition=0, offset=32, value=71}
0: {partition=0, offset=33, value=72}
0: {partition=0, offset=34, value=76}
0: {partition=0, offset=35, value=79}
0: {partition=0, offset=36, value=80}
0: {partition=0, offset=37, value=82}
0: {partition=0, offset=38, value=88}
0: {partition=0, offset=39, value=89}
0: {partition=0, offset=40, value=92}
0: {partition=0, offset=41, value=93}
0: {partition=0, offset=42, value=96}
0: {partition=0, offset=43, value=98}
0: {partition=0, offset=44, value=99}
0: {partition=1, offset=0, value=4}
0: {partition=1, offset=1, value=6}
0: {partition=1, offset=2, value=10}
0: {partition=1, offset=3, value=12}
0: {partition=1, offset=4, value=13}
0: {partition=1, offset=5, value=14}
0: {partition=1, offset=6, value=18}
0: {partition=1, offset=7, value=19}
0: {partition=1, offset=8, value=20}
0: {partition=1, offset=9, value=24}
0: {partition=1, offset=10, value=26}
0: {partition=1, offset=11, value=31}
0: {partition=1, offset=12, value=35}
0: {partition=1, offset=13, value=38}
0: {partition=1, offset=14, value=39}
0: {partition=1, offset=15, value=42}
0: {partition=1, offset=16, value=46}
0: {partition=1, offset=17, value=51}
0: {partition=1, offset=18, value=53}
0: {partition=1, offset=19, value=58}
0: {partition=1, offset=20, value=62}
0: {partition=1, offset=21, value=65}
0: {partition=1, offset=22, value=66}
0: {partition=1, offset=23, value=67}
0: {partition=1, offset=24, value=68}
0: {partition=1, offset=25, value=75}
0: {partition=1, offset=26, value=77}
0: {partition=1, offset=27, value=81}
0: {partition=1, offset=28, value=84}
0: {partition=1, offset=29, value=86}
0: {partition=1, offset=30, value=87}
0: {partition=1, offset=31, value=94}
0: {partition=1, offset=32, value=95}

• 线程0 消费了partition 1; 线程 1消费了 partition 0, partition 2

一个分区只能被一个线程消费，但一个消费线程可以消费多个分区的数据