kafka学习笔记四Consumer学习

1.Consumer and Consumer Group  消费者与消费者分组

group.id  分组属性

client.id  consumer的id

2.Kafka Consumer programming  编程coding

package com.hanwan.kafka.demo2;


import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collection;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * @ClassName SimpleConsumer
 * @Description Consumer config demo
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/17 16:24
 * @Version 1.0
 **/
public class SimpleConsumer {

    private final static Logger LOGGER = LoggerFactory.getLogger(SimpleConsumer.class);

    public static void main(String[] args) {
        /*
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProps());
        consumer.subscribe(Collections.singleton("test_c"));
        //表示永久循环
        for (;;) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            records.forEach(record ->{
                LOGGER.info("-----------------------------------------------------------------");
                LOGGER.info("offset {}",record.offset());
                LOGGER.info("value {}", record.value());
                LOGGER.info("partition {}", record.partition());
                LOGGER.info("key {}",record.key());
                LOGGER.info("-----------------------------------------------------------------");
            });
        }*/

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProps2());
        consumer.subscribe(Collections.singleton("test_c"));
        final AtomicInteger counter = new AtomicInteger();
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            records.forEach(record ->{
                LOGGER.info("--------------------------------------------------------");
                LOGGER.info("offset {}", record.offset());
                LOGGER.info("value {}", record.value());
                LOGGER.info("key {}", record.key());
                LOGGER.info("partition {}", record.partition());
                LOGGER.info("--------------------------------------------------------");
                int cnt = counter.getAndIncrement();
                if (cnt >= 3) {
                    Runtime.getRuntime().halt(-1);
                }
            });
        }
    }

    private static Properties loadProps(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id", "test_group3");
        prop.put("client.id", "demo-consumer-client");
        return prop;
    }

    private static Properties loadProps2(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id","test_group4");
        prop.put("client.id", "demo-consumer-client");
        prop.put("auto.offset.reset", "earliest");
        prop.put("enable.auto.commit", "true"); //默认为true自动提交
        prop.put("auto.commit.interval.ms", "10000"); //设置默认自动提交时间 ，默认值为5000ms
        return prop;
    }
}

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * @ClassName ConsumerSyncCommit
 * @Description Consumer 同步commit
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 10:07
 * @Version 1.0
 **/
public class ConsumerSyncCommit {

    private final static Logger LOGGER = LoggerFactory.getLogger(ConsumerSyncCommit.class);

    public static void main(String[] args) {
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProps());
        consumer.subscribe(Collections.singleton("test12"));
        final AtomicInteger count = new AtomicInteger(0);
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            records.forEach(record ->{
                LOGGER.info("key {}", record.key());
                LOGGER.info("partition {}", record.partition());
                LOGGER.info("offset {}", record.offset());
                LOGGER.info("value {}", record.value());
//                if (count.incrementAndGet() == 100) {
//                    consumer.commitSync();
//                    count.set(0);
//                }
            });
            /*
            * can retry
            * block
            * */
            consumer.commitSync();
        }
    }

    private static Properties loadProps(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id", "test_group1");
        prop.put("client.id", "demo-consumer-client");
        prop.put("auto.offset.reset", "earliest");
        prop.put("enable.auto.commit", "false");
        return prop;
    }
}

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.Properties;

/**
 * @ClassName ConsumerAsyncCommit
 * @Description Consumer 异步commit
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 13:28
 * @Version 1.0
 **/
public class ConsumerAsyncCommit {

    private final static Logger LOGGER = LoggerFactory.getLogger(ConsumerAsyncCommit.class);

    public static void main(String[] args) {
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProp());
        consumer.subscribe(Collections.singleton("test12"));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            records.forEach(record ->{
                LOGGER.info("key {}", record.key());
                LOGGER.info("partition {}", record.partition());
                LOGGER.info("offset {}", record.offset());
                LOGGER.info("value {}", record.value());
            });
            /*
            * can not by retry
            * non-block
            * */
            consumer.commitAsync((map, e) -> {
                if (e != null) {
                    e.printStackTrace();
                } else {
                    System.out.println(map);
                }
            });
        }
    }

    private static Properties loadProp(){
        final  Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id", "test_group1");
        prop.put("client.id", "demo-consumer-client");
        prop.put("auto.offset.reset", "earliest");
        prop.put("enable.auto.commit", "false");
        return prop;
    }

}

4.Consumer configuration in deep  Consumer 的核心配置

·bootstrap.servers

在启动consumer时配置的broker地址的。不需要将cluster中所有的broker都配置上，因为启动后会自动的发现cluster所有的broker。

    它配置的格式是：host1:port1;host2:port2…

·key.descrializer、value.descrializer

Message record 的key, value的反序列化类。

·group.id

用于表示该consumer想要加入到哪个group中。默认值是 “”。

·heartbeat.interval.ms

心跳间隔。心跳是在consumer与coordinator之间进行的。心跳是确定consumer存活，加入或者退出group的有效手段。

    这个值必须设置的小于session.timeout.ms，因为：

当Consumer由于某种原因不能发Heartbeat到coordinator时，并且时间超过session.timeout.ms时，就会认为该consumer已退出，它所订阅的partition会分配到同一group 内的其它的consumer上。

    通常设置的值要低于session.timeout.ms的1/3。

    默认值是：3000 （3s）

·session.timeout.ms

Consumer session 过期时间。这个值必须设置在broker configuration中的group.min.session.timeout.ms 与 group.max.session.timeout.ms之间。

其默认值是：10000 （10 s）

·enable.auto.commit

Consumer 在commit offset时有两种模式：自动提交，手动提交。手动提交在前面已经说过。自动提交：是Kafka Consumer会在后台周期性的去commit。

默认值是true。

·auto.commit.interval.ms

    自动提交间隔。范围：[0,Integer.MAX]，默认值是 5000 （5 s）

·auto.offset.reset

    这个配置项，是告诉Kafka Broker在发现kafka在没有初始offset，或者当前的offset是一个不存在的值（如果一个record被删除，就肯定不存在了）时，该如何处理。它有4种处理方式：

1） earliest：自动重置到最早的offset。

2） latest：看上去重置到最晚的offset。

3） none：如果边更早的offset也没有的话，就抛出异常给consumer，告诉consumer在整个consumer group中都没有发现有这样的offset。

4） 如果不是上述3种，只抛出异常给consumer。

默认值是latest。

·connections.max.idle.ms

连接空闲超时时间。因为consumer只与broker有连接（coordinator也是一个broker），所以这个配置的是consumer到broker之间的。

默认值是：540000 (9 min)

·fetch.max.wait.ms

Fetch请求发给broker后，在broker中可能会被阻塞的（当topic中records的总size小于fetch.min.bytes时），此时这个fetch请求耗时就会比较长。这个配置就是来配置consumer最多等待response多久。

·fetch.min.bytes

当consumer向一个broker发起fetch请求时，broker返回的records的大小最小值。如果broker中数据量不够的话会wait，直到数据大小满足这个条件。

取值范围是：[0, Integer.Max]，默认值是1。

默认值设置为1的目的是：使得consumer的请求能够尽快的返回。

·fetch.max.bytes

一次fetch请求，从一个broker中取得的records最大大小。如果在从topic中第一个非空的partition取消息时，如果取到的第一个record的大小就超过这个配置时，仍然会读取这个record，也就是说在这片情况下，只会返回这一条record。

    broker、topic都会对producer发给它的message size做限制。所以在配置这值时，可以参考broker的message.max.bytes 和 topic的max.message.bytes的配置。

取值范围是：[0, Integer.Max]，默认值是：52428800 （5 MB）

·max.partition.fetch.bytes

一次fetch请求，从一个partition中取得的records最大大小。如果在从topic中第一个非空的partition取消息时，如果取到的第一个record的大小就超过这个配置时，仍然会读取这个record，也就是说在这片情况下，只会返回这一条record。

    broker、topic都会对producer发给它的message size做限制。所以在配置这值时，可以参考broker的message.max.bytes 和 topic的max.message.bytes的配置。

·max.poll.interval.ms

前面说过要求程序中不间断的调用poll()。如果长时间没有调用poll，且间隔超过这个值时，就会认为这个consumer失败了。

·max.poll.records

    Consumer每次调用poll()时取到的records的最大数。

·receive.buffer.byte

Consumer receiver buffer （SO_RCVBUF）的大小。这个值在创建Socket连接时会用到。

取值范围是：[-1, Integer.MAX]。默认值是：65536 （64 KB）

如果值设置为-1，则会使用操作系统默认的值。

·request.timeout.ms

请求发起后，并不一定会很快接收到响应信息。这个配置就是来配置请求超时时间的。默认值是：305000 （305 s）

·client.id

Consumer进程的标识。如果设置一个人为可读的值，跟踪问题会比较方便。

·interceptor.classes

    用户自定义interceptor。

·metadata.max.age.ms

Metadata数据的刷新间隔。即便没有任何的partition订阅关系变更也行执行。

范围是：[0, Integer.MAX]，默认值是：300000 （5 min）

4.Commit and offset  in deep Consumer的commit数据的方式

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

/**
 * @ClassName CommitSpecifiedOffset
 * @Description 指定partition的offset Commit
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 15:29
 * @Version 1.0
 **/
public class CommitSpecifiedOffset {

    private final static Logger LOGGER = LoggerFactory.getLogger(CommitSpecifiedOffset.class);

    public static void main(String[] args) {
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProp());
        consumer.subscribe(Collections.singletonList("test12"));
        Map<TopicPartition, OffsetAndMetadata> offset = new HashMap<>();
        TopicPartition tp = new TopicPartition("test12", 1);
        OffsetAndMetadata om = new OffsetAndMetadata(35, "no meta data");
        offset.put(tp, om);
        try {
            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(100);
                records.forEach(record -> {
                    LOGGER.info("partition {}, offset {}", record.partition(), record.offset());
                });
//                consumer.commitAsync();
                consumer.commitSync(offset);
            }
        } finally {
//            consumer.commitSync(offset);
        }
    }

    private static Properties loadProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id", "group_test11");
        prop.put("client.id", "demo-consumer-client");
        prop.put("auto.offset.reset", "earliest");
        prop.put("enable.auto.commit", "false");
        return prop;
    }
}

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.Properties;

/**
 * @ClassName CommitOffsetBothAsyncSync
 * @Description 同步异步结合的方式commit，推荐使用这种方式
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 14:30
 * @Version 1.0
 **/
public class CommitOffsetBothAsyncSync {

    private static Logger LOGGER = LoggerFactory.getLogger(CommitOffsetBothAsyncSync.class);

    public static void main(String[] args) {
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProp());
        consumer.subscribe(Collections.singleton("test12"));
        try {

            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(100);
                records.forEach(record -> {
                    LOGGER.info("key {}", record.key());
                    LOGGER.info("value {}", record.value());
                    LOGGER.info("partition {}", record.partition());
                    LOGGER.info("offset {}", record.offset());
                });
                //异步提交
                consumer.commitAsync();
            }
        } finally {
            //同步提交
            consumer.commitSync();
        }
    }

    private static Properties loadProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id", "group_test3");
        prop.put("client.id", "demo-consumer-client");
        prop.put("auto.offset.reset", "earliest");
        prop.put("enable.auto.commit", "false");
        return prop;
    }
}

5.Consumer programming in deep 深入分析Consumer的代码

6.Serializers & deserializers  序列化与反序列化

package com.hanwan.kafka.demo2.internal;

/**
 * @ClassName User
 * @Description TODO
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 17:08
 * @Version 1.0
 **/
public class User {

    private int id;
    private String name;
    private String address;

    public User(int id, String name, String address) {
        this.id = id;
        this.name = name;
        this.address = address;
    }

    public int getId() {
        return id;
    }

    public String getName() {
        return name;
    }

    public String getAddress() {
        return address;
    }

    @Override
    public String toString() {
        return "User{" +
                "id=" + id +
                ", name='" + name + '\'' +
                ", address='" + address + '\'' +
                '}';
    }
}

package com.hanwan.kafka.demo2.internal;

import org.apache.kafka.common.serialization.Serializer;

import java.nio.ByteBuffer;
import java.util.Map;

/**
 * @ClassName UserSerializer
 * @Description TODO
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 17:33
 * @Version 1.0
 **/
public class UserSerializer implements Serializer<User> {


    @Override
    public void configure(Map<String, ?> map, boolean b) {
        //do nothing
    }

    @Override
    public byte[] serialize(String topic, User data) {
        if (data == null)
            return null;
        int id = data.getId();
        String name = data.getName();
        String address = data.getAddress();
        byte[] nameBytes;
        if (name != null)
            nameBytes = name.getBytes();
        else
            nameBytes = new byte[0];

        byte[] addressBytes;
        if (address != null)
            addressBytes = address.getBytes();
        else
            addressBytes = new byte[0];
        ByteBuffer buffer = ByteBuffer.allocate(4 + 4 + nameBytes.length + 4 + addressBytes.length);
        buffer.putInt(id);
        buffer.putInt(nameBytes.length);
        buffer.put(nameBytes);
        buffer.putInt(addressBytes.length);
        buffer.put(addressBytes);
        return buffer.array();
    }

    @Override
    public void close() {

    }
}

package com.hanwan.kafka.demo2.internal;

import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.serialization.Deserializer;

import java.nio.ByteBuffer;
import java.util.Map;

/**
 * @ClassName UserDeserializer
 * @Description TODO
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 17:59
 * @Version 1.0
 **/
public class UserDeserializer implements Deserializer<User> {
    @Override
    public void configure(Map<String, ?> map, boolean b) {

    }

    @Override
    public User deserialize(String topic, byte[] data) {
        if (data == null)
            return null;
        if (data.length < 12)
            throw new SerializationException("The User data bytes length should not be less than 12.");
        ByteBuffer buffer = ByteBuffer.wrap(data);
        int id = buffer.getInt();
        int nameLength = buffer.getInt();
        byte[] nameBytes = new byte[nameLength];
        buffer.get(nameBytes);
        String name = new String(nameBytes);

        int addressLength = buffer.getInt();
        byte[] addressBytes = new byte[addressLength];
        buffer.get(addressBytes);
        String address = new String(addressBytes);
        return new User(id, name, address);
    }

    @Override
    public void close() {

    }
}

package com.hanwan.kafka.demo2;

import com.hanwan.kafka.demo2.internal.User;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.IntStream;

/**
 * @ClassName CustomerSerDeseSender
 * @Description 序列化 与反序列化producer方法
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/21 9:11
 * @Version 1.0
 **/
public class CustomerSerDeseSender {

    private final static Logger LOGGER = LoggerFactory.getLogger(CustomerSerDeseSender.class);

    public static void main(String[] args) {
        KafkaProducer<String, User> producer = new KafkaProducer<>(initProp());
        IntStream.range(1, 10).forEach(i ->{
            ProducerRecord<String, User> record = new ProducerRecord<>("test13", String.valueOf(i),new User(i, "name-"+i,"address-"+i));
            Future<RecordMetadata> future = producer.send(record);
            try {
                RecordMetadata recordMetadata = future.get();
                LOGGER.info("this message is send done the key is {}, offset {}", i, recordMetadata.offset());
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        });
        producer.flush();
        producer.close();
    }

    private static Properties initProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        prop.put("value.serializer", "com.hanwan.kafka.demo2.internal.UserSerializer");
        prop.put("acks", "all");
        return prop;
    }
}

package com.hanwan.kafka.demo2;

import com.hanwan.kafka.demo2.internal.User;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.Properties;

/**
 * @ClassName CustomerSerializerAndDeserializer
 * @Description consumer 的序列化与反序列化
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/20 18:18
 * @Version 1.0
 **/
public class CustomerSerializerAndDeserializer {

    private final static Logger LOGGER = LoggerFactory.getLogger(CustomerSerializerAndDeserializer.class);

    public static void main(String[] args) {
        KafkaConsumer<String, User> consumer = new KafkaConsumer<>(loadProp());
        consumer.subscribe(Collections.singletonList("test13"));
        while (true) {
            ConsumerRecords<String, User> records = consumer.poll(100);
            records.forEach(record ->{
                LOGGER.info("key {} value {}", record.key(), record.value());
            });
        }
    }

    private static Properties loadProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "com.hanwan.kafka.demo2.internal.UserDeserializer");
        prop.put("group.id", "group_test15");
        prop.put("client.id", "demo-consumer-client");
        prop.put("auto.offset.reset", "earliest");
        prop.put("enable.auto.commit", "false");
        return prop;
    }

}

7.Consumer and Producer Interceptor生产者消费者拦截器

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.producer.ProducerInterceptor;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.Map;

/**
 * @ClassName MyProducerInterceptor
 * @Description 生产者拦截器重定义
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/21 10:30
 * @Version 1.0
 **/
public class MyProducerInterceptor implements ProducerInterceptor<String, String> {
    @Override
    public ProducerRecord<String, String> onSend(ProducerRecord<String, String> producerRecord) {
        return new ProducerRecord<>(producerRecord.topic(), producerRecord.partition(),
                producerRecord.timestamp(), producerRecord.key(),
                producerRecord.value().toUpperCase(), producerRecord.headers());
    }

    @Override
    public void onAcknowledgement(RecordMetadata recordMetadata, Exception e) {

    }

    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> map) {

    }
}

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.consumer.ConsumerInterceptor;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static java.util.stream.Collectors.toList;

/**
 * @ClassName MyConsumerInterceptor
 * @Description Consumer拦截器demo
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/21 11:58
 * @Version 1.0
 **/
public class MyConsumerInterceptor implements ConsumerInterceptor<String, String> {
    @Override
    public ConsumerRecords<String, String> onConsume(ConsumerRecords<String, String> consumerRecords) {
        Map<TopicPartition, List<ConsumerRecord<String, String>>> results = new HashMap<>();
        Set<TopicPartition> partitions =  consumerRecords.partitions();
        partitions.forEach(p ->{
            List<ConsumerRecord<String, String>> result = consumerRecords.records(p)
                                        .stream().filter(record -> record.value().equals("HELLO8"))
                                        .collect(toList());
            results.put(p, result);
        });
        return new ConsumerRecords<>(results);
    }

    @Override
    public void onCommit(Map<TopicPartition, OffsetAndMetadata> offsets) {
        System.out.println("-----------begin--------------");
        System.out.println(offsets);
        System.out.println("-----------end--------------");
    }

    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> map) {

    }
}

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.IntStream;

/**
 * @ClassName SyncSenderWithInterceptor
 * @Description 拦截器producer测试类
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/21 10:44
 * @Version 1.0
 **/
public class SyncSenderWithInterceptor {

    private final static Logger LOGGER = LoggerFactory.getLogger(SyncSenderWithInterceptor.class);

    public static void main(String[] args) {
        KafkaProducer<String, String> producer = new KafkaProducer<>(initProp());
        IntStream.range(0, 10).forEach(i ->{
            ProducerRecord<String, String> record = new ProducerRecord<>("test14", String.valueOf(i),"hello"+i);
            Future<RecordMetadata> future = producer.send(record);
            try {
                RecordMetadata recordMetadata = future.get();
                LOGGER.info("this message is send done and the key is {}, offset {}", i, recordMetadata.offset());
            } catch (InterruptedException | ExecutionException e) {
                e.printStackTrace();
            }
        });
        producer.flush();
        producer.close();
    }

    private static Properties initProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        prop.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        prop.put("acks", "all");
        prop.put(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, "com.hanwan.kafka.demo2.MyProducerInterceptor");
        return prop;
    }
}

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.Properties;

/**
 * @ClassName SimpleConsumerWithInterceptor
 * @Description Consumer拦截器demo测试
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/21 14:19
 * @Version 1.0
 **/
public class SimpleConsumerWithInterceptor {

    private final static Logger LOGGER = LoggerFactory.getLogger(SimpleConsumerWithInterceptor.class);

    public static void main(String[] args) {
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProp());
        consumer.subscribe(Collections.singletonList("test14"));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            records.forEach(record ->{
                LOGGER.info("partition {}, value {}, offset {}", record.partition(), record.value(), record.offset());
            });
        }
    }

    private static Properties loadProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id", "g5");
        prop.put("client.id", "demo-interceptor-client");
        prop.put("auto.offset.reset", "earliest");
        prop.put("auto.commit.interval.ms", "10000");
        prop.put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, "com.hanwan.kafka.demo2.MyConsumerInterceptor");
        return prop;
    }
}

8.Consumer and Rebalance  消费者的平衡

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.IntStream;

/**
 * @ClassName RebalanceSender
 * @Description Rebalance Producer
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/21 15:33
 * @Version 1.0
 **/
public class RebalanceSender {

    private final static Logger LOGGER = LoggerFactory.getLogger(RebalanceSender.class);

    public static void main(String[] args) {
        KafkaProducer<String, String> producer = new KafkaProducer<>(initProp());
        IntStream.range(20, 30).forEach(i ->{
            ProducerRecord<String, String> record = new ProducerRecord<>("test16",String.valueOf(i), "hello"+i);
            Future<RecordMetadata> future = producer.send(record);
            try {
                RecordMetadata recordMetadata = future.get();
                LOGGER.info("this message is send done and the key is{}, offset {}", i, recordMetadata.offset());
            } catch (InterruptedException | ExecutionException e) {
                e.printStackTrace();
            }
        });
        producer.flush();
        producer.close();
    }

    private static Properties initProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        prop.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        prop.put("acks", "all");
        return prop;
    }
}

package com.hanwan.kafka.demo2;

import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collection;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

/**
 * @ClassName RebalanceConsumer
 * @Description Rebalance  Consumer  demo
 * @Copyright: Copyright (c) 2018</p>
 * @Company: www.lowan.com</ p>
 * @Author hanwan
 * @Date 2018/8/21 15:12
 * @Version 1.0
 **/
public class RebalanceConsumer {

    private final static Logger LOGGER = LoggerFactory.getLogger(RebalanceConsumer.class);

    public static void main(String[] args) {
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(loadProp());
        consumer.subscribe(Collections.singletonList("test16"));
        for (; ; ) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            records.forEach(record ->{
                LOGGER.info("partition {}, key {}, value {}, offset {}", record.partition(), record.key(), record.value(), record.offset());
                try {
                    TimeUnit.SECONDS.sleep(10);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            });
            consumer.commitSync();
        }
    }

    private static Properties loadProp(){
        final Properties prop = new Properties();
        prop.put("bootstrap.servers", "120.55.125.58:9092,120.26.198.248:9092,121.40.200.37:9092");
        prop.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.put("group.id", "group-rebalance");
        prop.put("client.id", "demo-rebalance-client3");
        prop.put("auto.offset.reset", "earliest");
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
//        prop.put("auto.commit.interval.ms", "10000");
        return prop;
    }

    private static class MyConsumerBanalanceListener implements ConsumerRebalanceListener {

        private final KafkaConsumer<String, String> consumer;

        private MyConsumerBanalanceListener(KafkaConsumer<String, String> consumer) {
            this.consumer = consumer;
        }

        @Override
        public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
            for (TopicPartition partition : partitions) {
                long nextOffset = consumer.position(partition);
                //nextOffset partition save
            }
            LOGGER.info("onPartitionsRevoked=> {}", partitions);
        }

        @Override
        public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
            for (TopicPartition partition : partitions) {
                consumer.seek(partition, 0);
            }
            LOGGER.info("onPartitionsAssigned=> {}", partitions);
        }
    }
}

再均衡监听器

      在为消费者分配新的partition或者移除旧的partition时，可以通过消费者API执行一些应用程序代码，在使用subscribe()方法时传入一个ConsumerRebalanceListener实例。

     ConsumerRebalanceListener需要实现的两个方法：

    1) public void onPartitionRevoked(Collection<TopicPartition> partitions)方法会在再均衡开始之前和消费者停止读取消息之后被调用。如果在这里提交偏移量，下一个接管partition的消费者就知道该从哪里开始读取了。

    2) public void onPartitionAssigned(Collection<TopicPartition> partitions)方法会在重新分配partition之后和消费者开始读取消息之前被调用。