Getting started with Kafka, manually submitting offsets, synchronously submitting, asynchronously submitting, specifying Offset consumption (23)

Submit the offset manually

Although offset is very traversal, it is difficult for developers to grasp the reality of offset submission because it is submitted based on time. Therefore, Kafka also provides an API for manually submitting offsets.
There are two methods for manually submitting offsets: commitSync (synchronous submission) and commitAsync (asynchronous submission). The same point between the two is that the highest offset of a batch of data submitted this time will be submitted: the difference is that the synchronous submission blocks the current thread until the submission is successful, and will automatically fail and retry (caused by uncontrollable factors) , there will also be a submission failure) and asynchronous submission has no retry mechanism, so the submission may fail.
commitSync (synchronous submission): You must wait for the offset to be submitted before consuming the next batch of data.
commitAsync (asynchronous submission): After sending and submitting the offset request, the next batch of data will be consumed

synchronous commit

是否自动提交offset properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,false);
同步提交offset kafkaConsumer.commitSync();

Since the synchronous submission of offset has a failure retry mechanism, it is more reliable, but due to the unanimous waiting for the submission result, the submission efficiency is relatively low. The following is an example of submitting offset synchronously

package com.longer.handsync;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Properties;

public class CustomConsumerByHandSync {
    
    
    public static void main(String[] args) {
    
    
        //创建消费者的配置对象
        Properties properties=new Properties();
        //2、给消费者配置对象添加参数
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop100:9092");
        //配置序列化
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class.getName());
        //配置消费者组（组名任意起名）必须
        properties.put(ConsumerConfig.GROUP_ID_CONFIG,"test");
        //修改分区策略
        properties.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG,"org.apache.kafka.clients.consumer.StickyAssignor");
//        properties.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG,"false");
        //是否自动提交offset
        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,false);
        //创建消费者对象
        KafkaConsumer<String,String> kafkaConsumer=new KafkaConsumer<String, String>(properties);
        //注册要消费的主题
        ArrayList<String> topics=new ArrayList<>();
        topics.add("two");
        kafkaConsumer.subscribe(topics);
        while (true){
    
    
            //设置1s中消费一批数据
            ConsumerRecords<String,String> consumerRecords=kafkaConsumer.poll(Duration.ofSeconds(1));
            //打印消费到的数据
            for(ConsumerRecord<String,String> record:consumerRecords){
    
    
                System.out.println(record);
            }
            //同步提交offset
             kafkaConsumer.commitSync();
        }
    }
}

asynchronous commit

Although synchronous submission of offset is more reliable, it will block the current thread until the submission is successful. Therefore, the throughput will be greatly affected, so in more cases, the asynchronous offset method will be chosen
kafkaConsumer.commitAsync();

package com.longer.handasync;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Properties;

/**
 * 同步提交
 */
public class CustomConsumerByHandAsync {
    
    
    public static void main(String[] args) {
    
    
        //创建消费者的配置对象
        Properties properties=new Properties();
        //2、给消费者配置对象添加参数
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop100:9092");
        //配置序列化
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class.getName());
        //配置消费者组（组名任意起名）必须
        properties.put(ConsumerConfig.GROUP_ID_CONFIG,"test");
        //修改分区策略
        properties.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG,"org.apache.kafka.clients.consumer.StickyAssignor");
//        properties.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG,"false");
        //是否自动提交offset
        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,false);
        //创建消费者对象
        KafkaConsumer<String,String> kafkaConsumer=new KafkaConsumer<String, String>(properties);
        //注册要消费的主题
        ArrayList<String> topics=new ArrayList<>();
        topics.add("two");
        kafkaConsumer.subscribe(topics);
        while (true){
    
    
            //设置1s中消费一批数据
            ConsumerRecords<String,String> consumerRecords=kafkaConsumer.poll(Duration.ofSeconds(1));
            //打印消费到的数据
            for(ConsumerRecord<String,String> record:consumerRecords){
    
    
                System.out.println(record);
            }
            //同步提交offset
            kafkaConsumer.commitAsync();
        }
    }
}

Specify Offset Consumption

auto.offset.reset = earliest | latest | none The default is latest
when there is no initial offset in Kafka (consumer group consumes for the first time) or the current offset no longer exists on the server (for example, the data has been deleted) ,what to do?
1) earliest: automatically reset the offset to the earliest offset, --from-beginning
2) latest (default): automatically reset the offset to the latest offset
3) if no consumer group is found , an exception is thrown to the consumer.

main code

		Set<TopicPartition> assigment=new HashSet<>();

        while (assigment.size()==0){
    
    
            kafkaConsumer.poll(Duration.ofSeconds(1));
            //获取消费者分区分配信息（有了分区分配信息才能开始消费）
            assigment= kafkaConsumer.assignment();
        }
        //遍历所有分区，并指定从100得位置开始消费
        for (TopicPartition tp : assigment) {
    
    
            kafkaConsumer.seek(tp,100);
        }

package com.longer.seek;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;

public class CustomConsumerSeek {
    
    
    public static void main(String[] args) {
    
    
        //创建消费者的配置对象
        Properties properties=new Properties();
        //2、给消费者配置对象添加参数
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop100:9092");
        //配置序列化
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class.getName());
        //配置消费者组（组名任意起名）必须
        properties.put(ConsumerConfig.GROUP_ID_CONFIG,"test");
        //修改分区策略
        properties.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG,"org.apache.kafka.clients.consumer.StickyAssignor");
//        properties.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG,"false");

        //创建消费者对象
        KafkaConsumer<String,String> kafkaConsumer=new KafkaConsumer<String, String>(properties);
        //注册要消费的主题
        ArrayList<String> topics=new ArrayList<>();
        topics.add("two");
        kafkaConsumer.subscribe(topics);

        Set<TopicPartition> assigment=new HashSet<>();

        while (assigment.size()==0){
    
    
            kafkaConsumer.poll(Duration.ofSeconds(1));
            //获取消费者分区分配信息（有了分区分配信息才能开始消费）
            assigment= kafkaConsumer.assignment();
        }
        //遍历所有分区，并指定从100得位置开始消费
        for (TopicPartition tp : assigment) {
    
    
            kafkaConsumer.seek(tp,100);
        }


        while (true){
    
    
            //设置1s中消费一批数据
            ConsumerRecords<String,String> consumerRecords=kafkaConsumer.poll(Duration.ofSeconds(1));
            //打印消费到的数据
            for(ConsumerRecord<String,String> record:consumerRecords){
    
    
                System.out.println(record);
            }
        }
    }
}