项目中最近使用kafka需要定时清理消息，我们知道kafka有周期性清理消息机制,但是项目中往往因为数据量较大，需要手动控制分区已消费消息的清理。
此处使用的是反编译程序，具体几个方法已标出，
个人清理想法：大致流程就是根据topic获取所有的分区，然后根据遍历每个分区的偏移量，然后存入map，存入时先判断偏移量大小，保留最小该分区消费者最小偏移量的值,然后删除该偏移量以前的数据，
下面的反码程序实际上没有做最小偏移量识别，直接用的分区最后一个最新的的偏移量，是通过创建一个消费者直接将其Offset toEnd，然后删掉该分区该offset之前的消息，具体的大家可根据自己要求自行修改
package com.atguigu.kafkaclean;
import org.springframework.stereotype.*;
import org.springframework.context.annotation.*;
import com.ankki.kafkaclient.mapper.*;
import javax.annotation.*;
import java.time.format.*;
import com.ankki.kafkaclient.utils.*;
import java.util.concurrent.*;
import org.apache.kafka.clients.admin.*;
import org.springframework.scheduling.annotation.*;
import com.ankki.kafkaclient.consumer.*;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.*;
import com.ankki.kafkaclient.model.*;
import java.time.*;
import org.slf4j.*;
import java.util.*;
@Slf4j
@Component
@Configuration
@EnableScheduling
public class SaticScheduleTask
{
private static final Logger log;
private static final String SASL_VALUE = "yes";
private static final String TOPIC;
private static final String KAFKA_SERVER_ADDRR;
@Resource
private SysLogMapper sysLogMapper;
@Scheduled(cron = "${cleanTime}")
public void configureTasks() {
log.info("\u6267\u884c\u5b9a\u65f6\u6e05\u7406\u4efb\u52a1\u65f6\u95f4: {}", (Object)LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
final Properties props = new Properties();
if ("yes".equals(PropertyUtils.getProperty("openSasl"))) {
KafkaUtils.jaasConfig(props);
}
((Hashtable<String, String>)props).put("bootstrap.servers", SaticScheduleTask.KAFKA_SERVER_ADDRR);
try (final AdminClient adminClient = KafkaAdminClient.create(props)) {
final String[] topics = SaticScheduleTask.TOPIC.split(",");
final Map<TopicPartition, RecordsToDelete> recordsToDeleteMap = new HashMap<TopicPartition, RecordsToDelete>(16);
DeleteRecordsResult deleteRecordsResult = null;
for (final String topic : topics) {
final Map<Integer, Long> partitionInfoMap = this.getPartitionsForTopic(topic);
for (final Map.Entry<Integer, Long> entry2 : partitionInfoMap.entrySet()) {
final TopicPartition topicPartition = new TopicPartition(topic, (int)entry2.getKey());
final RecordsToDelete recordsToDelete = RecordsToDelete.beforeOffset((long)entry2.getValue());
recordsToDeleteMap.put(topicPartition, recordsToDelete);
}
}
SaticScheduleTask.log.debug("\u6e05\u7406\u96c6\u5408\uff1a{}", (Object)recordsToDeleteMap.toString());
deleteRecordsResult = adminClient.deleteRecords((Map)recordsToDeleteMap);
final Map<TopicPartition, KafkaFuture<DeletedRecords>> lowWatermarks = (Map<TopicPartition, KafkaFuture<DeletedRecords>>)deleteRecordsResult.lowWatermarks();
final Exception ex;
Exception e;
lowWatermarks.entrySet().forEach(entry -> {
try {
SaticScheduleTask.log.info("\u6e05\u7406\u6570\u636e\u4fe1\u606f \u4e3b\u9898\uff1a{} \u5206\u533a\uff1a{} \u6700\u65b0\u504f\u79fb\u91cf\uff1a{} \u7684\u7ed3\u679c\uff1a{}", new Object[] { entry.getKey().topic(), entry.getKey().partition(), ((DeletedRecords)((KafkaFuture)entry.getValue()).get()).lowWatermark(), ((KafkaFuture)entry.getValue()).isDone() });
}
catch (InterruptedException | ExecutionException ex2) {
e = ex;
SaticScheduleTask.log.error("\u83b7\u53d6kafka\u6e05\u7406\u6807\u8bb0\u7ed3\u679c\u5f02\u5e38\uff1a", (Throwable)e);
}
return;
});
this.recordSystemLog("true");
SaticScheduleTask.log.debug("\u5b9a\u65f6\u6e05\u7406\u5df2\u6d88\u8d39\u7684\u6570\u636e\u6210\u529f");
}
catch (Exception e2) {
this.recordSystemLog(null);
SaticScheduleTask.log.error("kafka\u5b9a\u65f6\u6267\u884c\u6e05\u7406\u6807\u8bb0\u5f02\u5e38\uff1a", (Throwable)e2);
}
}
private Consumer<Long, String> createConsumer() {
final Consumer<Long, String> consumer = (Consumer<Long, String>)new KafkaConsumer(KafkaConsumerConfig.consumerConfigs());
return consumer;
}
private Map<Integer, Long> getPartitionsForTopic(final String topic) {
final Map<Integer, Long> partitionInfoMap = new HashMap<Integer, Long>(16);
final Consumer<Long, String> consumer = this.createConsumer();
final Collection<PartitionInfo> partitionInfos = (Collection<PartitionInfo>)consumer.partitionsFor(topic);
final List<TopicPartition> tp = new ArrayList<TopicPartition>();
final List<TopicPartition> list;
final Consumer consumer2;
final Map<Integer, Long> map;
partitionInfos.forEach(str -> {
list.add(new TopicPartition(topic, str.partition()));
consumer2.assign((Collection)list);
consumer2.seekToEnd((Collection)list);
map.put(str.partition(), consumer2.position(new TopicPartition(topic, str.partition())));
return;
});
return partitionInfoMap;
}
private void recordSystemLog(final String value) {
final SysLog systemLog = new SysLog();
systemLog.setLogTime(Instant.now().toEpochMilli());
systemLog.setLogType((Byte)2);
if ("true".equals(value)) {
systemLog.setLogStatus((Byte)1);
systemLog.setLogLevel((Byte)2);
systemLog.setLogDesc("\u5b9a\u65f6\u6e05\u7406\u5df2\u6d88\u8d39\u7684\u6570\u636e\u6210\u529f\uff01");
}
else {
systemLog.setLogStatus((Byte)0);
systemLog.setLogLevel((Byte)1);
systemLog.setLogDesc("\u5b9a\u65f6\u6e05\u7406\u5df2\u6d88\u8d39\u7684\u6570\u636e\u5931\u8d25\uff01");
}
this.sysLogMapper.insertSelective(systemLog);
}
static {
log = LoggerFactory.getLogger((Class)SaticScheduleTask.class);
TOPIC = PropertyUtils.getProperty("cleanTopics");
KAFKA_SERVER_ADDRR = PropertyUtils.getProperty("bootstrapServers");
}
}