This article will introduce the monitoring of redis stream under springboot, creating consumer groups, deleting consumer groups, and pending queue monitoring and message ack and deletion
1 Infrastructure
import dependencies
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
</dependency>
configuration file
spring:
redis.:
host: 192.168.0.3
port: 6379
database: 0
timeout: 15000
lettuce:
pool:
max-idle: 50 # 连接池中的最大空闲连接
min-idle: 10 # 连接池中的最小空闲连接
max-active: 300 # 连接池的最大数据库连接数
max-wait: -1 #连接池最大阻塞等待时间
redis configuration class
@Configuration
public class MyRedisConfig {
@Bean
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
redisTemplate.setConnectionFactory(connectionFactory);
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.setValueSerializer(new StringRedisSerializer());
redisTemplate.setHashValueSerializer(RedisSerializer.string());
// hash的key也采用String的序列化方式
redisTemplate.setHashKeySerializer(new StringRedisSerializer());
Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<>(Object.class);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
jackson2JsonRedisSerializer.setObjectMapper(objectMapper);
objectMapper.activateDefaultTyping(objectMapper.getPolymorphicTypeValidator(), ObjectMapper.DefaultTyping.NON_FINAL, JsonTypeInfo.As.PROPERTY);
jackson2JsonRedisSerializer.setObjectMapper(objectMapper);
redisTemplate.setHashValueSerializer(jackson2JsonRedisSerializer);
return redisTemplate;
}
}
2 Objects that encapsulate messages and message IDs
Creation of the message object:
org.springframework.data.redis.connection.stream.StreamRecordsUse the static method of to create a message instance
. A stream message has two contents. It can be understood as: one is key and the other is value.
Both key and value can be defined using custom objects, bytes, and strings
ByteRecord rawBytes(Map<byte[], byte[]> raw)
ByteBufferRecord rawBuffer(Map<ByteBuffer, ByteBuffer> raw)
StringRecord string(Map<String, String> raw)
<S, K, V> MapRecord<S, K, V> mapBacked(Map<K, V> map)
<S, V> ObjectRecord<S, V> objectBacked(V value)
RecordBuilder<?> newRecord() // 通过builder方式来创建消息
RecordId represents the message ID
A message ID is unique. and has 2 parts
// ----------- 读取ID属性的实例方法
// 是否是系统自动生成的
boolean shouldBeAutoGenerated();
// 获取原始的id字符串
String getValue();
// 获取序列号部分
long getSequence();
// 获取时间戳部分
long getTimestamp();
// ----------- 创建ID的静态方法
RecordId of(@Nullable String value)
RecordId of(long millisecondsTime, long sequenceNumber)
RecordId autoGenerate()
3 Push Map message to Stream
3.1 Using RedisTemplate
@Autowired
private RedisTemplate<String, Object> redisTemplate;
public void send1() {
HashMap<Object, Object> map = new HashMap<>();
map.put("xxx", "xxx");
final MapRecord<String, Object, Object> record = StreamRecords.newRecord()
.in("mystream")
.ofMap(map)
.withId(RecordId.autoGenerate());
RecordId recordId = redisTemplate.opsForStream().add(record);
// 是否是自动生成的
boolean autoGenerated = recordId.shouldBeAutoGenerated();
// id值
String value = recordId.getValue();
// 序列号部分
long sequence = recordId.getSequence();
// 时间戳部分
long timestamp = recordId.getTimestamp();
}
public void send2() {
StringRecord record = StreamRecords.string(Collections.singletonMap("xxx", xxx))
.withStreamKey("mystream")
.withId(RecordId.autoGenerate());
RecordId recordId = redisTemplate.opsForStream().add(record);
// 是否是自动生成的
boolean autoGenerated = recordId.shouldBeAutoGenerated();
// id值
String value = recordId.getValue();
// 序列号部分
long sequence = recordId.getSequence();
// 时间戳部分
long timestamp = recordId.getTimestamp();
}
3.2 Using RedisConnection
@Autowired
private RedisConnectionFactory redisConnectionFactory;
public void test () {
// 创建消息记录, 以及指定stream
ByteRecord byteRecord = StreamRecords.rawBytes(Collections.singletonMap("name".getBytes(), "KevinBlandy".getBytes())).withStreamKey("mystream".getBytes());
// 获取连接
RedisConnection redisConnection = this.redisConnectionFactory.getConnection();
RecordId recordId = redisConnection.xAdd(byteRecord);
// 是否是自动生成的
boolean autoGenerated = recordId.shouldBeAutoGenerated();
// id值
String value = recordId.getValue();
// 序列号部分
long sequence = recordId.getSequence();
// 时间戳部分
long timestamp = recordId.getTimestamp();
}
4 Push object message to Stream
@Data
class Book {
private String title;
private String author;
public static Book create() {
Book book = new Book();
book.setTitle("xxx");
book.setAuthor("xxx");
return book;
}
}
@Autowired
private RedisTemplate<String, Object> redisTemplate;
public void sendRecord(String streamKey) {
Book book = Book.create();
log.info("产生一本书的信息:[{}]", book);
ObjectRecord<String, Book> record = StreamRecords.newRecord()
.in(streamKey)
.ofObject(book)
.withId(RecordId.autoGenerate());
RecordId recordId = redisTemplate.opsForStream()
.add(record);
log.info("返回的record-id:[{}]", recordId);
}
5 Create a specified consumer group, if this consumer group existed before, delete it
@Autowired
private RedisTemplate<String, Object> redisTemplate;
private final AtomicBoolean isCreated = new AtomicBoolean(false);
@PostConstruct
public void groupInfo() {
// 发送个心跳,保证stream已经存在
HashMap<Object, Object> map = new HashMap<>();
map.put("fileBeat","fileBeat...");
final MapRecord<String, Object, Object> record = StreamRecords.newRecord()
.in(StreamConstant.Document.streamName)
.ofMap(map)
.withId(RecordId.autoGenerate());
final StreamOperations<String, Object, Object> stream = redisTemplate.opsForStream();
stream.add(record);
final StreamInfo.XInfoGroups xInfoGroups = stream.groups(StreamConstant.Document.streamName);
Collection<StreamInfo.XInfoGroup> needDestroyColl = new ArrayList<>();
xInfoGroups.forEach(xInfoStream -> {
if (xInfoStream.groupName().equals(StreamConstant.Document.consumerGroup)) {
isCreated.set(true);
} else {
needDestroyColl.add(xInfoStream);
}
});
for (StreamInfo.XInfoGroup xInfoGroup : needDestroyColl) {
log.info("destroy consumer group[{}]...", xInfoGroup.groupName());
stream.destroyGroup(StreamConstant.Document.streamName,xInfoGroup.groupName());
}
if (isCreated.get()) return;
log.info("create consumer group[{}]...", StreamConstant.Document.consumerGroup);
stream.createGroup(StreamConstant.Document.streamName, StreamConstant.Document.consumerGroup);
}
6 Consumer group mode consumption information
6.1 Blocking consumption
Block consumption to monitor MapRecord
Notice:
- MapRecord means that the result is mapped into one
MapRecord<S, K, V>, S is stream, k is the key of a custom message, V is a specific message, and it must be sent when sendingMapRecord<S, K, V> - Need to specify
.keySerializer(new StringRedisSerializer())
@Configuration
public class RedisStreamConfiguration {
@Autowired
private RedisConnectionFactory redisConnectionFactory;
@Autowired
private StateListener2 stateListener;
@Autowired
@Qualifier("stream-core-pool")
private ThreadPoolTaskExecutor executor;
@Bean(initMethod = "start", destroyMethod = "stop")
public StreamMessageListenerContainer<String, MapRecord<String, String, String>> streamMessageListenerContainer() {
StreamMessageListenerContainer.StreamMessageListenerContainerOptions<String, MapRecord<String, String, String>>
options = StreamMessageListenerContainer.StreamMessageListenerContainerOptions
.builder()
// 一次最多获取多少条消息
.batchSize(5)
// 执行消息轮询的执行器
.executor(executor)
// 超时时间，设置为0，表示不超时（超时后会抛出异常）
.pollTimeout(Duration.ZERO)
// 消息消费异常的handler
.errorHandler(e-> log.error("发生了异常", e))
// 序列化器 或者RedisSerializer.string()
.serializer(new StringRedisSerializer())
.build();
StreamMessageListenerContainer<String, MapRecord<String, String, String>> streamMessageListenerContainer = StreamMessageListenerContainer
.create(this.redisConnectionFactory, options);
// 消费组B,手动ack
// receiveAutoAck(自动ack)
streamMessageListenerContainer.receive(
Consumer.from(StreamConstant.Document.consumerGroup, StreamConstant.Document.consumerName),
StreamOffset.create(StreamConstant.Document.streamName, ReadOffset.lastConsumed()),
stateListener
);
return streamMessageListenerContainer;
}
}
Combine with your own business implementation StreamListener, and ack or delete messages according to the business
@Slf4j
@Configuration
public class StateListener implements StreamListener<String, MapRecord<String,String, String>> {
@Autowired
private RedisTemplate<String, Object> redisTemplate;
@Override
@Async("stream-core-pool")
public void onMessage(MapRecord<String,String, String> message) {
StreamOperations<String, Object, Object> opsForStream = redisTemplate.opsForStream();
RecordId id = message.getId();
String value = message.getValue().get("xxx");
if (StrUtil.isBlank(value)){
log.error("接收到非法信息：{}",message.getValue());
opsForStream.acknowledge(StreamConstant.Document.streamName, StreamConstant.Document.consumerGroup, id);
opsForStream.delete(message);
return;
}
// 除去不合法字符
value = value.replaceAll("\"","");
log.info("received id:{} uuid:{}", id, value);
try {
// todo 业务
} catch (Exception e) {
log.error(e.getMessage());
e.printStackTrace();
} finally {
try {
final Long acknowledge = opsForStream.acknowledge(
StreamConstant.Document.streamName,
StreamConstant.Document.consumerGroup,
id
);
Long delete = 0L;
if (acknowledge != null && acknowledge == 1L) {
delete = opsForStream.delete(message);
}
log.info("acknowledge:{} delete:{}",acknowledge, delete);
} catch (Exception e) {
log.error(e.getMessage());
e.printStackTrace();
}
}
}
}
Block consumption to monitor ObjectRecord
Notice:
ObjectRecordIt means that the result is mapped into an object, and the object is also sent when sending- Need to specify
.keySerializer(new StringRedisSerializer()) - When using
ObjectRecordthe receive object, specify.objectMapper(new ObjectHashMapper())
@Configuration
public class RedisStreamConfiguration {
@Data
public static class Book {
private String title;
private String author;
public static Book create() {
Book book = new Book();
book.setTitle("xxx");
book.setAuthor("xxx");
return book;
}
}
@Autowired
private RedisConnectionFactory redisConnectionFactory;
@Autowired
private StateListener stateListener;
@Autowired
@Qualifier("stream-core-pool")
private ThreadPoolTaskExecutor executor;
@Bean(initMethod = "start", destroyMethod = "stop")
public StreamMessageListenerContainer<String, ObjectRecord<String, Book>> streamMessageListenerContainer() {
StreamMessageListenerContainer.StreamMessageListenerContainerOptions<String, ObjectRecord<String, Book>>
options = StreamMessageListenerContainer.StreamMessageListenerContainerOptions
.builder()
// 一次最多获取多少条消息
.batchSize(5)
// 执行消息轮询的执行器
.executor(executor)
// 超时时间，设置为0，表示不超时（超时后会抛出异常）
.pollTimeout(Duration.ZERO)
// 消息消费异常的handler
.errorHandler(new CustomErrorHandler())
// 序列化器 或者RedisSerializer.string()
.serializer(new StringRedisSerializer())
.keySerializer(new StringRedisSerializer())
.hashKeySerializer(new StringRedisSerializer())
.hashValueSerializer(new StringRedisSerializer())
.objectMapper(new ObjectHashMapper())
.targetType(Book.class)
.build();
StreamMessageListenerContainer<String, ObjectRecord<String, Book>> streamMessageListenerContainer =
StreamMessageListenerContainer.create(redisConnectionFactory, options);
// 消费组B,手动ack
// receiveAutoAck(自动ack)
streamMessageListenerContainer.receive(
Consumer.from(StreamConstant.Document.consumerGroup, StreamConstant.Document.consumerName),
StreamOffset.create(StreamConstant.Document.streamName, ReadOffset.lastConsumed()),
stateListener
);
return streamMessageListenerContainer;
}
}
Combine with your own business implementation StreamListener, and ack or delete messages according to the business
@Slf4j
@Configuration
public class StateListener implements StreamListener<String, ObjectRecord<String, Book>> {
@Autowired
private RedisTemplate<String, Object> redisTemplate;
@Override
@Async("stream-core-pool")
public void onMessage(ObjectRecord<String, Book> message) {
// 同上
}
}
6.2 Non-blocking consumption
Random consumption of messages is mainly through StreamOperations or RedicConnection consumption API
Obtained from RedisTemplateStreamOperations
StreamOperations<String, Object, Object> opsForStream = redisTemplate.opsForStream();
Read API for StreamOperations:
// 随机范围读取
<V> List<ObjectRecord<K, V>> range(Class<V> targetType, K key, Range<String> range)
<V> List<ObjectRecord<K, V>> range(Class<V> targetType, K key, Range<String> range, Limit limit)
// 根据消息ID或者偏移量读取
List<MapRecord<K, HK, HV>> read(StreamOffset<K>... streams)
<V> List<ObjectRecord<K, V>> read(Class<V> targetType, StreamOffset<K>... streams)
List<MapRecord<K, HK, HV>> read(StreamReadOptions readOptions, StreamOffset<K>... streams)
<V> List<ObjectRecord<K, V>> read(Class<V> targetType, StreamReadOptions readOptions, StreamOffset<K>... streams)
List<MapRecord<K, HK, HV>> read(Consumer consumer, StreamOffset<K>... streams)
<V> List<ObjectRecord<K, V>> read(Class<V> targetType, Consumer consumer, StreamOffset<K>... streams)
List<MapRecord<K, HK, HV>> read(Consumer consumer, StreamReadOptions readOptions, StreamOffset<K>... streams)
List<ObjectRecord<K, V>> read(Class<V> targetType, Consumer consumer, StreamReadOptions readOptions, StreamOffset<K>... streams)
// 随机逆向范围读取
List<MapRecord<K, HK, HV>> reverseRange(K key, Range<String> range)
List<MapRecord<K, HK, HV>> reverseRange(K key, Range<String> range, Limit limit)
<V> List<ObjectRecord<K, V>> reverseRange(Class<V> targetType, K key, Range<String> range)
<V> List<ObjectRecord<K, V>> reverseRange(Class<V> targetType, K key, Range<String> range, Limit limit)
// 消费者信息
XInfoConsumers consumers(K key, String group);
// 消费者信息
XInfoGroups groups(K key);
// stream信息
XInfoStream info(K key);
// 获取消费组，消费者中未确认的消息
PendingMessagesSummary pending(K key, String group);
PendingMessages pending(K key, Consumer consumer)
PendingMessages pending(K key, String group, Range<?> range, long count)
PendingMessages pending(K key, String group, Range<?> range, long count)
7 Listen for messages that are not ACKed and process them
Note: To use this method, you need to configure the hashValue serialization method of redisTemplate as String
@Component
@Slf4j
public class NonAckHandle {
@Autowired
@Qualifier("myStringRedisTemplate")
private RedisTemplate<String, Object> redisTemplate;
// @Scheduled(cron = "0 0/1 * * * ?")
@Scheduled(cron = "0/10 0/1 * * * ?")
public void doMonitor2() {
// 一次取10条未ack的消息
StreamOperations<String, String, String> stream = redisTemplate.opsForStream();
// XPENDING doc-state-stream doc-state-stream-group - + 10
PendingMessages pendingMessages = stream.pending(
StreamConstant.Document.streamName,
StreamConstant.Document.consumerGroup,
Range.of(Range.Bound.inclusive("-"),Range.Bound.inclusive("+")),
10
);
// 如果空
if (pendingMessages.isEmpty()) {
return;
}
for (PendingMessage next : pendingMessages) {
Consumer consumer = next.getConsumer();
RecordId recordId = next.getId();
log.warn("consumer={} RecordId={}", consumer, recordId);
// XRANGE doc-state-stream 1676974558423 1676974558423 count 1
final List<MapRecord<String, String, String>> res = stream.range(
StreamConstant.Document.streamName,
Range.of(Range.Bound.inclusive(recordId.getValue()),Range.Bound.inclusive(recordId.getValue())),
RedisZSetCommands.Limit.limit().count(1)
);
//
// List<MapRecord<String, String, String>> res2 = stream.read(
// StreamReadOptions.empty().count(1),
// StreamOffset.create(StreamConstant.Document.streamName, ReadOffset.from(recordId))
// );
res.forEach(ele -> this.acknowledge(ele, stream, recordId));
}
}
public void acknowledge(MapRecord<String, String, String> message, StreamOperations<String, String, String> opsForStream, RecordId recordId) {
String value = message.getValue().get("fileUuid");
if (StrUtil.isBlank(value)){
log.warn("接收到非法信息：{}", message.getValue());
ackAndDel(message, opsForStream, recordId);
return;
}
// 除去不合法字符
value = value.replaceAll("\"","");
log.info("received id:{} uuid:{}", recordId, value);
try {
// todo 业务
} catch (Exception e) {
log.error(e.getMessage());
e.printStackTrace();
} finally {
try {
ackAndDel(message, opsForStream, recordId);
} catch (Exception e) {
log.error(e.getMessage());
e.printStackTrace();
}
}
}
private void ackAndDel(MapRecord<String, String, String> message, StreamOperations<String, String, String> opsForStream, RecordId recordId) {
final Long acknowledge = opsForStream.acknowledge(
StreamConstant.Document.streamName,
StreamConstant.Document.consumerGroup,
recordId
);
Long delete = 0L;
if (acknowledge != null && acknowledge == 1L) {
delete = opsForStream.delete(message);
}
log.info("acknowledge:{} delete:{}",acknowledge, delete);
}
}