RabbitMq 解决队列拥堵 work模式 Topic模式 混合

其他 2020-04-16 10:38:46 阅读次数: 0

RabbitMq 一对多 多实例处理 解决拥堵
开发者难免会遇到坑爹的需求，在队列 同一组数据只能串行的情况下，适用下面操作。
探究 RabbitMq work模式 与Topic 模式

直接上代码：

package com.aspire.eab.common.configuration.rabbitmq.topic;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.amqp.core.*;
import org.springframework.amqp.core.Queue;
import org.springframework.beans.factory.annotation.Qualifier;

import java.util.HashMap;
import java.util.Map;


@Configuration
public class AsynTaskLogTopicConfig {

    // 异步交换机
    public static final String EAB_ASYN__TOPIC = "topicExchange.asynTaskLog";
    // 异步路由 多路由 匹配
    public static final String EAB_ASYN__ROUTING_FIRST = "topic.asyn.routing.FIRST.";
    public static final String EAB_ASYN_ROUTING_SECOND = "topic.asyn.routing.SECOND.";
    public static final String EAB_ASYN_ROUTING_THIRD = "topic.asyn.routing.THIRD.";
    public static final String EAB_ASYN_ROUTING_FOURTH = "topic.asyn.routing.FOURTH.";
    //总路由
    public static final String EAB_ASYN_ROUTING_ZERO = "topic.asyn.routing.ZERO.";

    // 异步队列
    public static final String EAB_ASYN_TASK_QUEUE_FIRST = "EAB_ASYN_TASK_QUEUE_FIRST";
    public static final String EAB_ASYN_TASK_QUEUE_SECOND = "EAB_ASYN_TASK_QUEUE_SECOND";
    public static final String EAB_ASYN_TASK_QUEUE_THIRD = "EAB_ASYN_TASK_QUEUE_THIRD";
    public static final String EAB_ASYN_TASK_QUEUE_FOURTH = "EAB_ASYN_TASK_QUEUE_FOURTH";
    public static final String EAB_ASYN_TASK_QUEUE_ZERO = "EAB_ASYN_TASK_QUEUE_ZERO";

    // Priority
    public static final String EAB_ASYN_TASK_QUEUE_PRIORITY = "EAB_ASYN_TASK_QUEUE_PRIORITY";



    @Bean(EAB_ASYN_TASK_QUEUE_FIRST)
    public Queue EAB_ASYN_TASK_QUEUE_FIRST(){
        return new Queue(EAB_ASYN_TASK_QUEUE_FIRST);
    }
    @Bean(EAB_ASYN_TASK_QUEUE_SECOND)
    public Queue EAB_ASYN_TASK_QUEUE_SECOND(){
        return new Queue(EAB_ASYN_TASK_QUEUE_SECOND);
    }
    @Bean(EAB_ASYN_TASK_QUEUE_THIRD)
    public Queue EAB_ASYN_TASK_QUEUE_THIRD(){
        return new Queue(EAB_ASYN_TASK_QUEUE_THIRD);
    }
    @Bean(EAB_ASYN_TASK_QUEUE_FOURTH)
    public Queue EAB_ASYN_TASK_QUEUE_FOURTH(){
        return new Queue(EAB_ASYN_TASK_QUEUE_FOURTH);
    }
    @Bean(EAB_ASYN_TASK_QUEUE_ZERO)
    public Queue EAB_ASYN_TASK_QUEUE_ZERO(){
        return new Queue(EAB_ASYN_TASK_QUEUE_ZERO);
    }

    @Bean(EAB_ASYN_TASK_QUEUE_PRIORITY)
    public Queue EAB_ASYN_TASK_QUEUE_PRIORITY() {
        Map<String, Object> args= new HashMap<>();
        args.put("x-max-priority", 100); //队列的最大属性参数 请参考源码
        return new Queue(AsynTaskLogTopicConfig.EAB_ASYN_TASK_QUEUE_PRIORITY, true, false, false, args);
    }

    @Bean(EAB_ASYN__TOPIC)
    public Exchange EAB_ASYN__TOPIC(){
        //Topic类型的交换机，durable是持久化
        return ExchangeBuilder.topicExchange(EAB_ASYN__TOPIC).durable(true).build();
    }

    @Bean
    public Binding BINDING_EAB_ASYN_TASK_QUEUE_FIRST(@Qualifier(EAB_ASYN_TASK_QUEUE_FIRST) Queue queue,
                                                      Exchange exchange){
        return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN__ROUTING_FIRST+"*").noargs();
    }
    @Bean
    public Binding BINDING_EAB_ASYN_TASK_QUEUE_SECOND(@Qualifier(EAB_ASYN_TASK_QUEUE_SECOND) Queue queue,
                                                     Exchange exchange){
        return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_SECOND+"*").noargs();
    }
    @Bean
    public Binding BINDING_EAB_ASYN_TASK_QUEUE_THIRD(@Qualifier(EAB_ASYN_TASK_QUEUE_THIRD) Queue queue,
                                                      Exchange exchange){
        return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_THIRD+".*").noargs();
    }
    @Bean
    public Binding BINDING_EAB_ASYN_TASK_QUEUE_FOURTH(@Qualifier(EAB_ASYN_TASK_QUEUE_FOURTH) Queue queue,
                                                     Exchange exchange){
        return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_FOURTH+"*").noargs();
    }
    @Bean
    public Binding BINDING_EAB_ASYN_TASK_QUEUE_FIFTH(@Qualifier(EAB_ASYN_TASK_QUEUE_ZERO) Queue queue,
                                                      Exchange exchange){
        return BindingBuilder.bind(queue).to(exchange).with(EAB_ASYN_ROUTING_ZERO+"*").noargs();
    }
}

生产者：

private void sendMassageTopic(String enterpriseId, AsynTask<Map<String, Object>> at) {
        if(StringUtils.isEmpty(enterpriseId) || null == at){
            logger.error("发送消息参数不完整！拒绝此消息！");
            return ;
        }
        //获取路由键
        String routingKey;
        String  hashEnt = StringUtil.getEntIdIndex(enterpriseId,rabbitMqParam.getGroupSize());

        if(this.checkPriorityEnterpriseId(enterpriseId)){
            if (logger.isInfoEnabled()) {
                logger.info(">>>>>>>>>>>>{} 企业发送数据变动优先队列 ", enterpriseId);
            }
            this.rabbitMq.convertAndSend(AsynTaskLogTopicConfig.EAB_ASYN_TASK_QUEUE_PRIORITY, JsonUtils.toJsonStr(at),
                    message -> {
                        message.getMessageProperties().setPriority(rabbitMqParam.getPriorityParameter());
                        return message;
                    });
        }else{
            if (logger.isInfoEnabled()) {
                logger.info(">>>>>>>>>>>>{} 企业发送Topic ", enterpriseId);
            }
            if(EabConstants.EAB_ASYN__ROUTING_FIRST.equals(hashEnt)){
                routingKey = AsynTaskLogTopicConfig.EAB_ASYN__ROUTING_FIRST+hashEnt;
            }else if(EabConstants.EAB_ASYN_ROUTING_SECOND.equals(hashEnt)){
                routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_SECOND+hashEnt;
            }else if(EabConstants.EAB_ASYN_ROUTING_SECOND.equals(hashEnt)){
                routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_THIRD+hashEnt;
            }else if(EabConstants.EAB_ASYN_ROUTING_SECOND.equals(hashEnt)){
                routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_FOURTH+hashEnt;
            }else{
                routingKey = AsynTaskLogTopicConfig.EAB_ASYN_ROUTING_ZERO+hashEnt;
            }
            this.rabbitMq.convertAndSend(AsynTaskLogTopicConfig.EAB_ASYN__TOPIC,routingKey,JsonUtils.toJsonStr(at));
        }

    }

消费者 ：

package com.aspire.eab.batch.application.job.WorkAsynTask;

import com.aspire.eab.batch.application.job.IncreaseImportJob;
import com.aspire.eab.batch.framework.constant.EabBatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;


@Component
public class WorkReceiver {
    private final Logger logger = LoggerFactory.getLogger(this.getClass());

    @Autowired
    private IncreaseImportJob importJob;

    @RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_FIRST)
    public void EAB_ASYN_TASK_QUEUE_FIRST(String message) {
        if (logger.isInfoEnabled()) {
            logger.info("FIRST 消息体mq请求参数：{}", message);
        }
        process(message);
    }
    @RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_SECOND)
    public void EAB_ASYN_TASK_QUEUE_SECOND(String message) {
        if (logger.isInfoEnabled()) {
            logger.info("SECOND 消息体mq请求参数：{}", message);
        }
        process(message);
    }
    @RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_THIRD)
    public void EAB_ASYN_TASK_QUEUE_THIRD(String message) {
        if (logger.isInfoEnabled()) {
            logger.info("THIRD 消息体mq请求参数：{}", message);
        }
        process(message);
    }
    @RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_FOURTH)
    public void EAB_ASYN_TASK_QUEUE_FOURTH(String message) {
        if (logger.isInfoEnabled()) {
            logger.info("FOURTH 消息体mq请求参数：{}", message);
        }
        process(message);
    }
    @RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_ZERO)
    public void EAB_ASYN_TASK_QUEUE_ZERO(String message) {
        if (logger.isInfoEnabled()) {
            logger.info("ZERO 消息体mq请求参数：{}", message);
        }
        process(message);
    }
    @RabbitListener(queues = EabBatchConstants.EAB_ASYN_TASK_QUEUE_PRIORITY)
    public void EAB_ASYN_TASK_QUEUE_PRIORITY(String message) {
        if (logger.isInfoEnabled()) {
            logger.info("PRIORITY 消息体mq请求参数：{}", message);
        }
        process(message);
    }
    private void process(String message) {
        this.importJob.process(message);
    }
}

生产者项目mq配置

@Configuration
public class RabbitMqConfig {

    @Autowired
    private RabbitMqParam rabbitMqParam;

    private static final Logger LOGGER = LoggerFactory.getLogger(RabbitMqConfig.class);


    /**
     * rabbitmq连接工厂
     *
     * @return ConnectionFactory
     */
    @Bean
    public ConnectionFactory connectionFactory() {
        CachingConnectionFactory connectionFactory = new CachingConnectionFactory();
        connectionFactory.setAddresses(rabbitMqParam.getAddresses());
        connectionFactory.setUsername(rabbitMqParam.getUsername());
        connectionFactory.setPassword(rabbitMqParam.getPassword());
        connectionFactory.setPublisherConfirms(rabbitMqParam.getPublisherconfirms());
        return connectionFactory;
    }

    /**
     * rabbitAdmin代理类
     *
     * @return RabbitAdmin
     */
    @Bean
    public RabbitAdmin rabbitAdmin(
            @Qualifier("connectionFactory") ConnectionFactory connectionFactory) {
        return new RabbitAdmin(connectionFactory);
    }

    /**
     * 创建rabbitTemplate 消息模板类
     * prototype原型模式:每次获取Bean的时候会有一个新的实例
     * 因为要设置回调类，所以应是prototype类型，如果是singleton类型，则回调类为最后一次设置
     *
     * @return RabbitTemplate
     */
    @Bean
    @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
    public RabbitTemplate rabbitTemplate() {
        RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory());
        rabbitTemplate.setMessageConverter(new Jackson2JsonMessageConverter());
        rabbitTemplate.setConfirmCallback(new ConfirmCallback() {
            @Override
            public void confirm(CorrelationData correlationData, boolean b, String s) {
                if (!b) {
                    throw new RuntimeException("send error " + s);
                }
            }
        });
        return rabbitTemplate;
    }

    /**
     * 监听工厂 - 全局
     *
     * @return SimpleRabbitListenerContainerFactory
     */
    @Bean
    public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory(
            ConnectionFactory connectionFactory, ThreadPoolTaskExecutor threadPoolTaskExecutor) {
        return this.getContainerFactory(connectionFactory, threadPoolTaskExecutor,
                rabbitMqParam.getConcurrentConsumers(),
                rabbitMqParam.getMaxConcurrentConsumers(),
                rabbitMqParam.getPrefetch());
    }

    /**
     * 监听工厂 - 需要单独引用的高并发消息消费
     *
     * @return SimpleRabbitListenerContainerFactory
     */
    @Bean("highConcurContainerFactory")
    public SimpleRabbitListenerContainerFactory pointTaskContainerFactory(
            ConnectionFactory connectionFactory,ThreadPoolTaskExecutor threadPoolTaskExecutor) {
        return this.getContainerFactory(connectionFactory, threadPoolTaskExecutor,
                rabbitMqParam.getHighConcurConsumers(),
                rabbitMqParam.getHighConcurMaxConsumers(),
                rabbitMqParam.getHighConcurPrefetch());
    }

    /**
     * 监听工厂 - 统一设置
     *
     * @return SimpleRabbitListenerContainerFactory
     */
    private SimpleRabbitListenerContainerFactory getContainerFactory(ConnectionFactory connectionFactory,
                                                                     ThreadPoolTaskExecutor threadPoolTaskExecutor,
                                                                     Integer concurrentConsumers,
                                                                     Integer maxConcurrentConsumers,
                                                                     Integer prefetchCount){
        SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory();
        factory.setConnectionFactory(connectionFactory);
        factory.setMessageConverter(new Jackson2JsonMessageConverter());
        //每个队列默认消费者数量
        factory.setConcurrentConsumers(concurrentConsumers);
        //每个队列最高消费者数量
//        if (maxConcurrentConsumers > concurrentConsumers) {
            factory.setMaxConcurrentConsumers(maxConcurrentConsumers);
//        }
        //单次消费拉取的消息个数
        factory.setPrefetchCount(prefetchCount);
        //指定执行容器
        //factory.setTaskExecutor(threadPoolTaskExecutor);
        factory.setTaskExecutor(this.consumersBrianThreadPool());
        return factory;
    }


    @Bean("Consumers_TaskExecutor")
    public ThreadPoolTaskExecutor  consumersBrianThreadPool(){
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        //核心线程数
        executor.setCorePoolSize(rabbitMqParam.getConsumersCorePoolSize());
        //最大线程数
        executor.setMaxPoolSize(rabbitMqParam.getConsumersMaxPoolSize());
//        //队列中最大的数 默认值
//        executor.setQueueCapacity(Integer.MAX_VALUE);
        executor.setAllowCoreThreadTimeOut(true);
        executor.setWaitForTasksToCompleteOnShutdown(true);
        //县城名称前缀
        executor.setThreadNamePrefix("consumersThreadPool_");
        //对拒绝task的处理策略
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
        //线程空闲后最大的存活时间 设置默认值
        executor.setKeepAliveSeconds(60);
        //初始化加载
        executor.initialize();
        if (LOGGER.isInfoEnabled()) {
            LOGGER.info("核心线程数：{},最大线程数:{},线程空闲time :{}", executor.getCorePoolSize(),executor.getMaxPoolSize(),executor.getKeepAliveSeconds());
        }
        return executor;
    }
}

其他代码就不贴了，都是写基础，另外 StringUtil.getEntIdIndex(enterpriseId,rabbitMqParam.getGroupSize());为hash 求模运算， 来入队的， 本来想搞个策略来完善这块， 时间有限。将就着吧。 。。

@你与昨日
发布了3 篇原创文章 · 获赞 6 · 访问量 100
私信 关注

猜你喜欢

转载自blog.csdn.net/weixin_42274449/article/details/105096713
今日推荐
开放签电子签章：停止新增，优化体验，前进更进（五一假期前工作）
开源日报 | 中学生开源前端动画引擎；全球首个Llama3 8B中文版开源模型；联想电脑恐出局；Linus讽刺AI炒作
“百模大战”必有一战 | 2024中国“百模大战”竞争格局分析
最强开源大模型 Llama 3 上架 Gitee AI
虽然老乡鸡开源的不是代码，但背后的原因却让人很暖心
富文本编辑器 Quill 2.0 重磅发布，特性、可靠性与开发者体验大幅提升
周排行
使用Redis中间件解决商品秒杀活动中出现的超卖问题（使用Java多线程模拟高并发环境）
野指针及c++指针使用注意点
redis 3.0　新特性
(翻译)火狐操作系统javascript API
微信小程序开发入门
mysql数据查询之五子句(where、group by、having、order by和limit)
Codeforces Round #517 Div. 1翻车记
在caffe 中实现Generative Adversarial Nets（二）
企业级漏洞扫描工具
java byte数组与String互转
每日归档
更多
2024-04-23(26)
2024-04-22(39)
2024-04-21(0)
2024-04-20(6)
2024-04-19(5)
2024-04-18(0)
2024-04-17(5)
2024-04-16(70)
2024-04-15(42)
2024-04-14(0)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022