RocketMQ源码分析之Broker（1）

其他 2019-04-20 13:10:53 阅读次数: 0
版权声明： https://blog.csdn.net/ph3636/article/details/89415576

实例化

本次举例的broker的主方法在org.apache.rocketmq.broker.BrokerStartup，同样的通过命令行启动可以读取入参配置的具体文件，根据启动命令配置的-c属性读取文件，并设置broker配置类BrokerConfig，服务端通信配置类NettyServerConfig，客户端通信配置类NettyClientConfig，具体信息存储配置类MessageStoreConfig，并且设置网络安全加密TLS属性，更改server监听端口号为10911，设置高可用主从broker同步监听端口为10912.

nettyClientConfig.setUseTLS(Boolean.parseBoolean(System.getProperty(TLS_ENABLE,
String.valueOf(TlsSystemConfig.tlsMode == TlsMode.ENFORCING))));
nettyServerConfig.setListenPort(10911);
messageStoreConfig.setHaListenPort(nettyServerConfig.getListenPort() + 1);

BrokerController

broker服务端核心控制类

public BrokerController(
    final BrokerConfig brokerConfig,
    final NettyServerConfig nettyServerConfig,
    final NettyClientConfig nettyClientConfig,
    final MessageStoreConfig messageStoreConfig
) {
    this.brokerConfig = brokerConfig;
    this.nettyServerConfig = nettyServerConfig;
    this.nettyClientConfig = nettyClientConfig;
    this.messageStoreConfig = messageStoreConfig;
    this.consumerOffsetManager = new ConsumerOffsetManager(this);
    this.topicConfigManager = new TopicConfigManager(this);
    this.pullMessageProcessor = new PullMessageProcessor(this);
    this.pullRequestHoldService = new PullRequestHoldService(this);
    this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService);
    this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this);
    this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener);
    this.consumerFilterManager = new ConsumerFilterManager(this);
    this.producerManager = new ProducerManager();
    this.clientHousekeepingService = new ClientHousekeepingService(this);
    this.broker2Client = new Broker2Client(this);
    this.subscriptionGroupManager = new SubscriptionGroupManager(this);
    this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig);
    this.filterServerManager = new FilterServerManager(this);

    this.slaveSynchronize = new SlaveSynchronize(this);

    this.sendThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getSendThreadPoolQueueCapacity());
    this.pullThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getPullThreadPoolQueueCapacity());
    this.queryThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getQueryThreadPoolQueueCapacity());
    this.clientManagerThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getClientManagerThreadPoolQueueCapacity());
    this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity());
    this.heartbeatThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getHeartbeatThreadPoolQueueCapacity());
    this.endTransactionThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getEndTransactionPoolQueueCapacity());

    this.brokerStatsManager = new BrokerStatsManager(this.brokerConfig.getBrokerClusterName());
    this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort()));

    this.brokerFastFailure = new BrokerFastFailure(this);
    this.configuration = new Configuration(
        log,
        BrokerPathConfigHelper.getBrokerConfigPath(),
        this.brokerConfig, this.nettyServerConfig, this.nettyClientConfig, this.messageStoreConfig
    );
}

ConsumerOffsetManager

主要保存topic与消费组与队列id的消费偏移量

privateConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =
new ConcurrentHashMap<String, ConcurrentMap<Integer, Long>>(512);

TopicConfigManager

主要保存topic的配置信息，初始化时创建多个系统topic，在autoCreateTopicEnable生效的时候，其中包含"TBW102"，主要用于自动创建消息生产者自定义且broker不存在的topic。

private final ConcurrentMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<String, TopicConfig>(1024);

public TopicConfigManager(BrokerController brokerController) {
    this.brokerController = brokerController;
    {
        // MixAll.SELF_TEST_TOPIC
        String topic = MixAll.SELF_TEST_TOPIC;
        TopicConfig topicConfig = new TopicConfig(topic);
        this.systemTopicList.add(topic);
        topicConfig.setReadQueueNums(1);
        topicConfig.setWriteQueueNums(1);
        this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
    }
    {
        // MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC
        if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) {
            String topic = MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC;
            TopicConfig topicConfig = new TopicConfig(topic);
            this.systemTopicList.add(topic);
            topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig()
                .getDefaultTopicQueueNums());
            topicConfig.setWriteQueueNums(this.brokerController.getBrokerConfig()
                .getDefaultTopicQueueNums());
            int perm = PermName.PERM_INHERIT | PermName.PERM_READ | PermName.PERM_WRITE;
            topicConfig.setPerm(perm);
            this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
        }
    }
    {
        // MixAll.BENCHMARK_TOPIC
        String topic = MixAll.BENCHMARK_TOPIC;
        TopicConfig topicConfig = new TopicConfig(topic);
        this.systemTopicList.add(topic);
        topicConfig.setReadQueueNums(1024);
        topicConfig.setWriteQueueNums(1024);
        this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
    }
    {

        String topic = this.brokerController.getBrokerConfig().getBrokerClusterName();
        TopicConfig topicConfig = new TopicConfig(topic);
        this.systemTopicList.add(topic);
        int perm = PermName.PERM_INHERIT;
        if (this.brokerController.getBrokerConfig().isClusterTopicEnable()) {
            perm |= PermName.PERM_READ | PermName.PERM_WRITE;
        }
        topicConfig.setPerm(perm);
        this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
    }
    {

        String topic = this.brokerController.getBrokerConfig().getBrokerName();
        TopicConfig topicConfig = new TopicConfig(topic);
        this.systemTopicList.add(topic);
        int perm = PermName.PERM_INHERIT;
        if (this.brokerController.getBrokerConfig().isBrokerTopicEnable()) {
            perm |= PermName.PERM_READ | PermName.PERM_WRITE;
        }
        topicConfig.setReadQueueNums(1);
        topicConfig.setWriteQueueNums(1);
        topicConfig.setPerm(perm);
        this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
    }
    {
        // MixAll.OFFSET_MOVED_EVENT
        String topic = MixAll.OFFSET_MOVED_EVENT;
        TopicConfig topicConfig = new TopicConfig(topic);
        this.systemTopicList.add(topic);
        topicConfig.setReadQueueNums(1);
        topicConfig.setWriteQueueNums(1);
        this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
    }
    {
        if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) {
            String topic = this.brokerController.getBrokerConfig().getMsgTraceTopicName();
            TopicConfig topicConfig = new TopicConfig(topic);
            this.systemTopicList.add(topic);
            topicConfig.setReadQueueNums(1);
            topicConfig.setWriteQueueNums(1);
            this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
        }
    }
}

PullMessageProcessor

处理消费端拉取消息请求的处理类

PullRequestHoldService

拉取请求持有类，也就是暂时没有可用消息的请求都会存储到这里，通过自身线程和重置线程ReputMessageService定时扫描是否有可用消息

private ConcurrentMap<String/* topic@queueId */, ManyPullRequest> pullRequestTable = new ConcurrentHashMap<String, ManyPullRequest>(1024);

NotifyMessageArrivingListener

通知消息到达监听类，主要是被重置线程ReputMessageService调用，当消息到达时，通过调用PullRequestHoldService来消费拉取请求

public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
      long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
      this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode,
          msgStoreTime, filterBitMap, properties);
  }

ConsumerManager

消费者管理类，主要保存消费者组的注册信息，通信通道以及topic的订阅信息

private final ConcurrentMap<String/* Group */, ConsumerGroupInfo> consumerTable = new ConcurrentHashMap<String, ConsumerGroupInfo>(1024);

class ConsumerGroupInfo 
    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
    private final String groupName;
    private final ConcurrentMap<String/* Topic */, SubscriptionData> subscriptionTable =
        new ConcurrentHashMap<String, SubscriptionData>();
    private final ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =
        new ConcurrentHashMap<Channel, ClientChannelInfo>(16);
    private volatile ConsumeType consumeType;
    private volatile MessageModel messageModel;
    private volatile ConsumeFromWhere consumeFromWhere;
    private volatile long lastUpdateTimestamp = System.currentTimeMillis();

DefaultConsumerIdsChangeListener

消费者状态更改监听类，主要是被ConsumerManager调用

public void handle(ConsumerGroupEvent event, String group, Object... args) {
    if (event == null) {
        return;
    }
    switch (event) {
        case CHANGE:
            if (args == null || args.length < 1) {
                return;
            }
            List<Channel> channels = (List<Channel>) args[0];
            if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {
                for (Channel chl : channels) {
                    this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);
                }
            }
            break;
        case UNREGISTER:
            this.brokerController.getConsumerFilterManager().unRegister(group);
            break;
        case REGISTER:
            if (args == null || args.length < 1) {
                return;
            }
            Collection<SubscriptionData> subscriptionDataList = (Collection<SubscriptionData>) args[0];
            this.brokerController.getConsumerFilterManager().register(group, subscriptionDataList);
            break;
        default:
            throw new RuntimeException("Unknown event " + event);
    }
}

ConsumerFilterManager

消费者消息过滤类，先用topic加队列id拉取消息，再通过包装该类进行tag或者sql92表达式过滤，主要是在消费者状态变更时，更新消息的过滤信息。

public ConsumerFilterManager(BrokerController brokerController) {
    this.brokerController = brokerController;
    this.bloomFilter = BloomFilter.createByFn(
        brokerController.getBrokerConfig().getMaxErrorRateOfBloomFilter(),
        brokerController.getBrokerConfig().getExpectConsumerNumUseFilter()
    );
    // then set bit map length of store config.
    brokerController.getMessageStoreConfig().setBitMapLengthConsumeQueueExt(
        this.bloomFilter.getM()
    );
}

ProducerManager

生产者管理类，主要包含生产者的注册信息

private final HashMap<String /* group name */, HashMap<Channel, ClientChannelInfo>> groupChannelTable = new HashMap<String, HashMap<Channel, ClientChannelInfo>>();

ClientHousekeepingService

客户端心跳监听类，定时扫描生产者，消费者，过滤服务的心跳。剔除不存活的服务通道。

public void start() {

    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            try {
                ClientHousekeepingService.this.scanExceptionChannel();
            } catch (Throwable e) {
                log.error("Error occurred when scan not active client channels.", e);
            }
        }
    }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
}

private void scanExceptionChannel() {
    this.brokerController.getProducerManager().scanNotActiveChannel();
    this.brokerController.getConsumerManager().scanNotActiveChannel();
    this.brokerController.getFilterServerManager().scanNotActiveChannel();
}

Broker2Client

主要处理broker到客户端的请求，比如事务状态检查，消费服务平衡消息队列。

SubscriptionGroupManager

订阅组信息管理类，当autoCreateSubscriptionGroup有效时可以自动创建订阅组信息，

private void init() {
    {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setGroupName(MixAll.TOOLS_CONSUMER_GROUP);
        this.subscriptionGroupTable.put(MixAll.TOOLS_CONSUMER_GROUP, subscriptionGroupConfig);
    }

    {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setGroupName(MixAll.FILTERSRV_CONSUMER_GROUP);
        this.subscriptionGroupTable.put(MixAll.FILTERSRV_CONSUMER_GROUP, subscriptionGroupConfig);
    }

    {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setGroupName(MixAll.SELF_TEST_CONSUMER_GROUP);
        this.subscriptionGroupTable.put(MixAll.SELF_TEST_CONSUMER_GROUP, subscriptionGroupConfig);
    }

    {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setGroupName(MixAll.ONS_HTTP_PROXY_GROUP);
        subscriptionGroupConfig.setConsumeBroadcastEnable(true);
        this.subscriptionGroupTable.put(MixAll.ONS_HTTP_PROXY_GROUP, subscriptionGroupConfig);
    }

    {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_PULL_GROUP);
        subscriptionGroupConfig.setConsumeBroadcastEnable(true);
        this.subscriptionGroupTable.put(MixAll.CID_ONSAPI_PULL_GROUP, subscriptionGroupConfig);
    }

    {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_PERMISSION_GROUP);
        subscriptionGroupConfig.setConsumeBroadcastEnable(true);
        this.subscriptionGroupTable.put(MixAll.CID_ONSAPI_PERMISSION_GROUP, subscriptionGroupConfig);
    }

    {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_OWNER_GROUP);
        subscriptionGroupConfig.setConsumeBroadcastEnable(true);
        this.subscriptionGroupTable.put(MixAll.CID_ONSAPI_OWNER_GROUP, subscriptionGroupConfig);
    }
}

class SubscriptionGroupConfig 
    private String groupName;
    private boolean consumeEnable = true;
    private boolean consumeFromMinEnable = true;
    private boolean consumeBroadcastEnable = true;
    private int retryQueueNums = 1;
    private int retryMaxTimes = 16;
    private long brokerId = MixAll.MASTER_ID;
    private long whichBrokerWhenConsumeSlowly = 1;
    private boolean notifyConsumerIdsChangedEnable = true;

BrokerOuterAPI

主要处理broker对外的请求，例如到NameServer或者从broker到主的请求，往NameServer注册本服务的topic信息等，同步主broker的消息信息到从broker。

FilterServerManager

过滤服务管理类，定时创建满足数量的过滤服务，目前还没有用到，后续会把拉取消息的过滤处理放在该服务中，减轻broker的压力。

SlaveSynchronize

从服务同步类，主要存在于从服务器，负责从主broker同步各种信息。

BrokerFastFailure

broker快速失败响应类，定时处理各个线程池中过期的任务，发送消息，拉取消息，心跳消息，事务结束消息。

猜你喜欢

转载自blog.csdn.net/ph3636/article/details/89415576
今日推荐
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
周排行
【转】spring中对控制反转和依赖注入的理解
tms webcore 安装和使用
java程序员进阶相关书籍
SpringMVC接受请求参数、
如何保存训练好的机器学习模型
MyEclipse、Eclipse设置项目JDK的三个地方
商超行业微信小程序开发定制一般多少钱 （行业技术人员解读）
Markdown编辑器语言——30分钟入门到到精通
Linux系统下MongoDB的简单安装与基本操作
Power Strings
每日归档
更多
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022