RocketMQ源码分析之Broker（2）

编程语言 2019-04-20 13:10:50 阅读次数: 0
版权声明： https://blog.csdn.net/ph3636/article/details/89415855

初始化

BrokerController

initialize()首先加载四个配置文件，主要是恢复以前的数据，当然第一次重启的话，这四个文件中也没有任何数据，他们主要是继承了ConfigManager类扩展了具体的文件存储方法以及编解码方法。初始化通信服务类，快速通信服务类监听端口为10909

public boolean initialize() throws CloneNotSupportedException {
    boolean result = this.topicConfigManager.load();
    result = result && this.consumerOffsetManager.load();
    result = result && this.subscriptionGroupManager.load();
    result = result && this.consumerFilterManager.load();
    this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
    NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();
    fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2);
    this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
    return result;
}

文件具体存储地方如下，每个文件存储的内容对象的具体属性可见上文。

class BrokerPathConfigHelper 
private static String brokerConfigPath = System.getProperty("user.home") + File.separator + "store"
    + File.separator + "config" + File.separator + "broker.properties";
public static String getBrokerConfigPath() {
    return brokerConfigPath;
}
public static void setBrokerConfigPath(String path) {
    brokerConfigPath = path;
}
public static String getTopicConfigPath(final String rootDir) {
    return rootDir + File.separator + "config" + File.separator + "topics.json";
}
public static String getConsumerOffsetPath(final String rootDir) {
    return rootDir + File.separator + "config" + File.separator + "consumerOffset.json";
}
public static String getSubscriptionGroupPath(final String rootDir) {
    return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json";
}
public static String getConsumerFilterPath(final String rootDir) {
    return rootDir + File.separator + "config" + File.separator + "consumerFilter.json";
}

创建发送消息，拉取消息，查询消息，控制台命令接收消息，客户端链接管理，心跳消息，结束事务以及消费者管理这几个线程池，最后把code与线程以及处理类绑定到通信服务端处理请求。

this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor(
    this.brokerConfig.getSendMessageThreadPoolNums(),
    this.brokerConfig.getSendMessageThreadPoolNums(),
    1000 * 60,
    TimeUnit.MILLISECONDS,
    this.sendThreadPoolQueue,
    new ThreadFactoryImpl("SendMessageThread_"));

this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor(
    this.brokerConfig.getPullMessageThreadPoolNums(),
    this.brokerConfig.getPullMessageThreadPoolNums(),
    1000 * 60,
    TimeUnit.MILLISECONDS,
    this.pullThreadPoolQueue,
    new ThreadFactoryImpl("PullMessageThread_"));

this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor(
    this.brokerConfig.getQueryMessageThreadPoolNums(),
    this.brokerConfig.getQueryMessageThreadPoolNums(),
    1000 * 60,
    TimeUnit.MILLISECONDS,
    this.queryThreadPoolQueue,
    new ThreadFactoryImpl("QueryMessageThread_"));

this.adminBrokerExecutor =
    Executors.newFixedThreadPool(this.brokerConfig.getAdminBrokerThreadPoolNums(), new ThreadFactoryImpl(
        "AdminBrokerThread_"));

this.clientManageExecutor = new ThreadPoolExecutor(
    this.brokerConfig.getClientManageThreadPoolNums(),
    this.brokerConfig.getClientManageThreadPoolNums(),
    1000 * 60,
    TimeUnit.MILLISECONDS,
    this.clientManagerThreadPoolQueue,
    new ThreadFactoryImpl("ClientManageThread_"));

this.heartbeatExecutor = new BrokerFixedThreadPoolExecutor(
    this.brokerConfig.getHeartbeatThreadPoolNums(),
    this.brokerConfig.getHeartbeatThreadPoolNums(),
    1000 * 60,
    TimeUnit.MILLISECONDS,
    this.heartbeatThreadPoolQueue,
    new ThreadFactoryImpl("HeartbeatThread_", true));

this.endTransactionExecutor = new BrokerFixedThreadPoolExecutor(
    this.brokerConfig.getEndTransactionThreadPoolNums(),
    this.brokerConfig.getEndTransactionThreadPoolNums(),
    1000 * 60,
    TimeUnit.MILLISECONDS,
    this.endTransactionThreadPoolQueue,
    new ThreadFactoryImpl("EndTransactionThread_"));

this.consumerManageExecutor =
    Executors.newFixedThreadPool(this.brokerConfig.getConsumerManageThreadPoolNums(), new ThreadFactoryImpl(
        "ConsumerManageThread_"));

public void registerProcessor() {

    SendMessageProcessor sendProcessor = new SendMessageProcessor(this);
    sendProcessor.registerSendMessageHook(sendMessageHookList);
    sendProcessor.registerConsumeMessageHook(consumeMessageHookList);

    this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor);
    this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor);
    this.remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendProcessor, this.sendMessageExecutor);
    this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendProcessor, this.sendMessageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor);

    this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor);
    this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);

    NettyRequestProcessor queryProcessor = new QueryMessageProcessor(this);
    this.remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
    this.remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);

    this.fastRemotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.queryMessageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.queryMessageExecutor);

    ClientManageProcessor clientProcessor = new ClientManageProcessor(this);
    this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.heartbeatExecutor);
    this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor);
    this.remotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);

    this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.heartbeatExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.CHECK_CLIENT_CONFIG, clientProcessor, this.clientManageExecutor);

    ConsumerManageProcessor consumerManageProcessor = new ConsumerManageProcessor(this);
    this.remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
    this.remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
    this.remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);

    this.fastRemotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor);

    this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.endTransactionExecutor);
    this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.endTransactionExecutor);

    AdminBrokerProcessor adminProcessor = new AdminBrokerProcessor(this);
    this.remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
    this.fastRemotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor);
}

BrokerStats定时程序打印每天的消息量

this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
    @Override
    public void run() {
        try {
            BrokerController.this.getBrokerStats().record();
        } catch (Throwable e) {
            log.error("schedule record error.", e);
        }
    }
}, initialDelay, period, TimeUnit.MILLISECONDS);

public void record() {
    this.msgPutTotalYesterdayMorning = this.msgPutTotalTodayMorning;
    this.msgGetTotalYesterdayMorning = this.msgGetTotalTodayMorning;

    this.msgPutTotalTodayMorning =
        this.defaultMessageStore.getStoreStatsService().getPutMessageTimesTotal();
    this.msgGetTotalTodayMorning =
        this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().get();

    log.info("yesterday put message total: {}", msgPutTotalTodayMorning - msgPutTotalYesterdayMorning);
    log.info("yesterday get message total: {}", msgGetTotalTodayMorning - msgGetTotalYesterdayMorning);
}

定时持久化ConsumerOffsetManager和ConsumerFilterManager里的数据

this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
    @Override
    public void run() {
        try {
            BrokerController.this.consumerOffsetManager.persist();
        } catch (Throwable e) {
            log.error("schedule persist consumerOffset error.", e);
        }
    }
}, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS);

this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
    @Override
    public void run() {
        try {
            BrokerController.this.consumerFilterManager.persist();
        } catch (Throwable e) {
            log.error("schedule persist consumer filter error.", e);
        }
    }
}, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);

protectBroker方法主要是当disableConsumeIfConsumerReadSlowly开启时定时检查当前消费组消费消息的进度，超过consumerFallbehindThreshold = 1024L * 1024 * 1024 * 16配置的大小后会剔除掉该订阅组，停止消费消息用来保护broker，因为存储消息的commitLog一个文件大小才为1024L * 1024 * 1024。

this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
    @Override
    public void run() {
        try {
            BrokerController.this.protectBroker();
        } catch (Throwable e) {
            log.error("protectBroker error.", e);
        }
    }
}, 3, 3, TimeUnit.MINUTES);

public void protectBroker() {
    if (this.brokerConfig.isDisableConsumeIfConsumerReadSlowly()) {
        final Iterator<Map.Entry<String, MomentStatsItem>> it = this.brokerStatsManager.getMomentStatsItemSetFallSize().getStatsItemTable().entrySet().iterator();
        while (it.hasNext()) {
            final Map.Entry<String, MomentStatsItem> next = it.next();
            final long fallBehindBytes = next.getValue().getValue().get();
            if (fallBehindBytes > this.brokerConfig.getConsumerFallbehindThreshold()) {
                final String[] split = next.getValue().getStatsKey().split("@");
                final String group = split[2];
                LOG_PROTECTION.info("[PROTECT_BROKER] the consumer[{}] consume slowly, {} bytes, disable it", group, fallBehindBytes);
                this.subscriptionGroupManager.disableConsume(group);
            }
        }
    }
}

printWaterMark方法主要是打印各个队列的任务大小以及最早的放入时间

this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
    @Override
    public void run() {
        try {
            BrokerController.this.printWaterMark();
        } catch (Throwable e) {
            log.error("printWaterMark error.", e);
        }
    }
}, 10, 1, TimeUnit.SECONDS);

dispatchBehindBytes方法主要是打印目前最新的消息偏移量和已经分发给消费队列以及索引创建服务之间的差距。

this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
    @Override
    public void run() {
        try {
            log.info("dispatch behind commit log {} bytes", BrokerController.this.getMessageStore().dispatchBehindBytes());
        } catch (Throwable e) {
            log.error("schedule dispatchBehindBytes error.", e);
        }
    }
}, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);

public long behind() {
    return DefaultMessageStore.this.commitLog.getMaxOffset() - this.reputFromOffset;
}

更新配置中心命名服务的地址信息，一个是从broker配置中获取，另一个是从配置的url中获取，官方推荐后者，因为可以动态更改配置信息，

扫描二维码关注公众号，回复： 5954233 查看本文章 
if (this.brokerConfig.getNamesrvAddr() != null) {
    this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr());
    log.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr());
} else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {
    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            try {
                BrokerController.this.brokerOuterAPI.fetchNameServerAddr();
            } catch (Throwable e) {
                log.error("ScheduledTask fetchNameServerAddr exception", e);
            }
        }
    }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
}

TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr());

public static String getWSAddr() {
        String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP);
        String wsDomainSubgroup = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr");
        String wsAddr = "http://" + wsDomainName + ":8080/rocketmq/" + wsDomainSubgroup;
        if (wsDomainName.indexOf(":") > 0) {
            wsAddr = "http://" + wsDomainName + "/rocketmq/" + wsDomainSubgroup;
        }
        return wsAddr;
    }

当broker为SLAVE时，更新主broker的地址以便进行同步操作，主要同步topic信息，消费偏移量，延迟队列偏移量，以及订阅组信息，这些主要用于消费端从该从broker拉取消息时使用。主broker只打印双方消息的偏移量差距。

if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
    if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
        this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
        this.updateMasterHAServerAddrPeriodically = false;
    } else {
        this.updateMasterHAServerAddrPeriodically = true;
    }

    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            try {
                BrokerController.this.slaveSynchronize.syncAll();
            } catch (Throwable e) {
                log.error("ScheduledTask syncAll slave exception", e);
            }
        }
    }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
} else {
    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            try {
                BrokerController.this.printMasterAndSlaveDiff();
            } catch (Throwable e) {
                log.error("schedule printMasterAndSlaveDiff error.", e);
            }
        }
    }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
}

 public void syncAll() {
    this.syncTopicConfig();
    this.syncConsumerOffset();
    this.syncDelayOffset();
    this.syncSubscriptionGroupConfig();
}

猜你喜欢

转载自blog.csdn.net/ph3636/article/details/89415855
今日推荐
openKylin 社区生态委员会第六次会议圆满召开
阿里云正式发布通义千问 2.5
Python 3.13 发布首个 Beta：实验性自由线程模式和 JIT、改进交互式解释器
Stack Overflow 拿我的代码去训练 AI 大模型，还封了我的账号
Pop!_OS 的 COSMIC 桌面完成 App Store 上架工作
报告：Django 仍然是 74% 开发者的首选
《2024 年一季度互联网投融资运行情况》研究报告
15 年前上了“FFmpeg 耻辱柱”，今天他还得谢谢咱——腾讯QQPlayer一雪前耻？
TIOBE 5 月榜单：Fortran “复活”进入 Top 10
GCC 14.1 发布
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
周排行
计算机组成与设计（七）—— 除法器
Integer Approximation(分治+枚举)
大话数据库索引
windows10系统JDK的配置及下载地址
mysql实现秒值转换中原六仔平台搭建
Codeforces Round #556 (Div. 1)
百练1064 网线主管
Codeforces 995F Cowmpany Cowmpensation
子集生成之增量构造法，位向量法，二进制法
ERROR: cmd.exe failed with args /c "/APK\gradle\rungradle.bat...
每日归档
更多
2024-05-10(38)
2024-05-09(35)
2024-05-08(42)
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022