[Message queue] RocketMQ configuration file broker

 

Preface

     The broker mainly involves broker configuration files, broker startup process, broker delay messages, and broker message storage.

One, broker configuration file

1.1 broker directory structure

├── abort
├── checkpoint
├── commitlog
│   └── 00000000000000000000
├── config
│   ├── consumerFilter.json
│   ├── consumerOffset.json
│   ├── delayOffset.json
│   ├── subscriptionGroup.json
│   ├── topics.json
├── consumequeue
│   ├── SCHEDULE_TOPIC_XXXX
│   │   ├── 0
│   │   │   └── 00000000000000000000
│   │   ├── 1
│   │   │   └── 00000000000000000000
│   │   └── 2
│   │       └── 00000000000000000000
│   └── TopicTest
│       ├── 0
│       │   └── 00000000000000000000
│       ├── 1
│       │   └── 00000000000000000000
│       ├── 2
│       │   └── 00000000000000000000
│       ├── 3
│       │   └── 00000000000000000000
│       ├── 4
│       │   └── 00000000000000000000
│       ├── 5
│       │   └── 00000000000000000000
│       ├── 6
│       │   └── 00000000000000000000
│       └── 7
│           └── 00000000000000000000
├── index
│   └── 20200502170505253
└── lock

• The commitlog records the messages actually stored by the broker.
• config records various configuration information of mq.
• The consumequeue records the consumption of each queue of the topic.
• index records the index information of mq and points to commitlog.

 1.2 topic.json

• Record the topic information on the broker, including queue information, read and write permissions.

{
    "dataVersion":{
        "counter":8,
        "timestamp":1588575388560
    },
    "topicConfigTable":{
        "TopicTest":{
            "order":false,
            "perm":6,
            "readQueueNums":8,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"TopicTest",
            "topicSysFlag":0,
            "writeQueueNums":8
        },
        "SELF_TEST_TOPIC":{
            "order":false,
            "perm":6,
            "readQueueNums":1,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"SELF_TEST_TOPIC",
            "topicSysFlag":0,
            "writeQueueNums":1
        },
        "%RETRY%consumer_group_test":{
            "order":false,
            "perm":6,
            "readQueueNums":1,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"%RETRY%consumer_group_test",
            "topicSysFlag":0,
            "writeQueueNums":1
        },
        "DefaultCluster":{
            "order":false,
            "perm":7,
            "readQueueNums":16,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"DefaultCluster",
            "topicSysFlag":0,
            "writeQueueNums":16
        },
        "%RETRY%order_consumer":{
            "order":false,
            "perm":6,
            "readQueueNums":1,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"%RETRY%order_consumer",
            "topicSysFlag":0,
            "writeQueueNums":1
        },
        "RMQ_SYS_TRANS_HALF_TOPIC":{
            "order":false,
            "perm":6,
            "readQueueNums":1,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"RMQ_SYS_TRANS_HALF_TOPIC",
            "topicSysFlag":0,
            "writeQueueNums":1
        },
        "broker-a":{
            "order":false,
            "perm":7,
            "readQueueNums":1,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"broker-a",
            "topicSysFlag":0,
            "writeQueueNums":1
        },
        "TBW102":{
            "order":false,
            "perm":7,
            "readQueueNums":8,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"TBW102",
            "topicSysFlag":0,
            "writeQueueNums":8
        },
        "BenchmarkTest":{
            "order":false,
            "perm":6,
            "readQueueNums":1024,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"BenchmarkTest",
            "topicSysFlag":0,
            "writeQueueNums":1024
        },
        "OFFSET_MOVED_EVENT":{
            "order":false,
            "perm":6,
            "readQueueNums":1,
            "topicFilterType":"SINGLE_TAG",
            "topicName":"OFFSET_MOVED_EVENT",
            "topicSysFlag":0,
            "writeQueueNums":1
        }
    }
}

1.3 subscriptionGroup.json

• Record various subscription relationships on the broker.

{
    "dataVersion":{
        "counter":1,
        "timestamp":1588575546295
    },
    "subscriptionGroupTable":{
        "SELF_TEST_C_GROUP":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"SELF_TEST_C_GROUP",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "consumer_group_test":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"consumer_group_test",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "CID_ONSAPI_OWNER":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"CID_ONSAPI_OWNER",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "CID_ONSAPI_PERMISSION":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"CID_ONSAPI_PERMISSION",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "order_consumer":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"order_consumer",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "TOOLS_CONSUMER":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"TOOLS_CONSUMER",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "CID_ONS-HTTP-PROXY":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"CID_ONS-HTTP-PROXY",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "FILTERSRV_CONSUMER":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"FILTERSRV_CONSUMER",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        },
        "CID_ONSAPI_PULL":{
            "brokerId":0,
            "consumeBroadcastEnable":true,
            "consumeEnable":true,
            "consumeFromMinEnable":true,
            "groupName":"CID_ONSAPI_PULL",
            "notifyConsumerIdsChangedEnable":true,
            "retryMaxTimes":16,
            "retryQueueNums":1,
            "whichBrokerWhenConsumeSlowly":1
        }
    }
}

1.4 delayOffset.json

• Record the consumption displacement of the broker's delay queue. In 1:1000, 1 represents the delay granularity, and 1000 represents the displacement.

{
    "offsetTable":{1:1000,2:1000,3:1000
    }
}

1.5 consumerOffset.json

• Record the consumption displacement for each queue of each consumer group for each topic on the broker.

{
    "offsetTable":{
        "TopicTest@please_rename_unique_group_name_4":{0:125,1:125,2:125,3:125,4:125,5:125,6:125,7:125
        },
        "TopicTest@order_consumer":{0:165,1:165,2:111,3:145,4:145,5:145,6:145,7:145
        },
        "TopicTest@consumer_group_test":{0:540,1:540,2:520,3:520,4:520,5:520,6:520,7:520
        },
        "%RETRY%consumer_group_test@consumer_group_test":{0:0
        },
        "%RETRY%please_rename_unique_group_name_4@please_rename_unique_group_name_4":{0:0
        },
        "%RETRY%order_consumer@order_consumer":{0:0
        }
    }
}