Detailed explanation of canal deployer package & canal adapter package parameters (2)

2.4. Detailed explanation of instance.properties file

argument	illustrate	Defaults	example (--to keep the default)
canal.instance.mysql.slaveId	The serverId concept in the mysql cluster configuration needs to be guaranteed to be unique to the id in the current mysql cluster (canal will be automatically generated after v1.0.26+ version, no need to manually specify)	none	482801
canal.instance.gtidon	Whether to enable the subscription mode of mysql gtid	false	--
canal.instance.master.address	mysql main library address	127.0.0.1:3306
canal.instance.master.journal.name	The binlog file that starts when the mysql main library is linked	none	mysql-bin.000421
canal.instance.master.position	The starting binlog offset when the mysql main library is linked	none	1048741422
canal.instance.master.timestamp	Timestamp of the binlog started when the mysql main library is linked	none	--
canal.instance.master.gtid	The corresponding gtid site when the mysql main library is linked	none	--
canal.instance.rds.accesskey	The ak information of the aliyun account ( if you do not need to automatically download the binlog on the oss after the local binlog has been cleared for more than 18 hours, you can ignore this value)	none	--
canal.instance.rds.secretkey	The sk information of the aliyun account (if you do not want to automatically download the binlog on the oss after the local binlog has been cleared for more than 18 hours, you can ignore this value)	none	--
canal.instance.rds.instanceId	The instance id information corresponding to aliyun rds ( if you do not need to automatically download the binlog on oss after the local binlog has been cleared for more than 18 hours, you can ignore this value)	none	--
canal.instance.tsdb.enable	Whether to enable the tsdb function of tablemeta	true	false
canal.instance.tsdb.url	tsdb connection url	jdbc:mysql://127.0.0.1:3306/canal_tsdb	--
canal.instance.tsdb.dbUsername	tsdb connection username	--	--
canal.instance.tsdb.dbPassword	tsdb user password	--	--
canal.instance.standby.address	Mysql standby database connection address	--	--
canal.instance.standby.journal.name	The binlog file that starts when the mysql standby database is connected	--	--
canal.instance.standby.position	The starting binlog offset when the mysql standby database is connected	--	--
canal.instance.standby.timestamp	The timestamp of the binlog started when the mysql standby database is connected	--	--
canal.instance.standby.gtid	The timestamp of the binlog started when the mysql standby database is connected	--	--
canal.instance.dbUsername	mysql database account	canal	u_canal_src_db
canal.instance.dbPassword	mysql database password	canal	xxxxxx
canal.instance.connectionCharset	mysql character set	UTF-8	--
canal.instance.enableDruid	Whether to use druid database password encryption	false	--
canal.instance.pwdPublicKey	Cipher public key, need to open canal.instance.enableDruid	xxxxx	--
canal.instance.filter.regex	mysql 过滤表白名单，Perl正则表达式	.\\..	ddo_main.objective_release,ddo_main.ddo_post_info, dhr_organization.department,dhr_organization.position_info
canal.instance.filter.black.regex	mysql 过滤表黑名单，Perl正则表达式	--	--
canal.mq.topic	mq 中的 topic 名	--	mysql_mysql
canal.mq.dynamicTopic	mq 中的动态 topic 规则，1.1.3 开始支持	mytest1.user,mytest2\\..,.\\..*	--
canal.mq.partition	单队列模式下的分区下标	0	--
canal.mq.partitionsNum	hash 模式下的分区数	--	--
canal.mq.partitionHash	hash 规则定义，库名.表名:唯一主键，1.1.3 支持新语法	--	--

3. canal adapter 包参数详解

3.1. canal adapter 目录结构

tree canal 折叠源码

[root]# ll

总用量 8

drwxr-xr-x 2 root root 90 7月 27 19:52 bin

drwxrwxrwx 6 root root 134 7月 28 16:52 conf

drwxr-xr-x 2 root root 4096 6月 29 09:39 lib

drwxrwxrwx 3 root root 20 6月 29 10:58 logs

drwxrwxrwx 2 root root 4096 9月 2 2019 plugin

3.2. conf 目录

tree canal conf 折叠源码

[root]# ll

总用量 20

-rwxrwxrwx 1 root root 2099 7月 28 11:30 application.yml

-rwxrwxrwx 1 root root 170 9月 2 2019 bootstrap.yml

drwxr-xr-x 2 root root 67 6月 29 18:50 es

drwxr-xr-x 2 root root 31 6月 29 09:39 hbase

-rwxrwxrwx 1 root root 2172 9月 2 2019 logback.xml

drwxrwxrwx 2 root root 29 6月 29 09:39 META-INF

drwxrwxrwx 2 root root 4096 7月 28 15:19 rdb

application.yml —— canal client 主配置文件
logback.xml —— 日志功能配置文件
es、hbase、rdb —— 分别对应三类数据库的配置目录
bootstrap.yml —— canal manager 连接信息配置文件

3.3. application.yml 文件配置详解

application.yml 折叠源码

server:

port: 8081 # canal metrics 收集端口

spring:

jackson:

date-format: yyyy-MM-dd HH:mm:ss

time-zone: GMT+8

default-property-inclusion: non_null

canal.conf:

mode: tcp # kafka rocketMQ

# canalServerHost: # canal server 连接地址，单机模式下使用

zookeeperHosts: # zk 集群连接地址，HA 模式时配置

# mqServers: 127.0.0.1:6667 #or rocketmq

# flatMessage: true

batchSize: 500

syncBatchSize: 1000

retries: 0

timeout:

accessKey:

secretKey:

srcDataSources:

defaultDS:

url: jdbc:mysql://:3306/ddo_main?useUnicode=true # 源库的连接信息

username: u_canal_src_db # 源库连接用户名

password: xxxxxx # 源库用户名密码

canalAdapters:

- instance: mid_db # canal instance Name or mq topic name # canal 实例名

groups: # canal 分组列表

- groupId: g1 # canal 分组id, 如果是MQ模式将用到该值

outerAdapters: # 分组内适配器列表

- name: logger # 日志打印适配器

- name: rdb # 指定为rdb类型同步

key: mysql1 # 指定adapter的唯一key, 与表映射配置中outerAdapterKey对应

properties:

jdbc.driverClassName: com.mysql.jdbc.Driver # jdbc驱动名, 部分jdbc的jar包需要自行放致lib目录下

jdbc.url: jdbc:mysql://:3306/canal_mid_db?useUnicode=true # 源数据库 jdbc url 地址

jdbc.username: u_canal_src_mid_db # 源数据库 jdbc 连接用户名

jdbc.password: xxxxxx # 源数据库 jdbc 用户密码

3.4. rdb 配置文件详解

rdb yml 展开源码

ataSourceKey: defaultDS # 源数据源的key, 对应上面配置的srcDataSources中的值

destination: mysql_mysql # cannal的instance或者MQ的topic

groupId: g1 # 对应MQ模式下的groupId, 只会同步对应groupId的数据

uterAdapterKey: mysql1 # adapter key, 对应上面配置outAdapters中的key

concurrent: true # 是否按主键hash并行同步, 并行同步的表必须保证主键不会更改及主键不能为其他同步表的外键!!

dbMapping:

database: ddo_main # 源数据源的database/shcema

table: ddo_post_info # 源数据源表名

targetTable: canal_mid_db.ddo_post_info # 目标数据源的库名.表名

targetPk: # 主键映射

id: id # 如果是复合主键可以换行映射多个

mapAll: true # 是否整表映射, 要求源表和目标表字段名一模一样 (如果targetColumns也配置了映射,则以targetColumns配置为准)

# targetColumns: # 字段映射, 格式: 目标表字段: 源表字段, 如果字段名一样源表字段名可不填

# id:

# name:

# role_id:

# c_time:

# test1:

# etlCondition: "where c_time>={}"

commitBatch: 3000 # 批量提交的大小

3.5. es 配置文件详解

es yml 折叠源码

dataSourceKey: defaultDS # 源数据源的key, 对应上面配置的srcDataSources中的值

destination: mysql_es # cannal的instance或者MQ的topic

groupId: g1 # 对应MQ模式下的groupId, 只会同步对应groupId的数据

esMapping:

_index: objective_index # es 索引名称

_type: _doc # es 的type名称, es7下无需配置此项

_id: oId # es 的_id, 如果不配置该项必须配置下面的pk项_id则会由es自动分配

upsert: true # 是否开启更新插入

# pk: id

sql: "SELECT

o.id AS oId,

o.type AS oType,

o.kind AS oKind,

o.strategy AS oStrategy,

o.version AS oVersion,

o.interval_id AS oIntervalId,

o.ldap AS oLdap,

o.dept_id AS oDeptId,

o.update_time AS oUpdateTime,

o.create_time AS oCreateTime,

pi.id AS piDdoPostId,

pi.state AS piState,

p.emp_dept_role_id AS pEmpDeptRoleId,

p.emp_id AS pEmpId,

p.full_ancestor_emp_dept_role_ids AS pFullAncestorEmpDeptRoleIds,

p.offspring_detection AS pOffSpringDetection,

d.tenement_id AS dTenementId,

d.dept_id AS dDeptId,

d.dept_name AS dDeptName,

d.dept_level AS dDeptLevel,

d.dept_path AS dDeptPath,

d.manager_emp_id AS dManagerEmpId,

d.manager_ldap AS dManagerLdap,

d.hrbp_emp_id AS dHrbpEmpId,

d.hrbp_ldap AS dHrbpLdap,

d.hrbp_manager_emp_id AS dHrbpManagerEmpId,

d.hrbp_manager_ldap AS dHrbpManagerLdap

FROM objective_release o

LEFT JOIN ddo_post_info pi ON o.ldap = pi.ldap AND o.dept_id = pi.dept_id AND o.interval_id = pi.interval_id

LEFT JOIN position_info p ON o.ldap = p.ldap AND o.dept_id = p.dept_id

LEFT JOIN department d ON o.dept_id = d.dept_id"

# objFields:

# _labels: array:;

etlCondition: "WHERE p.tenement_id = 1 AND p.deleted = 0 AND d.tenement_id = 1" # etl 的条件参数

commitBatch: 3000 # 提交批大小

Detailed explanation of canal deployer package & canal adapter package parameters (2)

Guess you like