fmysql-finkcdc-doris

流程：
mysql(5.7) -> kakfa(3.3.1)-> kafka -> doris(1.0.0)

--1.mysql建表
CREATE TABLE ods_mysql2 (
 item_id BIGINT,
 source_type BIGINT
);

--2.doris建表
CREATE TABLE flink_sink1
(
    item_id      INT,
    source_type  INT
)
AGGREGATE KEY(item_id, source_type)
DISTRIBUTED BY HASH(item_id) BUCKETS 1
PROPERTIES (
"replication_num" = "1",
"in_memory" = "false",
"storage_format" = "V2"
);

--3.flink创建数据源连接表
CREATE TABLE src_mysql(
 item_id BIGINT,
 source_type BIGINT,
 PRIMARY KEY (`item_id`) NOT ENFORCED
) WITH (
 'connector' = 'mysql-cdc',
 'hostname' = 'hadoop002',
 'port' = '3306',
 'username' = 'root',
 'password' = 'root',
 'database-name' = 'test',
 'table-name' = 'ods_mysql1',
 'scan.incremental.snapshot.enabled' = 'false'
);

--4.flink创建基于kafka的ods连接表
CREATE TABLE ods_mysql_2_kafka (
 item_id BIGINT,
 source_type BIGINT,
 PRIMARY KEY (`item_id`) NOT ENFORCED
) WITH (
 'connector' = 'upsert-kafka',
 'topic' = 'mysql_2_kafka',
 'properties.bootstrap.servers' = 'hadoop001:9092',
 'properties.group.id' = 'mysql_2_kafka_group',
 'key.format' = 'json',
 'value.format' = 'json'
);

--5.flink创建基于kafka的dwd连接表
CREATE TABLE dwd_kafka_2_kafka (
 item_id BIGINT,
 source_type BIGINT,
 PRIMARY KEY (`item_id`) NOT ENFORCED
) WITH (
 'connector' = 'upsert-kafka',
 'topic' = 'kafka_2_kafka',
 'properties.bootstrap.servers' = 'hadoop001:9092',
 'properties.group.id' = 'kafka_2_kafka_group',
  'key.format' = 'json',
 'value.format' = 'json'
);

--6.flink创建基于doris的下沉表
CREATE TABLE flink_sink1 ( 
    item_id      bigint,
    source_type  bigint
) 
WITH ( 
'connector' = 'doris', 
'fenodes' = 'hadoop001:8030', 
'table.identifier' = 'test.flink_sink1', 
'username' = 'root',
'password' = 'root' 
);

--7.执行数据流向insert into任务
insert into ods_mysql_2_kafka select * from src_mysql;
insert into dwd_kafka_2_kafka select * from ods_mysql_2_kafka;
insert into flink_sink1 select * from dwd_kafka_2_kafka;

配置返回格式
SET execution.result-mode=tableau;

start_fe.sh --daemon

CREATE TABLE src_mysql2(
 item_id BIGINT,
 source_type BIGINT
) WITH (
 'connector' = 'mysql-cdc',
 'hostname' = 'hadoop002',
 'port' = '3306',
 'username' = 'root',
 'password' = 'root',
 'database-name' = 'test',
 'table-name' = 'ods_mysql2',
 'scan.incremental.snapshot.enabled' = 'false'
);

CREATE TABLE mysql_2_doris
(
    item_id      bigint,
    source_type  bigint
)
AGGREGATE KEY(item_id, source_type)
DISTRIBUTED BY HASH(item_id) BUCKETS 1
PROPERTIES (
"replication_num" = "1",
"in_memory" = "false",
"storage_format" = "V2"
);

CREATE TABLE mysql_2_doris ( 
    item_id      bigint,
    source_type  bigint
) 
WITH ( 
'connector' = 'doris', 
'fenodes' = 'hadoop001:8030', 
'table.identifier' = 'test.mysql_2_doris', 
'username' = 'root',
'password' = 'root' 
);

insert into mysql_2_doris select * from src_mysql2;