Article Directory
video link
Chapter 2: Data Acquisition Based on Flink CDC
bxg code documentation
Project offline data preparation
- mysql
create database if not exists bxg character set utf8;
create database if not exists bxg_1 character set utf8;
- Run scripts bxg.sql, bxg1.sql
MySQL mapping table
create database if not exists mysql;
DROP TABLE IF EXISTS mysql.mysql_oe_stu_course_order;
CREATE TABLE if not exists mysql.mysql_oe_stu_course_order
(
`id` INT,
`student_course_id` INT,
`order_id` STRING,
`order_detail_id` STRING,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
PRIMARY KEY (`id`) NOT ENFORCED
) WITH (
'connector' = 'mysql',
'hostname' = 'rm-2ze047w59ovk9299x.rwlb.rds.aliyuncs.com',
'port' = '3306',
'username' = 'itcast',
'password' = 'Itcast123',
'database-name' = 'bxg',
'table-name' = 'oe_stu_course_order'
);
CREATE TABLE if not exists mysql.mysql_oe_stu_course
(
`id` INT,
`student_id` STRING,
`course_id` INT,
`status` TINYINT,
`contract_status` TINYINT,
`learn_status` TINYINT,
`service_days` SMALLINT,
`service_expires` TIMESTAMP(3),
`validity_days` INT,
`validity_expires` TIMESTAMP(3),
`terminate_cause` TINYINT,
`effective_date` TIMESTAMP(3),
`finished_time` TIMESTAMP(3),
`total_progress` DECIMAL(10, 2),
`purchase_time` INT,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
PRIMARY KEY (`id`) NOT ENFORCED
) WITH (
'connector' = 'mysql',
'hostname' = 'rm-2ze047w59ovk9299x.rwlb.rds.aliyuncs.com',
'port' = '3306',
'username' = 'itcast',
'password' = 'Itcast123',
'database-name' = 'bxg',
'table-name' = 'oe_stu_course'
);
CREATE TABLE if not exists mysql.mysql_oe_order
(
`id` STRING,
`channel` STRING,
`student_id` STRING,
`order_no` STRING,
`total_amount` DECIMAL(10, 2),
`discount_amount` DECIMAL(10, 2),
`charge_against_amount` DECIMAL(10, 2),
`payable_amount` DECIMAL(10, 2),
`status` TINYINT,
`pay_status` TINYINT,
`pay_time` TIMESTAMP(3),
`paid_amount` DECIMAL(10, 2),
`effective_date` TIMESTAMP(3),
`terminal` TINYINT,
`refund_status` TINYINT,
`refund_amount` DECIMAL(10, 2),
`refund_time` TIMESTAMP(3),
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
PRIMARY KEY (id) NOT ENFORCED
) WITH (
'connector' = 'mysql',
'hostname' = 'rm-2ze047w59ovk9299x.rwlb.rds.aliyuncs.com',
'port' = '3306',
'username' = 'itcast',
'password' = 'Itcast123',
'database-name' = 'bxg',
'table-name' = 'oe_order'
);
CREATE TABLE if not exists mysql.mysql_oe_course
(
`id` INT,
`grade_name` STRING,
`bigimg_path` STRING,
`video_url` STRING,
`img_alt` STRING,
`description` STRING,
`detailimg_path` STRING,
`smallimg_path` STRING,
`sort` INT,
`status` STRING,
`learnd_count` INT,
`learnd_count_flag` INT,
`original_cost` DECIMAL(10, 2),
`current_price` DECIMAL(10, 2),
`course_length` DECIMAL(10, 2),
`menu_id` INT,
`is_free` BOOLEAN,
`course_detail` STRING,
`course_detail_mobile` STRING,
`course_detail1` STRING,
`course_detail1_mobile` STRING,
`course_plan_detail` STRING,
`course_plan_detail_mobile` STRING,
`course_detail2` STRING,
`course_detail2_mobile` STRING,
`course_outline` STRING,
`common_problem` STRING,
`common_problem_mobile` STRING,
`lecturer_id` INT,
`is_recommend` INT,
`recommend_sort` INT,
`qqno` STRING,
`description_show` INT,
`rec_img_path` STRING,
`pv` INT,
`course_type` INT,
`default_student_count` INT,
`study_status` INT,
`online_course` INT,
`course_level` INT,
`content_type` INT,
`recommend_type` INT,
`employment_rate` STRING,
`employment_salary` STRING,
`score` STRING,
`cover_url` STRING,
`offline_course_url` STRING,
`outline_url` STRING,
`project_page_url` STRING,
`preschool_test_flag` BOOLEAN,
`service_period` INT,
`included_validity_period` TINYINT,
`validity_period` INT,
`qualified_jobs` STRING,
`work_year_min` INT,
`work_year_max` INT,
`promote_flag` BOOLEAN,
`create_person` STRING,
`update_person` STRING,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`is_delete` BOOLEAN,
PRIMARY KEY (`id`) NOT ENFORCED
) WITH (
'connector' = 'mysql',
'hostname' = 'rm-2ze047w59ovk9299x.rwlb.rds.aliyuncs.com',
'port' = '3306',
'username' = 'itcast',
'password' = 'Itcast123',
'database-name' = 'bxg',
'table-name' = 'oe_course'
);
stream data preparation
num.txt
1
makedata.log (empty file)
start.sh
#!/bin/bash
echo "开始时间：`date '+%Y-%m-%d %H:%M:%S'`"
echo "大概时间：`expr $2 \* 60`秒"
nohup sh ./create-log.sh $@ > makedata.log 2>&1 &
tail -f makedata.log
create-log.sh
#!/bin/bash
echo "执行流数据写入脚本，10s后开启流数据写入!"
sleep 10
step=$1
total=`expr $2 \* 60`
echo "1" > /root/sql/num.txt
num=$(cat num.txt)
for ((i=0;i<total;i=(i+step)));do
sh ./insert-data.sh $num $@
num=`expr $num + 1`
echo $num > num.txt
sleep $step
done
exit 0
insert-data.sh
#!/bin/bash
num=$1
oe_stu_course_order="
replace into bxg.oe_stu_course_order
select id,
student_course_id,
order_id,
order_detail_id,
timestampadd(second, timestampdiff(second, update_time, now()), create_time) as create_time,
now() as update_time,
delete_flag
from bxg_1.oe_stu_course_order
where rn = ${num}
;
"
oe_stu_course="
replace into bxg.oe_stu_course
select id,
student_id,
course_id,
status,
contract_status,
learn_status,
service_days,
timestampadd(second, timestampdiff(second, update_time, now()), service_expires) as service_expires,
validity_days,
timestampadd(second, timestampdiff(second, update_time, now()), validity_expires) as validity_expires,
terminate_cause,
timestampadd(second, timestampdiff(second, update_time, now()), effective_date) as effective_date,
timestampadd(second, timestampdiff(second, update_time, now()), finished_time) as finished_time,
total_progress,
purchase_time,
timestampadd(second, timestampdiff(second, update_time, now()), create_time) as create_time,
now() as update_time,
delete_flag
from bxg_1.oe_stu_course
where rn = ${num}
;
"
oe_order="
replace into bxg.oe_order
select id,
channel,
student_id,
order_no,
total_amount,
discount_amount,
charge_against_amount,
payable_amount,
status,
pay_status,
timestampadd(second, timestampdiff(second, update_time, now()), pay_time) as pay_time,
paid_amount,
timestampadd(second, timestampdiff(second, update_time, now()), effective_date) as effective_date,
terminal,
refund_status,
refund_amount,
timestampadd(second, timestampdiff(second, update_time, now()), refund_time) as refund_time,
timestampadd(second, timestampdiff(second, update_time, now()), create_time) as create_time,
now() as update_time,
delete_flag
from bxg_1.oe_order
where rn = ${num}
;
"
i=1
for argument in $@
do
if [ $i -ge 4 ];then
case $argument in
"oe_stu_course_order")
sql=$oe_stu_course_order
;;
"oe_stu_course")
sql=$oe_stu_course
;;
"oe_order")
sql=$oe_order
;;
"all")
sql=$oe_stu_course_order$oe_stu_course$oe_order
;;
esac
echo $sql
mysql --host=rm-2ze047w59ovk9299x.rwlb.rds.aliyuncs.com --port=3306 --user=itcast --password=Itcast123 --database=bxg -e"${sql}"
fi
i=`expr $i + 1`
done
dimension table
Create paimon_dim table
- Build the library in the dev job
create database if not exists paimon.dim;
CREATE TABLE if not exists paimon.dim.dim_oe_course
(
`id` INT,
`grade_name` STRING,
`bigimg_path` STRING,
`video_url` STRING,
`img_alt` STRING,
`description` STRING,
`detailimg_path` STRING,
`smallimg_path` STRING,
`sort` INT,
`status` STRING,
`learnd_count` INT,
`learnd_count_flag` INT,
`original_cost` DECIMAL(10, 2),
`current_price` DECIMAL(10, 2),
`course_length` DECIMAL(10, 2),
`menu_id` INT,
`is_free` BOOLEAN,
`course_detail` STRING,
`course_detail_mobile` STRING,
`course_detail1` STRING,
`course_detail1_mobile` STRING,
`course_plan_detail` STRING,
`course_plan_detail_mobile` STRING,
`course_detail2` STRING,
`course_detail2_mobile` STRING,
`course_outline` STRING,
`common_problem` STRING,
`common_problem_mobile` STRING,
`lecturer_id` INT,
`is_recommend` INT,
`recommend_sort` INT,
`qqno` STRING,
`description_show` INT,
`rec_img_path` STRING,
`pv` INT,
`course_type` INT,
`default_student_count` INT,
`study_status` INT,
`online_course` INT,
`course_level` INT,
`content_type` INT,
`recommend_type` INT,
`employment_rate` STRING,
`employment_salary` STRING,
`score` STRING,
`cover_url` STRING,
`offline_course_url` STRING,
`outline_url` STRING,
`project_page_url` STRING,
`preschool_test_flag` BOOLEAN,
`service_period` INT,
`included_validity_period` INT,
`validity_period` INT,
`qualified_jobs` STRING,
`work_year_min` INT,
`work_year_max` INT,
`promote_flag` BOOLEAN,
`create_person` STRING,
`update_person` STRING,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`is_delete` BOOLEAN,
PRIMARY KEY (id) NOT ENFORCED
);
mysql_to_paimon_dim task
- Configure the job in the mysql_to_paimon_dim job, data insertion
insert into paimon.dim.dim_oe_course
select * from mysql.mysql_oe_course;
fact sheet
ODS layer
Kafka start
cd /export/server/kafka/bin/
./kafka-server-start.sh -daemon ../config/server.properties
ECS creates ods layer kafka topic
kafka-topics.sh --create --topic ods_oe_order --bootstrap-server 172.16.10.106:9092
kafka-topics.sh --create --topic ods_oe_stu_course --bootstrap-server 172.16.10.106:9092
kafka-topics.sh --create --topic ods_oe_stu_course_order --bootstrap-server 172.16.10.106:9092
kafka_ods table
Build databases and tables in dev jobs
create database if not exists dw;
CREATE TABLE if not exists dw.kafka_ods_oe_stu_course_order
(
`id` INT,
`student_course_id` INT,
`order_id` STRING,
`order_detail_id` STRING,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
proctime as proctime(),
PRIMARY KEY (id) NOT ENFORCED
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'ods_oe_stu_course_order',
'properties.bootstrap.servers' = '172.16.10.106:9092',
'key.format' = 'json',
'value.format' = 'json'
);
CREATE TABLE if not exists dw.kafka_ods_oe_stu_course
(
`id` INT,
`student_id` STRING,
`course_id` INT,
`status` INT,
`contract_status` INT,
`learn_status` INT,
`service_days` INT,
`service_expires` TIMESTAMP(3),
`validity_days` INT,
`validity_expires` TIMESTAMP(3),
`terminate_cause` INT,
`effective_date` TIMESTAMP(3),
`finished_time` TIMESTAMP(3),
`total_progress` DECIMAL(10, 2),
`purchase_time` INT,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
proctime as proctime(),
PRIMARY KEY (id) NOT ENFORCED
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'ods_oe_stu_course',
'properties.bootstrap.servers' = '172.16.10.106:9092',
'key.format' = 'json',
'value.format' = 'json'
);
CREATE TABLE if not exists dw.kafka_ods_oe_order
(
`id` STRING,
`channel` STRING,
`student_id` STRING,
`order_no` STRING,
`total_amount` DECIMAL(10, 2),
`discount_amount` DECIMAL(10, 2),
`charge_against_amount` DECIMAL(10, 2),
`payable_amount` DECIMAL(10, 2),
`status` INT,
`pay_status` INT,
`pay_time` TIMESTAMP(3),
`paid_amount` DECIMAL(10, 2),
`effective_date` TIMESTAMP(3),
`terminal` INT,
`refund_status` INT,
`refund_amount` DECIMAL(10, 2),
`refund_time` TIMESTAMP(3),
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
proctime as proctime(),
PRIMARY KEY (id) NOT ENFORCED
) WITH (
'connector' = 'upsert-kafka',
'topic' = 'ods_oe_order',
'properties.bootstrap.servers' = '172.16.10.106:9092',
'key.format' = 'json',
'value.format' = 'json'
);
mysql_to_kafka_ods data insertion
- Create a mysql_to_kafka_ods job draft
BEGIN STATEMENT SET;
insert into dw.kafka_ods_oe_stu_course_order
select *
from mysql.mysql_oe_stu_course_order
-- /*+ OPTIONS('scan.startup.mode' ='latest-offset') */
;
insert into dw.kafka_ods_oe_stu_course
select *
from mysql.mysql_oe_stu_course
-- /*+ OPTIONS('scan.startup.mode' ='latest-offset') */
;
insert into dw.kafka_ods_oe_order
select *
from mysql.mysql_oe_order
-- /*+ OPTIONS('scan.startup.mode' ='latest-offset') */
;
END;
View Topic data
kafka-console-consumer.sh --bootstrap-server 172.16.10.106:9092 --topic ods_oe_order --from-beginning
Delete Topic
kafka-topics.sh --delete --topic ods_oe_order --bootstrap-server 172.16.10.106:9092
kafka-topics.sh --delete --topic ods_oe_stu_course --bootstrap-server 172.16.10.106:9092
kafka-topics.sh --delete --topic ods_oe_stu_course_order --bootstrap-server 172.16.10.106:9092
Verify the data volume of the kafka_ods layer:
select count(*) from dw.kafka_ods_oe_order;
The number of entries in the bxg.oe_order table of MySQL in RDS should be the same.
Solutions for insufficient Flink resources:
Solution: Adjust the number of Task Managers to 2 or more, or set the TaskManager's Cpu cores to be larger
The number of Task Managers was originally 1. Due to the large amount of calculation caused by the large amount of data, it needs to be increased. Here it is modified to 2 or more
Solutions for insufficient ECS memory:
Since free resources receive ecs server 1 core 2g memory, so when there are many running components, it is recommended to restart the ecs cloud server before starting the service
重启ECS：
1、关闭Flink作业运维中的作业
2、重启ECS，可以在控制台关闭，可以init 6
重启后的相关组件重启：
1、启动Zookpeer
cd /export/server/zookeeper/bin/
./zkServer.sh start
2、启动Kafka服务
cd /export/server/kafka/bin/
./kafka-server-start.sh -daemon ../config/server.properties
3、查看进程jps
4、重新开启Flink作业中作业运维中的作业
5、重新查询对应Kafka_ods层的数据
paimon_ods table
create database if not exists paimon.ods;
CREATE TABLE if not exists paimon.ods.paimon_ods_oe_stu_course_order (
`id` INT,
`student_course_id` INT,
`order_id` STRING,
`order_detail_id` STRING,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
`proctime` timestamp,
dt string comment '日期',
PRIMARY KEY (id,dt) NOT ENFORCED
) partitioned by (dt)
;
CREATE TABLE if not exists paimon.ods.paimon_ods_oe_stu_course (
`id` INT,
`student_id` STRING,
`course_id` INT,
`status` INT,
`contract_status` INT,
`learn_status` INT,
`service_days` INT,
`service_expires` TIMESTAMP(3),
`validity_days` INT,
`validity_expires` TIMESTAMP(3),
`terminate_cause` INT,
`effective_date` TIMESTAMP(3),
`finished_time` TIMESTAMP(3),
`total_progress` DECIMAL(10,2),
`purchase_time` INT,
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
`proctime` timestamp,
dt string comment '日期',
PRIMARY KEY (id,dt) NOT ENFORCED
) partitioned by (dt)
;
CREATE TABLE if not exists paimon.ods.paimon_ods_oe_order (
`id` STRING,
`channel` STRING,
`student_id` STRING,
`order_no` STRING,
`total_amount` DECIMAL(10,2),
`discount_amount` DECIMAL(10,2),
`charge_against_amount` DECIMAL(10,2),
`payable_amount` DECIMAL(10,2),
`status` INT,
`pay_status` INT,
`pay_time` TIMESTAMP(3),
`paid_amount` DECIMAL(10,2),
`effective_date` TIMESTAMP(3),
`terminal` INT,
`refund_status` INT,
`refund_amount` DECIMAL(10,2),
`refund_time` TIMESTAMP(3),
`create_time` TIMESTAMP(3),
`update_time` TIMESTAMP(3),
`delete_flag` BOOLEAN,
`proctime` timestamp,
dt string comment '日期',
PRIMARY KEY (id,dt) NOT ENFORCED
) partitioned by (dt)
;
kafka_ods_to_paimon_ods
BEGIN STATEMENT SET;
insert into paimon.ods.paimon_ods_oe_stu_course_order
select
*,
date_format(create_time,'yyyy-MM-dd') as dt
from dw.kafka_ods_oe_stu_course_order;
insert into paimon.ods.paimon_ods_oe_stu_course
select
*,
date_format(create_time,'yyyy-MM-dd') as dt
from dw.kafka_ods_oe_stu_course;
insert into paimon.ods.paimon_ods_oe_order
select
*,
date_format(create_time,'yyyy-MM-dd') as dt
from dw.kafka_ods_oe_order;
END;
appendix
component installation
Install jdk:
yum list java
yum install java-1.8.0-openjdk-devel.x86_64
cd /usr/lib/jvm/
environment variable configuration:
vim /etc/profile
export JAVA_HOME=/usr/lib/jvm/java
export JRE_HOME= J A V A H O M E / j r e e x p o r t C L A S S P A T H = . : JAVA_HOME/jre export CLASSPATH=.: JAVAHOME/jreexportCLASSPATH=.:JAVA_HOME/lib: J R E H O M E / l i b : JRE_HOME/lib: JREHOME/lib:CLASSPATH
export PATH= P A T H : PATH: PATH:JRE_HOME/bin:$JAVA_HOME/bin
Install zk:
Create a directory:
mkdir -p /export/software
mkdir -p /export/server
mkdir -p /export/data
cd /export/software
wget https://archive.apache.org/dist/zookeeper/zookeeper-3.5.10/apache-zookeeper-3.5.10-bin.tar.gz
tar -zxf apache-zookeeper-3.5.10-bin.tar.gz -C /export/server/
cd /export/server
ln -s apache-zookeeper-3.5.10-bin/ zookeeper
cd /export/data
mkdir zkdata
cd /export/server/zookeeper/conf
cp zoo_sample.cfg zoo.cfg
vim zoo.cfg
#Modify
dataDir=/export/data/zkdata
#At the end of the file add 2888 heartbeat port 3888 election port
server.1=172.26.254.71:2888: 3888
#Create myid file for intranet ip
mkdir -p /export/data/zkdata
echo 1 >/export/data/zkdata/myid
#Start, check whether the process is started
cd /export/server/zookeeper/bin/
./zkServer. sh start
#Kafka installation
Download the installation package
cd /export/software
wget https://downloads.apache.org/kafka/3.5.0/kafka_2.12-3.5.0.tgz
Unzip the installation package
cd /export/software/
tar -zxvf kafka_2 .12-3.5.0.tgz -C /export/server/
@Create soft link
cd /export/server
ln -s /export/server/kafka_2.12-3.5.0 kafka
Create /export/data/kafka-logs directory
cd /export/data
mkdir kafka-logs
modify configuration
cd /export/server/kafka/config
vim server.properties
#specify broker id
broker.id=0
#specify kafka's binding listening address
listeners=PLAINTEXT://172.18 .12.37:9092
#Specify the location of Kafka data
log.dirs=/export/data/kafka-logs
#Configure zk node
zookeeper.connect=172.18.12.37:2181
Note: The server intranet ip must be filled in the configuration file instead of the domain name, otherwise flink will not be able to connect to kafka. Environment variable
vim
/etc/profile
export KAFKA_HOME=/export/server/kafka
export PATH=: PATH: PATH:PATH: {KAFKA_HOME}/bin
refresh environment variable
source /etc/profile
start service
cd /export/server/kafka/bin/
./kafka-server-start.sh -daemon …/config/server.properties
view service jps
common command
# View all topics
kafka-topics.sh --list --bootstrap-server 172.26.254.71:9092
#Create topic
kafka-topics.sh --create --topic test --bootstrap-server 172.26.254.71:9092
#MySQL installation
yum install -y perl.x86_64
yum install -y libaio.x86_64
yum install -y net-tools.x86_64
Download and decompress
mkdir /export/server/mysql5.7
cd /export/software/
wget https://cdn. mysql.com/archives/mysql-5.7/mysql-5.7.29-1.el7.x86_64.rpm-bundle.tar
tar -xvf /export/software/mysql-5.7.29-1.el7.x86_64.rpm-bundle .tar -C /export/server/mysql5.7
cd /export/server/mysql5.7
rpm install mysql
rpm -ivh mysql-community-common-5.7.29-1.el7.x86_64.rpm mysql-community-libs- 5.7.29-1.el7.x86_64.rpm mysql-community-client-5.7.29-1.el7.x86_64.rpm mysql-community-server-5.7.29-1.el7.x86_64.rpm
Initialize services and modify files Permission
mysqld --initialize
chown mysql:mysql /var/lib/mysql -R
to start the service and set it to start automatically
systemctl start mysqld.service
systemctl enable mysqld.service
login Mysql change password and configure remote login
Pass=$(grep 'A temporary password' /var/log/mysqld.log |awk '{print KaTeX parse error: Expected 'EOF', got '}' at position 3: NF}̲') mysql -uroot… Pass"
mysgl>alter user user() identified by “123456”;
use mysql;
GRANT ALL PRIVILEGES ON . TO ‘root’@‘%’ IDENTIFIED BY ‘123456’ WITH GRANT OPTION;
FLUSH PRIVILEGES;
exit;