flink实时数仓(四):mysql的安装以及canal测试

[root@note02 ~]# yum list installed | grep mysql
mysql-libs.x86_64      5.1.73-3.el6_5   @anaconda-CentOS-201410241409.x86_64/6.6

删除系统已经安装的mysql以及依赖

[root@note02 ~]# yum -y remove mysql-libs.x86_64

给CentOS添加rpm源,选择较新的源

wget http://repo.mysql.com/mysql-community-release-el6-5.noarch.rpm

[root@note02 ~]# ll
total 8
-rw-r--r--. 1 root root 5824 Nov 12  2015 mysql-community-release-el6-5.noarch.rpm

安装下载的rpm包

[root@note02 ~]# yum install mysql-community-release-el6-5.noarch.rpm  -y

安装成功后,在/etc/yum.repos.d/目录下增加了两个文件

查看mysql57安装源是否可用,不可用需要修改配置文件,使参数 enable=1

[root@note02 ~]# vim /etc/yum.repos.d/mysql-community.repo

# Note: MySQL 5.7 is currently in development. For use at your own risk.
# Please read with sub pages: https://dev.mysql.com/doc/relnotes/mysql/5.7/en/
[mysql57-community-dmr]
name=MySQL 5.7 Community Server Development Milestone Release
baseurl=http://repo.mysql.com/yum/mysql-5.7-community/el/6/$basearch/
enabled=1
gpgcheck=1
gpgkey=file:/etc/pki/rpm-gpg/RPM-GPG-KEY-mysql

查看是否可用

[root@note02 ~]# yum repolist enabled | grep mysql
mysql-connectors-community MySQL Connectors Community                       113
mysql-tools-community      MySQL Tools Community                             84
mysql56-community          MySQL 5.6 Community Server                       549
mysql57-community-dmr      MySQL 5.7 Community Server Development Milesto   360

使用yum 安装mysql,安装过程较慢

[root@note02 ~]# yum install mysql-community-server

这样安装会失败,修复待续

创建cannal账号

mysql> create user 'canal' identified by '000000';

mysql账号授权,授权给slave1节点通过cannal用户访问数据库

扫描二维码关注公众号，回复： 10357530 查看本文章

mysql> grant select, replication slave,replication client on *.* to 'canal' @'note02';

刷新缓存

mysql> flush privileges;

授权所有远程地址都可以通过canal用户访问数据库

mysql> grant select, replication slave,replication client on *.* to 'canal' @'%';

mysql> flush privileges;

查看user表的用户信息

mysql> select host,user,password from mysql.user;

开启mysql的binlog功能

[root@note01 canal]# find / -name my.cnf
/usr/my.cnf
[root@note01 canal]# vim /usr/my.cnf 

#开启binlog
log-bin=mysql-bin
#选择ROW模式
binlog-format=ROW
#配置Mysql replaction需要定义,不可与canal的slavedId重复
server_id=1

下载canal
https://github.com/alibaba/canal/releases/tag/canal-1.1.2
选择canal.deployer-1.1.2.tar.gz下载并解压

cd /opt/module/canal/conf/example

修改instance.properties文件

# position info
canal.instance.master.address=note01:3306
canal.instance.dbUsername=canal
canal.instance.dbPassword=000000
canal.instance.defaultDatabaseName =test
# mq config
canal.mq.topic=test

修改canal.properties

vim /opt/module/canal/conf/canal.properties

# tcp, kafka, RocketMQ
canal.serverMode = kafka
canal.zkServers =note01:2181,note02:2181,note03:2181
canal.mq.servers = note01:9092,note02:9092,note03:9092

启动zk,启动kafka,启动kafka消费者

kafka-console-consumer.sh --zookeeper note01:2181 --topic test

启动canal

sh /opt/module/canal/bin startup.sh

在myql中创建一张表

CREATE TABLE `user_test` (
  `id` int(11) NOT NULL,
  `name` varchar(255) DEFAULT NULL,
  `age` int(11) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

canal日志

2019-12-29 19:39:29.041 [destination = example , address = note01/192.168.18.100:3306 , EventParser] WARN  c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - ---> find start position successfully, EntryPosition[included=false,journalName=mysql-bin.000003,position=4,serverId=1,gtid=<null>,timestamp=1577619252000] cost : 1337ms , the next step is binlog dump

kafka中消息

[root@note03 ~]# kafka-console-consumer.sh --zookeeper note01:2181 --topic test
Using the ConsoleConsumer with old consumer is deprecated and will be removed in a future major release. Consider using the new consumer by passing [bootstrap-server] instead of [zookeeper].
{"data":null,"database":"test","es":1577619741000,"id":1,"isDdl":true,"mysqlType":null,"old":null,"sql":"CREATE TABLE `user_test` (\r\n`id`  int NOT NULL ,\r\n`name`  varchar(255) NULL ,\r\n`age`  int NULL ,\r\nPRIMARY KEY (`id`)\r\n)","sqlType":null,"table":"user_test","ts":1577619741222,"type":"CREATE"}

插入数据

{"data":[{"id":"1","name":"zs","age":"34"}],"database":"test","es":1577619951000,"id":2,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(255)","age":"int"},"old":null,"sql":"","sqlType":{"id":4,"name":12,"age":4},"table":"user_test","ts":1577619951535,"type":"INSERT"}

更新数据

{"data":[{"id":"1","name":"zs","age":"56"}],"database":"test","es":1577619972000,"id":3,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(255)","age":"int"},"old":[{"age":"34"}],"sql":"","sqlType":{"id":4,"name":12,"age":4},"table":"user_test","ts":1577619972984,"type":"UPDATE"}

删除数据

{"data":[{"id":"1","name":"zs","age":"56"}],"database":"test","es":1577620001000,"id":4,"isDdl":false,"mysqlType":{"id":"int","name":"varchar(255)","age":"int"},"old":null,"sql":"","sqlType":{"id":4,"name":12,"age":4},"table":"user_test","ts":1577620002074,"type":"DELETE"}