- 安装JDK
-# tar zxvf jdk-8u281-linux-x64.tar.gz -C /opt/
-# vi /etc/profile
-#### zookeeper ####
export JAVA_HOME=/opt/jdk1.8.0_281
export CLASSPATH=$JAVA_HOME/jre/lib:$JAVA_HOME/lib
export PATH=$JAVA_HOME/bin:$PATH
-# source /etc/profile
-# java -version
java version "1.8.0_281"
Java(TM) SE Runtime Environment (build 1.8.0_281-b09)
Java HotSpot(TM) 64-Bit Server VM (build 25.281-b09, mixed mode)
-#
- 安装Kafka
-# tar zxvf kafka_2.13-2.7.0.tgz -C /opt/
-# cd /opt/kafka_2.13-2.7.0/config
-# vi /etc/profile
-#### zookeeper ####
export JAVA_HOME=/opt/jdk1.8.0_281
export KAFKA_HOME=/opt/kafka_2.13-2.7.0
export CLASSPATH=$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$KAFKA_HOME/lib
export PATH=$KAFKA_HOME/bin:$JAVA_HOME/bin:$PATH
-#### 启动zookeeper #####
-# cd /opt/kafka_2.13-2.7.0/
-# nohup bin/zookeeper-server-start.sh config/zookeeper.properties &
-#### 启动kafka #####
-# nohup bin/kafka-server-start.sh config/server.properties &
-### 创建topic ###
-# kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
Created topic test.
-#
-### 显示topic ###
-# kafka-topics.sh -list -zookeeper localhost:2181
test
-#
创建kafka生产者：
-# kafka-console-producer.sh --broker-list localhost:9092 --topic test
hello
test_msg
test_topic
创建kafka消费者：
-# kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
hello
test_msg
test_topic
解释：
--broker-list:默认端口为9092.可自行更改
- 数据库准备
postgres=# create user sync_user with replication password 'sync_user';
postgres=# alter user sync_user with superuser ;
postgres=# create database tdb01;
CREATE DATABASE
postgres=# CREATE TABLE customers (
id SERIAL,
first_name VARCHAR(255) NOT NULL,
last_name VARCHAR(255) NOT NULL,
email VARCHAR(255) NOT NULL,
PRIMARY KEY(id)
);
pg_hba.conf
-# TYPE DATABASE USER ADDRESS METHOD
-# "local" is for Unix domain socket connections only
local all all trust
-# IPv4 local connections:
host all all 127.0.0.1/32 trust
host all all 0.0.0.0/0 md5
-# IPv6 local connections:
host all all ::1/128 trust
-# Allow replication connections from localhost, by a user with the
-# replication privilege.
local replication all trust
host replication all 127.0.0.1/32 trust
host replication all 192.168.1.0/24 md5
$
-### 配置文件修改 ####
$ vi postgresql.conf
wal_level = logical
#max_wal_senders = 10
#max_replication_slots = 10
-#### reboot database ####
$ pg_ctl stop –D /pgdata
$ pg_ctl start –D /pgdata
- 配置安装connect
-# tar zxvf debezium-connector-postgres-1.4.1.Final-plugin.tar.gz -C /opt/
-# cd /opt/debezium-connector-postgres/
-# cp *.jar /opt/kafka_2.13-2.7.0/libs/
-# cd /opt/kafka_2.13-2.7.0/config
-# vi kafka-postgres.properties
name=postgres-connector
connector.class=io.debezium.connector.postgresql.PostgresConnector
database.hostname=192.168.1.141
database.port=5432
database.user=sync_user
database.password=sync_user
database.dbname=tdb01
database.server.name=tdb01
plugin.name=pgoutput
table.whitelist=
errors.log.enable=true
errors.logs.include.messages=true
-#
- 数据库插入数据测试
-#### 数据库端 ####
postgres=# \c tdb01
tdb01=# insert into customers(first_name,last_name,email) values ('he','mia','[email protected]');
INSERT 0 1
tdb01=#
-#### kafka端 #####
-# kafka-topics.sh -list -zookeeper localhost:2181
tdb01.public.customers
-# kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic tdb01.public.customers --from-beginning
{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"first_name"},{"type":"string","optional":false,"field":"last_name"},{"type":"string","optional":false,"field":"email"}],"optional":true,"name":"tdb01.public.customers.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"first_name"},{"type":"string","optional":false,"field":"last_name"},{"type":"string","optional":false,"field":"email"}],"optional":true,"name":"tdb01.public.customers.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"int64","optional":true,"field":"xmin"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"},{"type":"int64","optional":false,"field":"total_order"},{"type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"tdb01.public.customers.Envelope"},"payload":{"before":null,"after":{"id":1,"first_name":"he","last_name":"mia","email":"[email protected]"},"source":{"version":"1.4.1.Final","connector":"postgresql","name":"tdb01","ts_ms":1612544437937,"snapshot":"false","db":"tdb01","schema":"public","table":"customers","txId":493,"lsn":23092184,"xmin":null},"op":"c","ts_ms":1612515639176,"transaction":null}}