sqlserver debezium

EXEC sys.sp_cdc_enable_db
GO

SELECT FILEGROUP_NAME(1) AS [Filegroup Name];

Insert picture description here

EXEC sys.sp_cdc_enable_table
@source_schema = N'dbo',
@source_name   = N'stuInfo',       
@role_name     = N'MyRole',
@filegroup_name = N'PRIMARY',      
@supports_net_changes = 0
GO

EXEC sys.sp_cdc_help_change_data_capture

Insert picture description here

vim sqlserver.properties
#connector的名称，需要保证唯一
name=sqlserver_debezium_connector
tasks.max=1
connector.class=io.debezium.connector.sqlserver.SqlServerConnector
database.hostname=192.168.216.128
database.port=1433
database.user=sa
database.password=nio123456!A
database.dbname=dw
database.server.name=sqlserverdebezium
database.history.kafka.bootstrap.servers=bigdata1:9092,bigdata2:9092,bigdata4:9092
database.history.kafka.topic=sqlserverhistory

kafka-topics.sh --list --zookeeper bigdata1:2181
__consumer_offsets
sqlserverdebezium
sqlserverdebezium.dbo.stuInfo
sqlserverhistory

{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"schema"},{"type":"string","optional":false,"field":"table"},{"type":"string","optional":true,"field":"change_lsn"},{"type":"string","optional":true,"field":"commit_lsn"},{"type":"int64","optional":true,"field":"event_serial_no"}],"optional":false,"name":"io.debezium.connector.sqlserver.Source","field":"source"},{"type":"string","optional":true,"field":"databaseName"},{"type":"string","optional":true,"field":"schemaName"},{"type":"string","optional":true,"field":"ddl"},{"type":"array","items":{"type":"struct","fields":[{"type":"string","optional":false,"field":"type"},{"type":"string","optional":false,"field":"id"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"defaultCharsetName"},{"type":"array","items":{"type":"string","optional":false},"optional":true,"field":"primaryKeyColumnNames"},{"type":"array","items":{"type":"struct","fields":[{"type":"string","optional":false,"field":"name"},{"type":"int32","optional":false,"field":"jdbcType"},{"type":"int32","optional":true,"field":"nativeType"},{"type":"string","optional":false,"field":"typeName"},{"type":"string","optional":true,"field":"typeExpression"},{"type":"string","optional":true,"field":"charsetName"},{"type":"int32","optional":true,"field":"length"},{"type":"int32","optional":true,"field":"scale"},{"type":"int32","optional":false,"field":"position"},{"type":"boolean","optional":true,"field":"optional"},{"type":"boolean","optional":true,"field":"autoIncremented"},{"type":"boolean","optional":true,"field":"generated"}],"optional":false,"name":"io.debezium.connector.schema.Column"},"optional":false,"field":"columns"}],"optional":false,"name":"io.debezium.connector.schema.Table","field":"table"}],"optional":false,"name":"io.debezium.connector.schema.Change"},"optional":false,"field":"tableChanges"}],"optional":false,"name":"io.debezium.connector.sqlserver.SchemaChangeValue"},"payload":{"source":{"version":"1.4.2.Final","connector":"sqlserver","name":"sqlserverdebezium","ts_ms":1617150493285,"snapshot":"true","db":"dw","schema":"dbo","table":"stuInfo","change_lsn":null,"commit_lsn":null,"event_serial_no":null},"databaseName":"dw","schemaName":"dbo","ddl":null,"tableChanges":[{"type":"CREATE","id":"\"dw\".\"dbo\".\"stuInfo\"","table":{"defaultCharsetName":null,"primaryKeyColumnNames":[],"columns":[{"name":"id","jdbcType":4,"nativeType":null,"typeName":"int","typeExpression":"int","charsetName":null,"length":10,"scale":0,"position":1,"optional":true,"autoIncremented":false,"generated":false},{"name":"name","jdbcType":12,"nativeType":null,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":20,"scale":null,"position":2,"optional":true,"autoIncremented":false,"generated":false},{"name":"hobby","jdbcType":12,"nativeType":null,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":20,"scale":null,"position":3,"optional":true,"autoIncremented":false,"generated":false}]}}]}}

kafka-console-consumer.sh --bootstrap-server bigdata1:9092 --topic sqlserverdebezium.dbo.stuInfo --from-beginning
{
    
    "schema":{
    
    "type":"struct","fields":[{
    
    "type":"struct","fields":[{
    
    "type":"int32","optional":true,"field":"id"},{
    
    "type":"string","optional":true,"field":"name"},{
    
    "type":"string","optional":true,"field":"hobby"}],"optional":true,"name":"sqlserverdebezium.dbo.stuInfo.Value","field":"before"},{
    
    "type":"struct","fields":[{
    
    "type":"int32","optional":true,"field":"id"},{
    
    "type":"string","optional":true,"field":"name"},{
    
    "type":"string","optional":true,"field":"hobby"}],"optional":true,"name":"sqlserverdebezium.dbo.stuInfo.Value","field":"after"},{
    
    "type":"struct","fields":[{
    
    "type":"string","optional":false,"field":"version"},{
    
    "type":"string","optional":false,"field":"connector"},{
    
    "type":"string","optional":false,"field":"name"},{
    
    "type":"int64","optional":false,"field":"ts_ms"},{
    
    "type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{
    
    "allowed":"true,last,false"},"default":"false","field":"snapshot"},{
    
    "type":"string","optional":false,"field":"db"},{
    
    "type":"string","optional":false,"field":"schema"},{
    
    "type":"string","optional":false,"field":"table"},{
    
    "type":"string","optional":true,"field":"change_lsn"},{
    
    "type":"string","optional":true,"field":"commit_lsn"},{
    
    "type":"int64","optional":true,"field":"event_serial_no"}],"optional":false,"name":"io.debezium.connector.sqlserver.Source","field":"source"},{
    
    "type":"string","optional":false,"field":"op"},{
    
    "type":"int64","optional":true,"field":"ts_ms"},{
    
    "type":"struct","fields":[{
    
    "type":"string","optional":false,"field":"id"},{
    
    "type":"int64","optional":false,"field":"total_order"},{
    
    "type":"int64","optional":false,"field":"data_collection_order"}],"optional":true,"field":"transaction"}],"optional":false,"name":"sqlserverdebezium.dbo.stuInfo.Envelope"},"payload":{
    
    "before":null,"after":{
    
    "id":1,"name":"小乔","hobby":"美食"},"source":{
    
    "version":"1.4.2.Final","connector":"sqlserver","name":"sqlserverdebezium","ts_ms":1617150493390,"snapshot":"last","db":"dw","schema":"dbo","table":"stuInfo","change_lsn":null,"commit_lsn":null,"event_serial_no":null},"op":"r","ts_ms":1617150493392,"transaction":null}}

Insert picture description here

WARN No maximum LSN recorded in the database; please ensure that the SQL Server Agent is running (io.debezium.connector.sqlserver.SqlServerStreamingChangeEventSource:132)

WARN The configuration 'value.serializer' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration 'batch.size' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration 'max.block.ms' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration 'acks' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration 'buffer.memory' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)
[2021-03-31 08:53:22,718] WARN The configuration 'key.serializer' was supplied but isn't a known config. (org.apache.kafka.clients.admin.AdminClientConfig:369)

Guess you like