一、canal的message数据格式
1、数据的类型包
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
import com.alibaba.otter.canal.protocol.Message;
2、数据全部展示
可参考:https://blog.csdn.net/weixin_41047933/article/details/85293002
canal采用protobuff:
Entry=====> RowChange=====> Column
Entry
Header
logfileName [binlog文件名]
logfileOffset [binlog position]
executeTime [发生的变更]
schemaName
tableName
eventType [insert/update/delete类型]
entryType [事务头BEGIN/事务尾END/数据ROWDATA]
storeValue [byte数据,可展开,对应的类型为RowChange]
RowChange
isDdl [是否是ddl变更操作,比如create table/drop table]
sql [具体的ddl sql]
rowDatas [具体insert/update/delete的变更数据,可为多条,1个binlog event事件可对应多条变更,比如批处理]
beforeColumns [Column类型的数组]
afterColumns [Column类型的数组]
Column
index
sqlType [jdbc type]
name [column name]
isKey [是否为主键]
updated [是否发生过变更]
isNull [值是否为null]
value [具体的内容,注意为文本]
3、数据的代码逻辑(java)
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.protocol.Message;
import com.tank.RedisObserver;
import java.net.InetSocketAddress;
/**
* @ClassName: RedisClient
* @Description: TODO
* @Author: ****
* @Data: 2019/5/20 18:14
* @Version: 1.0
**/
public class RedisClient {
public static void main(String args[]) {
final String ip = "hadoop";
final int port = 11111;
final String destination = "demo2";
final String username = "canal";
final String password = "Canal2019!";
InetSocketAddress socketAddress = new InetSocketAddress(ip, port);
CanalConnector connector = CanalConnectors.newSingleConnector(socketAddress, destination, username, password);
int batchSize = 100;
try {
connector.connect();
connector.subscribe(destination + "\\..*");
connector.rollback();
while (true) {
//获取canal的信息,也即日志文件的数据
Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
//System.out.println("未获取到数据!");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
} else {
System.out.println("处理数据,输出数据!");
//RedisObserver.printEntry(message.getEntries());
//RedisObserver.printColumn(message.getEntries());
//在这个地方写处理逻辑。。。。。处理message。。。。。。。。
}
connector.ack(batchId); // 提交确认
// connector.rollback(batchId); // 处理失败, 回滚数据
}
} finally {
connector.disconnect();
}
}
}
public static void showAllMessage(Message message){
List<Entry> entries = message.getEntries();
for(Entry entry:entries){
Header header = entry.getHeader();
header.getLogfileName();
header.getLogfileOffset();
header.getExecuteTime();
header.getSchemaName();
header.getEventType();
EntryType entryType = entry.getEntryType();
ByteString storeValue = entry.getStoreValue();
try {
RowChange rowChage = RowChange.parseFrom(entry.getStoreValue());
List<RowData> rowDatas = rowChage.getRowDatasList();
for(RowData rowData:rowDatas){
List<Column> afterColumns = rowData.getAfterColumnsList();//用于非delete操作
List<Column> beforeColumns = rowData.getBeforeColumnsList();//用于delete操作
for(Column afterColumn:afterColumns){
afterColumn.getIndex();
afterColumn.getMysqlType();
afterColumn.getName();
afterColumn.getIsKey();
afterColumn.getUpdated();
afterColumn.getIsNull();
afterColumn.getValue();
}
}
} catch (InvalidProtocolBufferException e) {
e.printStackTrace();
}
}
}