大数据：canal：flatmessage解析
其他
2021-03-06 21:22:24
阅读次数: 0
主类
main方法入口
-
public static void main(String[] args) {
canalConnector = CanalConnectors.newSingleConnector(
new InetSocketAddress(SERVER_ADDRESS, PORT),
DESTINATION, USERNAME, PASSWORD);
try {
//建立连接
canalConnector.connect();
// 订阅
canalConnector.subscribe(".*\\..*");
// while(true) {
int batchSize = 1000;
// 获取指定数量的数据
Message message = canalConnector.getWithoutAck(batchSize, 1000L, TimeUnit.MILLISECONDS);
Long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
System.out.println("message信息有误");
} else {
//存在数据，解析处理
List<FlatMessage> flatMessages = parseFlatMessage(message);
Iterator flatMessages_iterator = flatMessages.iterator();
while (flatMessages_iterator.hasNext()) {
System.out.println(flatMessages_iterator.next() + ",");
}
}
canalConnector.ack(batchId); // 提交确认 处理数据
// connector.rollback(batchId); // 处理失败, 回滚数据
// }
} finally {
canalConnector.disconnect();
}
}
解析方法
-
public static List<FlatMessage> parseFlatMessage(Message message) {
try {
if (message == null) {
return null;
} else {
List<FlatMessage> flatMessages = new ArrayList();
List<CanalEntry.Entry> entries = message.getEntries();
//遍历entry entry--------->flatmessage
for (CanalEntry.Entry entry : entries) {
CanalEntry.Header header = entry.getHeader();
CanalEntry.EntryType entryType = entry.getEntryType();
//如果是事务则先忽略
if (entryType == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entryType == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
CanalEntry.EventType eventType = header.getEventType();
ByteString storeValue = entry.getStoreValue();
CanalEntry.RowChange rowChange = null;
try {
rowChange = CanalEntry.RowChange.parseFrom(storeValue);
} catch (InvalidProtocolBufferException e) {
e.printStackTrace();
}
//基本类型
FlatMessage flatMessage = new FlatMessage(message.getId());
flatMessage.setEs(header.getExecuteTime());
flatMessage.setTable(header.getTableName());
flatMessage.setDatabase(header.getSchemaName());
flatMessage.setType(eventType.toString());
flatMessage.setIsDdl(rowChange.getIsDdl());
flatMessage.setSql(rowChange.getSql());
flatMessage.setTs(System.currentTimeMillis());
if (!rowChange.getIsDdl()) {
//定义复杂类型
Map<String, Integer> sqlType = new LinkedHashMap();
Map<String, String> mysqlType = new LinkedHashMap();
List<Map<String, String>> data = new ArrayList();
List<Map<String, String>> old = new ArrayList();
Set<String> updateSet = new HashSet();
List<CanalEntry.RowData> rowDatas = rowChange.getRowDatasList();
//遍历rowDatas
for (CanalEntry.RowData rowData : rowDatas) {
//只分析insert update，delete
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
&& eventType != CanalEntry.EventType.DELETE) {
continue;
}
Map<String, String> row = new LinkedHashMap();
List<CanalEntry.Column> columns;
if (eventType == CanalEntry.EventType.DELETE) {
columns = rowData.getBeforeColumnsList();
} else {
columns = rowData.getAfterColumnsList();
}
for (CanalEntry.Column column : columns) {
sqlType.put(column.getName(), column.getSqlType());
mysqlType.put(column.getName(), column.getMysqlType());
if (column.getIsNull()) {
row.put(column.getName(), null);
} else {
row.put(column.getName(), column.getValue());
}
// 获取update为true的字段
if (column.getUpdated()) {
updateSet.add(column.getName());
}
}
if (!row.isEmpty()) {
data.add(row);
}
//update再添加old
Map<String, String> rowOld = new LinkedHashMap<>();
if (eventType == CanalEntry.EventType.UPDATE) {
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
if (column.getIsNull()) {
rowOld.put(column.getName(), null);
} else {
rowOld.put(column.getName(), column.getValue());
}
}
}
}
// update操作再添加before的值
if (!rowOld.isEmpty()) {
old.add(rowOld);
}
}
if (!sqlType.isEmpty()) {flatMessage.setSqlType(sqlType);}
if (!mysqlType.isEmpty()) { flatMessage.setMysqlType(mysqlType);}
if (!data.isEmpty()) {flatMessage.setData(data);}
if (!old.isEmpty()) {flatMessage.setOld(old);}
}
flatMessages.add(flatMessage);
}
return flatMessages;
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
转载自blog.csdn.net/JinChao94/article/details/109486821