canal读取mysql增量日志打引sql语句

1. canal 依赖

  <dependency>
            <groupId>com.alibaba.otter</groupId>
            <artifactId>canal.client</artifactId>
            <version>1.1.4</version>
        </dependency>

2. canal 实战

package org.test.canal;

import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import com.google.protobuf.InvalidProtocolBufferException;

import java.net.InetSocketAddress;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;

/**
 * @author fl
 * @date: 2021/5/24 13:51
 * @description canal读取mysql增量日志打引sql语句
 */
public class CanalClient {
    
    
    /**
     * 删除的sql语句集合
     */
    private static Queue<String> deleteList = new ConcurrentLinkedQueue<>();
    /**
     * 更新的sql语句集合
     */
    private static Queue<String> updateList = new ConcurrentLinkedQueue<>();

    /**
     * 插入的sql语句集合
     */
    private static Queue<String> insertList = new ConcurrentLinkedQueue<>();

    public static void main(String[] args) {
    
    
        CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("10.26.118.120",
                11111), "maxwell", "", "");
        int batchSize = 1000;
        try {
    
    
            connector.connect();
            connector.subscribe(".*\\..*");
            connector.rollback();
            try {
    
    
                while (true) {
    
    
                    //尝试从master那边拉去数据batchSize条记录，有多少取多少
                    Message message = connector.getWithoutAck(batchSize);
                    long batchId = message.getId();
                    int size = message.getEntries().size();
                    if (batchId == -1 || size == 0) {
    
    
                        Thread.sleep(1000);
                    } else {
    
    
                        dataHandle(message.getEntries());
                    }
                    connector.ack(batchId);
                }
            } catch (InterruptedException e) {
    
    
                e.printStackTrace();
            } catch (InvalidProtocolBufferException e) {
    
    
                e.printStackTrace();
            }
        } finally {
    
    
            connector.disconnect();
        }
    }

    /**
     * 数据处理
     *
     * @param entrys
     */
    private static void dataHandle(List<CanalEntry.Entry> entrys) throws InvalidProtocolBufferException {
    
    
        for (CanalEntry.Entry entry : entrys) {
    
    
            if (CanalEntry.EntryType.ROWDATA == entry.getEntryType()) {
    
    
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
                CanalEntry.EventType eventType = rowChange.getEventType();
                if (eventType == CanalEntry.EventType.DELETE) {
    
    
                    saveDeleteSql(entry);
                } else if (eventType == CanalEntry.EventType.UPDATE) {
    
    
                    saveUpdateSql(entry);
                } else if (eventType == CanalEntry.EventType.INSERT) {
    
    
                    saveInsertSql(entry);
                }
            }
        }
    }


    /**
     * 保存插入语句
     *
     * @param entry
     */
    private static void saveInsertSql(CanalEntry.Entry entry) {
    
    
        try {
    
    
            CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
            List<CanalEntry.RowData> rowDatasList = rowChange.getRowDatasList();
            for (CanalEntry.RowData rowData : rowDatasList) {
    
    
                List<CanalEntry.Column> columnList = rowData.getAfterColumnsList();
                StringBuffer sql = new StringBuffer("insert into " + entry.getHeader().getTableName() + " (");
                for (int i = 0; i < columnList.size(); i++) {
    
    
                    sql.append(columnList.get(i).getName());
                    if (i != columnList.size() - 1) {
    
    
                        sql.append(",");
                    }
                }
                sql.append(") VALUES (");
                for (int i = 0; i < columnList.size(); i++) {
    
    
                    sql.append("'" + columnList.get(i).getValue() + "'");
                    if (i != columnList.size() - 1) {
    
    
                        sql.append(",");
                    }
                }
                sql.append(")");
                insertList.add(sql.toString());
                System.out.println(insertList);
            }
        } catch (Exception e) {
    
    
            e.printStackTrace();
        }
    }

    /**
     * 保存删除语句
     *
     * @param entry
     */
    private static void saveDeleteSql(CanalEntry.Entry entry) {
    
    
        try {
    
    
            CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
            List<CanalEntry.RowData> rowDatasList = rowChange.getRowDatasList();
            for (CanalEntry.RowData rowData : rowDatasList) {
    
    
                List<CanalEntry.Column> columnList = rowData.getBeforeColumnsList();
                StringBuffer sql = new StringBuffer("delete from " + entry.getHeader().getTableName() + " where ");
                for (CanalEntry.Column column : columnList) {
    
    
                    if (column.getIsKey()) {
    
    
                        //暂时只支持单一主键
                        sql.append(column.getName() + "=" + column.getValue());
                        break;
                    }
                }
                deleteList.add(sql.toString());
                System.out.println(deleteList);
            }
        } catch (Exception e) {
    
    
            e.printStackTrace();
        }
    }

    /**
     * 保存更新语句
     *
     * @param entry
     */
    private static void saveUpdateSql(CanalEntry.Entry entry) {
    
    
        try {
    
    
            CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
            List<CanalEntry.RowData> rowDatasList = rowChange.getRowDatasList();
            for (CanalEntry.RowData rowData : rowDatasList) {
    
    
                List<CanalEntry.Column> newColumnList = rowData.getAfterColumnsList();
                StringBuffer sql = new StringBuffer("update " + entry.getHeader().getTableName() + " set ");
                for (int i = 0; i < newColumnList.size(); i++) {
    
    
                    sql.append(" " + newColumnList.get(i).getName()
                            + " = '" + newColumnList.get(i).getValue() + "'");
                    if (i != newColumnList.size() - 1) {
    
    
                        sql.append(",");
                    }
                }
                sql.append(" where ");
                List<CanalEntry.Column> oldColumnList = rowData.getBeforeColumnsList();
                for (CanalEntry.Column column : oldColumnList) {
    
    
                    if (column.getIsKey()) {
    
    
                        //暂时只支持单一主键
                        sql.append(column.getName() + "=" + column.getValue());
                        break;
                    }
                }
                updateList.add(sql.toString());
                System.out.println(updateList);
            }
        } catch (InvalidProtocolBufferException e) {
    
    
            e.printStackTrace();
        }
    }
}