Getting started with canal

　　Data extraction simply means extracting data from one table into another table. There are many ETL tools that can help us extract and convert data. ETL tools can extract data in one-time or scheduled operations. However, as an open source data extraction project provided by Alibaba, canal can extract data in real time. The principle is camouflage. Become a mysql slave node, read the mysql binlog, generate messages, and the client subscribes to these data change messages, processes and stores them. Let's build a canal service together

configure mysql

　　mysql does not open binlog by default, modify the my.cnf file of mysql, add the following configuration, note that binlog-format must be row , because if binlog is

STATEMENT or MIXED, then the sql statement is recorded in the binlog, not the specific data row, and canal cannot parse the specific data change.

log-bin=E:/mysql5.5/bin_log/mysql-bin.log
binlog-format=ROW
server-id=123

canal configuration

　　Download and install canal, you can see the canal.properties file in the conf directory, modify the file to

#Set the address and port of the mysql server to listen on
canal.instance.master.address = 127.0.0.1:3306 #Set 
a username and password that can access mysql and have corresponding permissions. In this example, the username and password are canal
canal.instance.dbUsername = canal
canal.instance.dbPassword = canal #connected 
database
canal.instance.defaultDatabaseName =test 
# 订阅实例中所有的数据库和表
canal.instance.filter.regex = .*\\..* #connect 
to canal's port 
canal.port= 11111 #Listen 
to the queue of data changes sent 
canal.destinations= example

　　Start bin/startup.sh or bin/startup.bat

 

client code

package com.cw.demo.channel;

import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.common.utils.AddressUtils;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import com.google.protobuf.InvalidProtocolBufferException;

import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Created by chenwei01 on 2017/4/9.
 */ 
public  class ClientChannel {

    public static void main(String[] args) {
        while (true) {
            //连接canal
            CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(AddressUtils.getHostIp(), 11111), "example", "canal", "canal");
            connector.connect();
            // Subscribe to the monitored database. Table 
            connector.subscribe("demo_db.user_tab" );
             // Fetch 5 
            Messages at a time msg = connector.getWithoutAck(5 );

            long batchId = msg.getId();
            int size = msg.getEntries().size();
            if (batchId < 0 || size == 0) {
                System.out.println( "No message, sleep for 5 seconds" );
                 try {
                    Thread.sleep(5000);
                } catch (InterruptedException e) {
                    e.printStackTrace ();
                }
            } else {
                //
                CanalEntry.RowChange row = null;
                for (CanalEntry.Entry entry : msg.getEntries()) {
                    try {
                        row = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
                        List<CanalEntry.RowData> rowDatasList = row.getRowDatasList();
                        for (CanalEntry.RowData rowdata : rowDatasList) {
                            List<CanalEntry.Column> afterColumnsList = rowdata.getAfterColumnsList();
                            Map<String, Object> dataMap = transforListToMap(afterColumnsList);
                            if (row.getEventType() == CanalEntry.EventType.INSERT) {
                                //具体业务操作
                                System.out.println(dataMap);
                            } else  if (row.getEventType() == CanalEntry.EventType.UPDATE) {
                                 // Specific business operation 
                                System.out.println(dataMap);
                            } else if (row.getEventType() == CanalEntry.EventType.DELETE) {
                                List<CanalEntry.Column> beforeColumnsList = rowdata.getBeforeColumnsList();
                                for (CanalEntry.Column column : beforeColumnsList) {
                                    if ("id".equals(column.getName())) {
                                        //具体业务操作
                                        System.out.println("删除的id：" + column.getValue());
                                    }
                                }
                            } else {
                                System.out.println( "Other operation types are not processed" );
                            }

                        }

                    } catch (InvalidProtocolBufferException e) {
                        e.printStackTrace ();
                    }
                }
                // Confirm message 
                connector.ack(batchId);
            }


        }
    }

    public static Map<String, Object> transforListToMap(List<CanalEntry.Column> afterColumnsList) {
        Map map = new HashMap();
        if (afterColumnsList != null && afterColumnsList.size() > 0) {
            for (CanalEntry.Column column : afterColumnsList) {
                map.put(column.getName(), column.getValue());
            }
        }
        return map;
    }


}