I. Introduction
to real-time synchronization of data, we must first be able to monitor real-time data changes in the database, you can use the canal, Maxwell and other tools. I chose canal, because it is more flexible, more qualified my project requirements.
Second, through the canal to monitor data changes in the database
Canal installation tutorial: https: //www.aliyun.com/jiaocheng/1117575.html
Third, the project overall architecture
project overall architecture, offline synchronization: https: //blog.csdn.net/beyond_qjm/article/details/83623738
四、主要代码
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.*;
import com.alibaba.otter.canal.protocol.Message;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.util.Bytes;
import qjm.data.synch.hbase.HbaseSerialization;
import qjm.data.synch.hbase.HbaseUtils;
import qjm.data.synch.modle.Employee;
import qjm.data.synch.service.SqlDataService;
java.io.IOException Import;
Import java.net.InetSocketAddress;
Import java.util.List;
/ **
* real-time synchronization of data
* /
public class OnlineSynch {
static Final the Log LogFactory.getLog the LOG = (OnlineSynch.class);
SqlDataService sqlDataService new new SqlDataService = ( "SqlMapConfig.xml");
hbaseUtils hbaseUtils = new new hbaseUtils ();
/ **
* synchronize data from a relational database to HBase
* /
public void synchToHbase () {
// create a link
CanalConnector connector = CanalConnectors.newSingleConnector (
the InetSocketAddress new new ( "192.168.135.132",
11111),
"Example",
"",
""
);
int batchSize = 1000;
Long batchId = null;
try {
connector.connect();
//指定监听数据库
connector.subscribe("grg_hr\\..*");
connector.rollback();
while (true) {
// 获取指定数量的数据
Message message = connector.getWithoutAck(batchSize);
batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
LOG.info("waitting...");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
} else {
LOG.info(String.format("\nmessage[batchId=%s,size=%s]", batchId, size));
handleEntry(message.getEntries());
}
connector.ack(batchId); // 提交确认
}
} catch (Exception e) {
// 处理失败, 回滚数据
if (batchId != null) connector.rollback(batchId);
LOG.error("Error: " + e.getMessage());
throw new RuntimeException(e);
} finally {
connector.disconnect();
}
}
/**
* 处理
* @param entries
*/
private void handleEntry(List<Entry> entries) {
//循环事件
for (Entry entry : entries) {
if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
continue;
}
RowChange rowChange = null;
try {
rowChange = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
}
//输出事件信息
CanalEntry.EventType eventType = rowChange.getEventType();
Header header = entry.getHeader();
LOG.info(String.format("\n================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
header.getLogfileName(), header.getLogfileOffset(),
header.getSchemaName(), header.getTableName(),
eventType));
//解析事件
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (eventType == EventType.DELETE) {
LOG.info("\n-------> delete");
deleteData(header.getTableName(), rowData.getBeforeColumnsList());
} else if (eventType == EventType.INSERT) {
LOG.info("\n-------> insert");
updateData(header.getTableName(), rowData.getAfterColumnsList());
} else if (eventType == EventType.UPDATE) {
//LOG.info("\n-------> before");
//printColumn(rowData.getBeforeColumnsList());
LOG.info("\n-------> after");
updateData(header.getTableName(),rowData.getAfterColumnsList());
}
}
}
}
/**
* 更新数据
*/
private void updateData(String tableName, List<Column> columns){
/**
* 1. 获取主键
* 2. 根据主键查询
* 3. 更新到hbase
*/
//获取主键
Long key = getKey(columns);
HbaseSerialization serialization = null;
//根据不同表做处理
if(tableName.equals("hr_employee")){
serialization = sqlDataService.getEmployeeById(key);
}
if (serialization != null){
try {
Employee employee = hbaseUtils.getData(new Get(Bytes.toBytes(key)), Employee.class);
LOG.info("before : \n"+employee);
hbaseUtils.putData(serialization);
employee = hbaseUtils.getData(new Get(Bytes.toBytes(key)), Employee.class);
LOG.info("before : \n"+employee);
} catch (Exception e) {
LOG.error(e.getMessage());
}
}
}
/**
* 删除数据
*/
private void deleteData(String tableName, List<Column> columns){
/ **
* Get the primary key 1.
* The primary data hbase 2. Delete key
* /
// Get the primary key
Long Key = getKey (Columns);
Class clazz = null;
// do the processing according to different epitopes
if (tableName.equals ( "hr_employee ")) {
clazz = Employee.class;
}
the try {
the Employee Employee = hbaseUtils.getData (the Get new new (Bytes.toBytes (Key)), Employee.class);
log.info (" before: \ n-"Employee +);
hbaseUtils.deleteData (clazz, the Delete new new (Bytes.toBytes (Key)));
Employee = hbaseUtils.getData (the Get new new (Bytes.toBytes (Key)), Employee.class);
log.info ( "After: \ n-" Employee +);
} the catch (Exception E) {
log.error (e.getMessage ());
}
}
/ **
* Get the primary key
* @return
* /
Private Long getKey (List <the Column> Columns) {
the try {
for (the Column column: Columns) {
IF (. Column.getName () the equals ( "ID")) {
return Long.valueOf (column.getValue ()) ;
}
}
} the catch (Exception E) {
e.printStackTrace ();
the throw a RuntimeException new new ( "! Not found Primary Key");
}
the throw a RuntimeException new new ( "! Not found Primary Key");
}
}
V. Code
- --------------------
author: beyond_qjm
source: CSDN
original: https: //blog.csdn.net/beyond_qjm/article/details/83624896
copyright: This article is bloggers original article, reproduced, please attach Bowen link!