一、背景：
今天下午在做模拟数据时发现200万条数据写入数据库，时间很长，思考通过使用多线程进行数据批量写入操作。
二、实现代码：
1、数据库工具类
package com.general.data.utils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
/**
* @description 数据库工具类
* @author 砌墙民工
* @date 2020年1月18日 上午12:56:40
* @version 1.0
*
* 修改信息说明：
* @updateDescription
* @updateAuthor
* @updateDate
*/
public class DataBaseUtils {
private static final String url = "jdbc:oracle:thin:@127.0.0.1:1521:orcl";
private static final String driver = "oracle.jdbc.driver.OracleDriver";
private static final String userName = "simulation";
private static final String password = "simulation";
/**
* 获取数据库连接
* @return 数据库连接
*/
public static Connection getConnection() {
try {
Class.forName(driver);
return DriverManager.getConnection(url, userName, password);
} catch (SQLException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
return null;
}
/**
* 关闭数据库操作资源
* @param conn 数据库连接
* @param stmt 数据库游标
* @param rs 结果集
*/
public static void closeDataBaseResources(Connection conn, Statement stmt, ResultSet rs) {
if (null != rs) {
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
rs = null;
}
if (null != stmt) {
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
stmt = null;
}
if (null != conn) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
conn = null;
}
}
/**
* 读取表信息
* @param tableName
* @return
*/
public static List<Map<String, String>> readTable(String tableName) {
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
List<Map<String, String>> row = null;
try {
conn = getConnection();
//ResultSet.TYPE_SCROLL_INSENSITIVE 结果集的游标可以上下移动，当数据库变化时，当前结果集不变。
//ResultSet.CONCUR_READ_ONLY 不能用结果集更新数据库中的表。
stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE);
String sql = "select * from " + tableName;
rs = stmt.executeQuery(sql);
ResultSetMetaData rsmd = rs.getMetaData();
int columnCount = rsmd.getColumnCount(); //ResultSet的总列数
row = new ArrayList<Map<String, String>>();
while (rs.next()) {
Map<String, String> column = new HashMap<String, String>();
for (int columnIndex = 1; columnIndex <= columnCount; columnIndex++) {
column.put(rsmd.getColumnName(columnIndex).toLowerCase(), rs.getString(columnIndex));
}
row.add(column);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
closeDataBaseResources(conn, stmt, rs);
}
return row;
}
/**
* 多线程将集合map数据写入表
* @param data 集合map数据
* @param tableName 表名
*/
public static void multiThreadWriteTable(List<Map<String, String>> data, String tableName) {
long startTime = System.currentTimeMillis();
System.out.println("数据总量：" + data.size());
List<String> sqlList = new ArrayList<String>();
for (int i = 0; i < data.size(); i++) {
Map<String, String> entity = data.get(i);
StringBuffer frontSql = new StringBuffer();
StringBuffer values = new StringBuffer();
frontSql.append("insert into ");
frontSql.append(tableName);
frontSql.append(" (");
for (String key : entity.keySet()) {
frontSql.append(key + ",");
values.append("'" + entity.get(key) + "',");
}
String sql = frontSql.substring(0, frontSql.length() - 1) + ") values(" + values.substring(0, values.length() - 1) + ")";
sqlList.add(sql);
}
int threadCount = 20;
int every = data.size() / threadCount;
final CountDownLatch latch = new CountDownLatch(threadCount);
for (int i = 0; i < threadCount; i++) {
if (i == threadCount - 1) {
new Thread(new ThreadWorker(latch, i * every, (i + 1) * every + (data.size() % threadCount), sqlList)).start();
} else {
new Thread(new ThreadWorker(latch, i * every, (i + 1) * every, sqlList)).start();
}
}
try {
latch.await();
long endTimes = System.currentTimeMillis();
System.out.println("所有线程执行完毕:" + (endTimes - startTime));
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
2、线程工作空间
package com.general.data.utils;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.concurrent.CountDownLatch;
/**
* @description 线程工作空间
* @author 砌墙民工
* @date 2020年1月18日 下午3:11:55
* @version 1.0
*
* 修改信息说明：
* @updateDescription
* @updateAuthor
* @updateDate
*/
public class ThreadWorker implements Runnable {
private int start = 0;
private int end = 0;
private CountDownLatch latch;
private List<String> sqlList;
public ThreadWorker(CountDownLatch latch, int start, int end, List<String> sqlList) {
this.start = start;
this.end = end;
this.latch = latch;
this.sqlList = sqlList;
}
public void run() {
System.out.println("线程" + Thread.currentThread().getName() + "正在执行。。线程数据量为:" + (end - start));
long startTime = System.currentTimeMillis();
Connection conn = null;
Statement stmt = null;
try {
conn = DataBaseUtils.getConnection();
boolean autoCommit = conn.getAutoCommit();
conn.setAutoCommit(false);
stmt = conn.createStatement();
int rowNum = 0;
for (int i = start; i < end; i++) {
stmt.addBatch(sqlList.get(i));
if (++rowNum % 10000 == 0) {
stmt.executeBatch();
conn.commit();
System.out.println("线程" + Thread.currentThread().getName() + "现已完成" + rowNum + "条记录插入");
}
}
stmt.executeBatch();
// 提交修改
conn.commit();
conn.setAutoCommit(autoCommit);
} catch (SQLException e) {
e.printStackTrace();
} finally {
DataBaseUtils.closeDataBaseResources(conn, stmt, null);
long endTime = System.currentTimeMillis();
System.out.println("线程" + Thread.currentThread().getName() + "数据写入时间：" + (endTime - startTime) + "ms");
}
latch.countDown();
}
}
三、呈现结果
经过实际测试本地虚拟机搭建数据库服务器，230万条数据写入，5分钟左右完成。由于笔记本CPU，好像超过20个线程性能没有太大提升。比单线程速度要快很多！
