Java implementation bulk import Excel data database
Overview:
This gadget class works in an episode Oh, because when you need to mention the number of cross-database derivative ... some need to import from oracle mysql, oracle import some from mysql
After practice we will find navicat use the tool to connect mysql database, export data from oracle database as csv format, then import mysql is very fast, probably the problem every minute of it
But when we use plsql oracle database connection, and then export the data in the form of a sql file mysql database you want to insert the oracle, very slow ..... I usually 50000 6 data fields in the open command window need to be performed almost two hours - this is really terrible tool and sometimes there are problems, was forced to interrupt .... to start again ... too low efficiency
So with the following derivative tools.
New maven project, the project name can be customized Oh
ExecuteDataExcelToOracle.java class, when used to directly execute the main method
You need to replace your Excel file
You need to replace your database connection information
Need to replace your table information, and fields (fields here and also inside the Data relevant oh, if you want to import the field is more than 10 ha ~ Please continue to add your own)
Mainly we do the following things:
Resolving Excel file, Excel data file one by one into the object
2. connects to the database data into the database
With this tool class import data, 50,000 data about it as long as two minutes, and most of the time is spent on parsing excel ~, efficiency is simply not an ordinary high ha ha ha ha ~
package com.imodule.dataImport.dataImport.main;
import java.io.File;
import java.io.FileInputStream;
import java.io.Serializable;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Date;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
{class ExecuteDataExcelToOracle public
public static void main (String [] args) {
the try {
List <the Data> = loadExcel DATAS ( "C: /Users/xxxx/Desktop/aa.xlsx"); // needs to be replaced
batchInsert (datas);
the catch} (Exception E) {
e.printStackTrace ();
}
}
/ **
* excel in data acquisition
* @param filepath
* @return
* @throws Exception
* /
public static List <the data> loadExcel (String filepath) throws Exception {
File File = new new File (filepath);
Workbook wb = WorkbookFactory.create(new FileInputStream(file));
Sheet Sheet wb.getSheetAt = (0);
List <the data> = new new DATAS the ArrayList <> (); // add data to the data line by line is added to the collection, as an insert data into the reference
Row row = null;
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
System.out.println("i="+i);
row = sheet.getRow(i);
if(row == null || StringUtils.isEmpty(row.getCell(0).getStringCellValue())){
continue;
}
Data data = new ExecuteDataExcelToOracle().new Data();
data.setStr1(row.getCell(0) == null? "":row.getCell(0).getStringCellValue());
data.setStr2(row.getCell(1) == null? "":row.getCell(1).getStringCellValue());
data.setStr3(row.getCell(2) == null? "":row.getCell(2).getStringCellValue());
data.setStr4(row.getCell(3) == null? "":row.getCell(3).getStringCellValue());
data.setStr5(row.getCell(4) == null? "":row.getCell(4).getStringCellValue());
data.setStr6(row.getCell(5) == null? "":row.getCell(5).getStringCellValue());
// row.getCell(5).setCellType(CellType.STRING);
if(row.getCell(6) == null){
data.setStr7("");
}else{
Date d = (Date) row.getCell(6).getDateCellValue();
DateFormat formater = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String date = formater.format(d);
data.setStr7(date);
}
data.setStr8(row.getCell(7) == null? "":row.getCell(7).getStringCellValue());
data.setStr9:; (row.getCell (8) == null "" row.getCell (8) .getStringCellValue ()?)
"data.setStr10 (row.getCell (9) == null?": row.getCell ( . 9) .getStringCellValue ());
datas.add (data);
}
return DATAS;
}
/ **
* batch execution insert data
* @param DATAS
* /
public static void batchInsert (List <the data> DATAS) {
Long the startTime = the System .currentTimeMillis ();
Connection Conn = null;
the try {
the Class.forName ( "oracle.jdbc.driver.OracleDriver"); // if you need to replace the oracle, if it is necessary to replace mysql
conn = DriverManager.getConnection ( " jdbc: oracle: thin: @ 10.11.22.33 : 1521: demo "," root "," 111 ");// needs to be replaced
conn.setAutoCommit(false);
PreparedStatement stmt = conn.prepareStatement("INSERT INTO temp_user_info VALUES (?,?,?,?,?,?,?,?,?,?)"); //需要替换
//System.out.println("数据大小：" + datas.size());
int num = 0;
for (Data v : datas) {
num++;
stmt.setString(1, v.getStr1());
stmt.setString(2, v.getStr2());
stmt.setString(3, v.getStr3());
stmt.setString(4, v.getStr4());
stmt.setString(5, v.getStr5());
// stmt.setString(6, v.getStr6());
stmt.setString(6, v.getStr6());
stmt.setString(7, v.getStr7());
stmt.setString(8, v.getStr8());
stmt.setString(9, v.getStr9());
stmt.setString(10, v.getStr10());
stmt.addBatch();
// 每5万，提交一次
if (num > 50000) {
stmt.executeBatch();
conn.commit();
num = 0;
}
}
stmt.executeBatch();
conn.commit();
} catch (Exception e) {
e.printStackTrace();
try {
conn.rollback();
} catch (SQLException e1) {
e1.printStackTrace();
}
} finally {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
long endTime = System.currentTimeMillis();
System.out.println("方法执行时间：" + (endTime - startTime) + "ms");
}
}
class Data implements Serializable{
private static final long serialVersionUID = 1L;
private String str1;
private String str2;
private String str3;
private String str4;
private String str5;
private String str6;
private String str7;
private String str8;
private String str9;
private String str10;
public String getStr7() {
return str7;
}
public void setStr7(String str7) {
this.str7 = str7;
}
public String getStr8() {
return str8;
}
public void setStr8(String str8) {
this.str8 = str8;
}
public String getStr9() {
return str9;
}
public void setStr9(String str9) {
this.str9 = str9;
}
public String getStr10() {
return str10;
}
public void setStr10(String str10) {
this.str10 = str10;
}
public String getStr1() {
return str1;
}
public void setStr1(String str1) {
this.str1 = str1;
}
public String getStr2() {
return str2;
}
public void setStr2(String str2) {
this.str2 = str2;
}
public String getStr3() {
return str3;
}
public void setStr3(String str3) {
this.str3 = str3;
}
public String getStr4() {
return str4;
}
public void setStr4(String str4) {
this.str4 = str4;
}
public String getStr5() {
return str5;
}
public void setStr5(String str5) {
this.str5 = str5;
}
public String getStr6() {
return str6;
}
public void setStr6(String str6) {
this.str6 = str6;
}
}
}
pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.imodule.dataImport</groupId>
<artifactId>dataImport</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>dataImport</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<!-- poi 相关 -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>4.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>4.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<version>4.0.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-lang/commons-lang -->
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.oracle/ojdbc6 -->
<dependency>
<groupId>ojdbc</groupId>
<artifactId>ojdbc</artifactId>
<version>6</version>
</dependency>
</dependencies>
</project>
The main pom which need this package seems to get more difficult
<dependency>
<groupId>ojdbc</groupId>
<artifactId>ojdbc</artifactId>
<version>6</version>
</dependency>
This time I have to see Shashi Hou put the package reached my github up, that time will be updated blog, available for download address
There is, when a data import to note two values
1.null
2. date value type
About null, my plan is to determine whether the process is null, if null on assignment ""
data.setStr1(row.getCell(0) == null? "":row.getCell(0).getStringCellValue());
About the date, my treatment program is a manual conversion (here can not directly use the data type, the date will automatically calculated as 49555.4 similar data, that time is saved to the database such data, so we use this string it ~)
if(row.getCell(6) == null){
data.setStr7("");
}else{
Date d = (Date) row.getCell(6).getDateCellValue();
DateFormat formater = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String date = formater.format(d);
data.setStr7(date);
}