基于POIExcel导入50+万条数据的基本写法

基本思路如下
poi 基于xml解析（event user model ）
多线程批量插入
软件环境
Springboot2.0 +
MyBatis
Mysql 5.7 +

poi 基于xml解析（event user model ）

package com.sunducation.waterflow.utils;

import java.io.InputStream;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;

import com.sunducation.waterflow.dto.DataDTO;
import org.apache.poi.openxml4j.opc.PackageAccess;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.model.SharedStringsTable;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;

import javax.xml.crypto.Data;

public class ParseXMLUtil {

private final BlockingQueue<DataDTO> results ;

public ParseXMLUtil(BlockingQueue<DataDTO> results) {
this.results = results;
}

public void processFirstSheet(String filename) throws Exception {

OPCPackage pkg = OPCPackage.open(filename, PackageAccess.READ);
try {
XSSFReader r = new XSSFReader(pkg);
SharedStringsTable sst = r.getSharedStringsTable();
XMLReader parser = fetchSheetParser(sst);
// process the first sheet
InputStream sheet2 = r.getSheetsData().next();
InputSource sheetSource = new InputSource(sheet2);
parser.parse(sheetSource);
sheet2.close();
} finally {
pkg.close();
}
}

public void processAllSheets(String filename) throws Exception {
OPCPackage pkg = OPCPackage.open(filename, PackageAccess.READ);
try {
XSSFReader r = new XSSFReader(pkg);
SharedStringsTable sst = r.getSharedStringsTable();
XMLReader parser = fetchSheetParser(sst);
Iterator<InputStream> sheets = r.getSheetsData();
while (sheets.hasNext()) {
System.out.println("Processing new sheet:\n");
InputStream sheet = sheets.next();
InputSource sheetSource = new InputSource(sheet);
parser.parse(sheetSource);
sheet.close();
}
} finally {
pkg.close();
}
}

public XMLReader fetchSheetParser(SharedStringsTable sst) throws SAXException {
XMLReader parser = XMLReaderFactory.createXMLReader();
ContentHandler handler = new SheetHandler(sst,this.results);
parser.setContentHandler(handler);
return parser;
}

/**
* See org.xml.sax.helpers.DefaultHandler javadocs
*/
private static class SheetHandler extends DefaultHandler {
private final SharedStringsTable sst;
private String lastContents;
private boolean nextIsString;
private boolean inlineStr;
private String row ="";
private final BlockingQueue<DataDTO> results;
private DataDTO dto = new DataDTO();
private final LruCache<Integer,String> lruCache = new LruCache<Integer,String>(50);

private static class LruCache<A,B> extends LinkedHashMap<A, B> {
private final int maxEntries;

public LruCache(final int maxEntries) {
super(maxEntries + 1, 1.0f, true);
this.maxEntries = maxEntries;
}

@Override
protected boolean removeEldestEntry(final Map.Entry<A, B> eldest) {
return super.size() > maxEntries;
}
}

private SheetHandler(SharedStringsTable sst,final BlockingQueue<DataDTO> results) {
this.sst = sst;
this.results = results;
}

@Override
public void startElement(String uri, String localName, String name,
Attributes attributes) throws SAXException {
// c => cell
if(name.equals("c")) {
// Print the cell reference
row = attributes.getValue("r") ;
String cellType = attributes.getValue("t");
nextIsString = cellType != null && cellType.equals("s");
inlineStr = cellType != null && cellType.equals("inlineStr");
}
// Clear contents cache
lastContents = "";
}

@Override
public void endElement(String uri, String localName, String name)
throws SAXException {
// Process the last contents as required.
// Do now, as characters() may be called more than once
if(nextIsString) {
Integer idx = Integer.valueOf(lastContents);
lastContents = lruCache.get(idx);
if (lastContents == null && !lruCache.containsKey(idx)) {
lastContents = new XSSFRichTextString(sst.getEntryAt(idx)).toString();
lruCache.put(idx, lastContents);
}
nextIsString = false;
}

// v => contents of a cell
// Output after we've seen the string contents
if(name.equals("v") || (inlineStr && name.equals("c"))) {
char rowC=row.substring(0,1).charAt(0);
int rowNum=Integer.parseInt(row.substring(1));
if(rowNum <=1){
return;
}
switch (rowC){
case 65:
dto.setNo(Integer.parseInt(lastContents.toString()));
break;
case 66:
dto.setGrade((String) lastContents);
break;
case 67:
dto.setName((String) lastContents);
break;
case 68:
dto.setAge(Integer.parseInt(lastContents.toString()));
break;
}
}
if(name.equals("row")){
int rowNum=Integer.parseInt(row.substring(1));
if(rowNum <=1){
return;
}
results.offer(dto);
dto = new DataDTO();

}

@Override
public void characters(char[] ch, int start, int length) throws SAXException { // NOSONAR
lastContents += new String(ch, start, length);
}
}

public static void main(String[] args) throws Exception {
long startMini = System.currentTimeMillis();
BlockingQueue<DataDTO> results = new ArrayBlockingQueue<DataDTO>(16);
ParseXMLUtil xmlUtil = new ParseXMLUtil(results);
xmlUtil.processFirstSheet("C:/Users/92039/Desktop/print/demo2.xlsx");
System.out.println(results.size());
long endMini = System.currentTimeMillis();
System.out.println(" take time is " + (endMini - startMini)/1000.0);
}
}
多线程批量插入
Mapper.xml

<insert id="batchInsert" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
INSERT INTO import_data (no,name,grade,age)
VALUES
<foreach collection="list" item="item" separator=",">
(#{item.no}, #{item.name},#{item.grade},#{item.age})
</foreach>
</insert>

Service 层
package com.sunducation.waterflow.service.impl;

import com.sunducation.waterflow.dto.DataDTO;
import com.sunducation.waterflow.muti.InsertDataTask;
import com.sunducation.waterflow.service.ImportDataService;
import com.sunducation.waterflow.utils.ParseXMLUtil;
import org.springframework.stereotype.Service;

import java.util.concurrent.*;

/**
* @version 1.0
* @description:
* @projectName: com.sunducation.rbac.service.impl
* @className: rbac
* @author:tannc
* @createTime:2018/8/2 14:26
*/
@Service
public class ImportDataServiceImpl implements ImportDataService {

@Override
public double parseAndinsertDb(String filepath) throws Exception {
//
final int maxThreadNums = 5;
long startMini = System.currentTimeMillis();
final BlockingQueue<DataDTO> results = new LinkedBlockingQueue<DataDTO>();
// 多线程插入
final CyclicBarrier barrier = new CyclicBarrier(maxThreadNums+1);
ExecutorService executorService = Executors.newFixedThreadPool(maxThreadNums);
executorService.execute(new Runnable() {
@Override
public void run() { // 解析
ParseXMLUtil xmlUtil = new ParseXMLUtil(results);
try {
xmlUtil.processFirstSheet(filepath);
barrier.await();
} catch (Exception e) {
e.printStackTrace();
}
}
});
Thread.sleep(500);
for(int i =1 ;i<maxThreadNums;i++){
// 提交任务
executorService.execute(new InsertDataTask(results, barrier, "task"+i));
}
barrier.await();
executorService.shutdown();
// System.out.println("执行完成");
// System.out.println(results.size());
long endMini = System.currentTimeMillis();
// System.out.println(" take time is " + (endMini - startMini)/1000.0);
return (endMini - startMini)/1000.0;
}
}

任务层 InsertDataTask
package com.sunducation.waterflow.muti;

import com.sunducation.waterflow.dao.mapper.ImportDataMapper;
import com.sunducation.waterflow.dto.DataDTO;
import com.sunducation.waterflow.utils.SpringContextUtils;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;

/**
* @version 1.0
* @description: 另外线程插入
* @projectName: com.kongxiang.muti
* @className: DesignModel
* @author:tannc
* @createTime:2018/9/15 9:36
*/
public class InsertDataTask implements Runnable {

private final ImportDataMapper importDataMapper;

private final BlockingQueue<DataDTO> queue;

private final CyclicBarrier barrier;
private final String taskName ;

public InsertDataTask(BlockingQueue<DataDTO> queue, CyclicBarrier barrier,String taskName) {
this.queue = queue;
this.barrier = barrier;
this.taskName =taskName;
importDataMapper = SpringContextUtils.getBean("importDataMapper");
}

@Override
public void run() {
List<DataDTO> list = new ArrayList<DataDTO>(256);
final ReentrantLock lock = new ReentrantLock();
while (!queue.isEmpty()) {
lock.lock();
list.clear();
// 出栈
queue.drainTo(list,10000);
//分批处理
if(null!=list&&list.size()>0){
// System.err.println(taskName + " 插入size +++> " + list.size());
int pointsDataLimit = 10000;//限制条数
Integer size = list.size();
//判断是否有必要分批
if(pointsDataLimit<size){
int part = size/pointsDataLimit;//分批数
// System.out.println("共有： "+size+"条，！"+" 分为："+part+"批");
//
for (int i = 0; i < part; i++) {
//1000条
List<DataDTO> listPage = list.subList(0, pointsDataLimit);
importDataMapper.batchInsert(listPage);
//剔除
list.subList(0, pointsDataLimit).clear();
}
if(!list.isEmpty()){
//表示最后剩下的数据
importDataMapper.batchInsert(list);
}
}else{
importDataMapper.batchInsert(list);
}
} // end if
if (queue.isEmpty()){
try {
Thread.sleep(500);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
lock.unlock();
} // while
try {
barrier.await();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (BrokenBarrierException e) {
e.printStackTrace();
}
}
}

Controller 层
package com.sunducation.waterflow.controller;

import com.sunducation.waterflow.ResultMsg;
import com.sunducation.waterflow.service.ImportDataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletRequest;
import java.io.*;

/**
* @version 1.0
* @description:
* @projectName: 上传文件
* @className: 上传文件
* @author:tannc
* @createTime:2018/9/6 19:32
*/
@RestController
@RequestMapping("/api/v1/uploads")
public class UploadFileController {

@Autowired
private ImportDataService importDataService ;
//处理文件上传
@RequestMapping(value="/excel", method = RequestMethod.POST)
public ResultMsg uploadImg(@RequestParam("file") MultipartFile file, HttpServletRequest request) throws Exception {
if (!file.isEmpty()) {
//
String filename = file.getOriginalFilename();
String prefix=filename.substring(filename.lastIndexOf(".")+1);
// Excel 2007版9
if( null != prefix && prefix.equalsIgnoreCase("xlsx")){
// 返回记录数
double count= 0.0d;
try {
File f = null;
f=File.createTempFile("tmp", null);
file.transferTo(f);
f.deleteOnExit();
// 文件路径
count = importDataService.parseAndinsertDb(f.getAbsolutePath());
} catch (IOException e) {
e.printStackTrace();
}
return new ResultMsg(0,"文件上传成功，共花费 " + count +" 秒");
}
else{
return new ResultMsg(400,"请选择文件");
}

} else {
return new ResultMsg(400,"文件不存在");
}
}

}

完整代码下载地址：https://github.com/tannongchun/bigdataimport.git
---------------------
作者：春馨梦
来源：CSDN
原文：https://blog.csdn.net/tannongchun/article/details/82729635
版权声明：本文为博主原创文章，转载请附上博文链接！

基于POIExcel导入50+万条数据的基本写法

猜你喜欢