一、需求分析
导入Excel数据，验证数据是否符合标准，如果不符合则返回所有错误的行数，数据量为每个Excel为2w~3w左右数据量，导入较慢，通过线程池把数据分块为每1000条为一批数据，同时导入数据库提高速度。
二、Excel文件标准
三、代码实现
Controller
/**
* 导入项目长势Excel
* @param id
* @return
* @author ygc
* @throws IOException
*/
@RequestMapping("/ImportProjectGrowthExcel")
@SecurityParameter(inDecode=false,outEncode=true)
@ResponseBody
@Transactional(propagation=Propagation.REQUIRED,rollbackFor = Exception.class)
public JsonResultInfo ImportProjectGrowthExcel(HttpServletRequest request,HttpServletResponse response) throws IOException {
response.setContentType("text/html;charset=UTF-8");
response.setHeader("Content-type", "application/json;charset=UTF-8");
JsonResultInfo info=new JsonResultInfo();
long startTime = System.currentTimeMillis(); //获取开始时间
boolean temp=projectService.ImportProjectGrowthExcel(request,info);
System.out.println("test:"+info.getResult());
long endTime = System.currentTimeMillis();
System.out.println("程序运行时间：" + (endTime - startTime) + "ms"); //输出程序运行时间
// info.setCode(200);
// info.setData(temp);
// info.setResult("成功");
// info.setMessage("导入项目定损Excel");
System.out.println(info);
return info;
}Service
public boolean ImportProjectGrowthExcel(HttpServletRequest request, JsonResultInfo info) {
MultipartResolver resolver = new CommonsMultipartResolver(request.getSession().getServletContext());
MultipartHttpServletRequest multipartRequest = resolver.resolveMultipart(request);
String requestData = multipartRequest.getParameter("requestData");
String encrypted = multipartRequest.getParameter("encrypted");
String content = DecryptionFileUtil.decryption(requestData, encrypted);
JSONObject jsonObject = JSON.parseObject(content);
String token = (String) jsonObject.get("token");
String userInfo = redisOperator.get(token);
JSONObject userJson = JSONObject.parseObject(userInfo);
SysUserInfo user = JSON.toJavaObject(userJson, SysUserInfo.class);
String uid = (String) jsonObject.get("uid");
info.setUid(uid);
if (RedisPool.checkToken(token) == false) {
return false;
}
int count=1;
JSONObject data = (JSONObject) jsonObject.get("data");
InsProjectGrowthtable image = JSONObject.toJavaObject(data, InsProjectGrowthtable.class);
String deletePath = Commons.MASTER_PATH + File.separator + Commons.IMPORT_PROJECT_GROWTH_EXCEL + File.separator+ image.getProjectid();
File itemPath = new File(deletePath);
FileUtil.deleteDir(itemPath);
Iterator<String> iter = multipartRequest.getFileNames();
String path = null;
String fileName=null;
while (iter.hasNext()) {
MultipartFile file = multipartRequest.getFile(iter.next().toString());
if (file.isEmpty()) {
continue;
} else {
fileName = file.getOriginalFilename();
// 文件路径，可以替换为自己的文件夹
String savePath = Commons.MASTER_PATH + File.separator + Commons.IMPORT_PROJECT_GROWTH_EXCEL
+ File.separator + image.getProjectid();
// String savePath="E:\\AMapUsermark";
File storeDirectory = new File(savePath);
if (!storeDirectory.exists()) {
storeDirectory.mkdirs();
}
// 文件路径+文件名
path = savePath + File.separator + fileName;
try {
// 保存文件
file.transferTo(new File(savePath, fileName));
} catch (IllegalStateException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
// Excel解析
List<Map<Integer, String>> result = new ArrayList<Map<Integer, String>>();
File file = new File(path);
XSSFWorkbook workbook;
try {
// workbook = new XSSFWorkbook(FileUtils.openInputStream(file));
OPCPackage opcPackage = OPCPackage.open(file.getAbsolutePath());
// workbook = new XSSFWorkbook(new FileInputStream(file));
workbook = new XSSFWorkbook(opcPackage);
opcPackage.close();
// 第二种获取方式通过index 获取第一个表
Sheet sheet = workbook.getSheetAt(0);
int lastRowNum = sheet.getLastRowNum();
Map<String,Integer> map1=new LinkedHashMap<String,Integer>();
Row row = sheet.getRow(0);
//获取当前最后单元格列号
int lastCellNum = row.getLastCellNum();
for(int j=0;j<lastCellNum;j++){
Cell cell = row.getCell(j);
if(cell==null) {
continue;
}else {
if(cell.toString().equals("县")) {
map1.put("县", cell.getColumnIndex());
}
if(cell.toString().equals("镇")) {
map1.put("镇", cell.getColumnIndex());
}
if(cell.toString().equals("村")) {
map1.put("村", cell.getColumnIndex());
}
if(cell.toString().equals("地块编号")) {
map1.put("地块编号", cell.getColumnIndex());
}
if(cell.toString().equals("用户")) {
map1.put("用户", cell.getColumnIndex());
}
if(cell.toString().equals("种植面积")){
map1.put("种植面积", cell.getColumnIndex());
}
if(cell.toString().equals("1.0")){
map1.put("1.0", cell.getColumnIndex());
}
if(cell.toString().equals("2.0")){
map1.put("2.0", cell.getColumnIndex());
}
if(cell.toString().equals("3.0")){
map1.put("3.0", cell.getColumnIndex());
}
if(cell.toString().equals("4.0")){
map1.put("4.0", cell.getColumnIndex());
}
if(cell.toString().equals("5.0")){
map1.put("5.0", cell.getColumnIndex());
}
if(cell.toString().equals("6.0")){
map1.put("6.0", cell.getColumnIndex());
}
if(cell.toString().equals("7.0")){
map1.put("7.0", cell.getColumnIndex());
}
if(cell.toString().equals("8.0")){
map1.put("8.0", cell.getColumnIndex());
}
if(cell.toString().equals("9.0")){
map1.put("9.0", cell.getColumnIndex());
}
if(cell.toString().equals("10.0")){
map1.put("10.0", cell.getColumnIndex());
}
}
}
// 从第三行开始导入
int firstRowNum = 1;
Map<Integer, String> map = null;
// 解析数据
for (int i = firstRowNum; i <= lastRowNum; i++) {
map=new HashMap<Integer,String>();
Row row1 = sheet.getRow(i);
//获取当前最后单元格列号
int lastCellNum1 = row1.getLastCellNum();
for(int j=0;j<lastCellNum1;j++){
if(map1.get("县")!=null) {
Cell cell = row1.getCell(map1.get("县"));
if (cell != null) {
cell.setCellType(Cell.CELL_TYPE_STRING);
String value = cell.getStringCellValue();
map.put(map1.get("县"), value);
// System.out.print(value + " ");//注意value后面的空格
} else {
map.put(map1.get("县"),null);
}
}
if(map1.get("镇")!=null) {
Cell cell1 = row1.getCell(map1.get("镇"));
if (cell1 != null) {
cell1.setCellType(Cell.CELL_TYPE_STRING);
String value = cell1.getStringCellValue();
map.put(map1.get("镇"), value);
} else {
map.put(map1.get("镇"),null);
}
}
if(map1.get("村")!=null) {
Cell cell2 = row1.getCell(map1.get("村"));
if (cell2 != null) {
cell2.setCellType(Cell.CELL_TYPE_STRING);
String value = cell2.getStringCellValue();
map.put(map1.get("村"), value);
} else {
map.put(map1.get("村"),null);
}
}
if(map1.get("地块编号")!=null) {
Cell cell3 = row1.getCell(map1.get("地块编号"));
if (cell3 != null) {
cell3.setCellType(Cell.CELL_TYPE_STRING);
String value = cell3.getStringCellValue();
map.put(map1.get("地块编号"), value);
} else {
map.put(map1.get("地块编号"),null);
}
}
if(map1.get("用户")!=null) {
Cell cell4 = row1.getCell(map1.get("用户"));
if (cell4 != null) {
cell4.setCellType(Cell.CELL_TYPE_STRING);
String value = cell4.getStringCellValue();
map.put(map1.get("用户"), value);
} else {
map.put(map1.get("用户"),null);
}
}
if(map1.get("种植面积")!=null) {
Cell cell5 = row1.getCell(map1.get("种植面积"));
if (cell5 != null) {
cell5.setCellType(Cell.CELL_TYPE_STRING);
String value = cell5.getStringCellValue();
map.put(map1.get("种植面积"), value);
} else {
map.put(map1.get("种植面积"),null);
}
}
//未完成
if(map1.get("1.0")!=null) {
Cell cell6 = row1.getCell(map1.get("1.0"));
if (cell6 != null) {
cell6.setCellType(Cell.CELL_TYPE_STRING);
String value = cell6.getStringCellValue();
map.put(map1.get("1.0"), value);
} else {
map.put(map1.get("1.0"),null);
}
}
//未完成
if(map1.get("2.0")!=null) {
Cell cell7 = row1.getCell(map1.get("2.0"));
if (cell7 != null) {
cell7.setCellType(Cell.CELL_TYPE_STRING);
String value = cell7.getStringCellValue();
map.put(map1.get("2.0"), value);
} else {
map.put(map1.get("2.0"),null);
}
}
if(map1.get("3.0")!=null) {
Cell cell8 = row1.getCell(map1.get("3.0"));
if (cell8 != null) {
cell8.setCellType(Cell.CELL_TYPE_STRING);
String value = cell8.getStringCellValue();
map.put(map1.get("3.0"), value);
} else {
map.put(map1.get("3.0"), null);
}
}
if(map1.get("4.0")!=null) {
Cell cell9 = row1.getCell(map1.get("4.0"));
if (cell9 != null) {
cell9.setCellType(Cell.CELL_TYPE_STRING);
String value = cell9.getStringCellValue();
map.put(map1.get("4.0"), value);
} else {
map.put(map1.get("4.0"), null);
}
}
if(map1.get("5.0")!=null) {
Cell cell10 = row1.getCell(map1.get("5.0"));
if (cell10 != null) {
cell10.setCellType(Cell.CELL_TYPE_STRING);
String value = cell10.getStringCellValue();
map.put(map1.get("5.0"), value);
} else {
map.put(map1.get("5.0"), null);
}
}
if(map1.get("6.0")!=null) {
Cell cell11 = row1.getCell(map1.get("6.0"));
if (cell11 != null) {
cell11.setCellType(Cell.CELL_TYPE_STRING);
String value = cell11.getStringCellValue();
map.put(map1.get("6.0"), value);
} else {
map.put(map1.get("6.0"), null);
}
}
if(map1.get("7.0")!=null) {
Cell cell12 = row1.getCell(map1.get("7.0"));
if (cell12 != null) {
cell12.setCellType(Cell.CELL_TYPE_STRING);
String value = cell12.getStringCellValue();
map.put(map1.get("7.0"), value);
} else {
map.put(map1.get("7.0"), null);
}
}
if(map1.get("8.0")!=null) {
Cell cell13 = row1.getCell(map1.get("8.0"));
if (cell13 != null) {
cell13.setCellType(Cell.CELL_TYPE_STRING);
String value = cell13.getStringCellValue();
map.put(map1.get("8.0"), value);
} else {
map.put(map1.get("8.0"), null);
}
}
if(map1.get("9.0")!=null) {
Cell cell14 = row1.getCell(map1.get("9.0"));
if (cell14 != null) {
cell14.setCellType(Cell.CELL_TYPE_STRING);
String value = cell14.getStringCellValue();
map.put(map1.get("9.0"), value);
} else {
map.put(map1.get("9.0"), null);
}
}
if(map1.get("10.0")!=null) {
Cell cell14 = row1.getCell(map1.get("10.0"));
if (cell14 != null) {
cell14.setCellType(Cell.CELL_TYPE_STRING);
String value = cell14.getStringCellValue();
map.put(map1.get("10.0"), value);
} else {
map.put(map1.get("10.0"), null);
}
}
}
result.add(map);
}
List<Integer> errorList=new ArrayList<Integer> ();
for (Map<Integer, String> eachValue : result) {
count++;
try {
InsProjectGrowthtable table = new InsProjectGrowthtable();
table.setProjectid(image.getProjectid());
if(map1.get("县")!=null) {
table.setCounty(eachValue.get(map1.get("县")));
}
if(map1.get("镇")!=null) {
table.setTown(eachValue.get(map1.get("镇")));
}
if(map1.get("村")!=null) {
table.setCountry(eachValue.get(map1.get("村")));
}
if(map1.get("地块编号")!=null) {
table.setGrowthnum(eachValue.get(map1.get("地块编号")));
}
if(map1.get("用户")!=null) {
table.setGrowthuser(eachValue.get(map1.get("用户")));
}
if(map1.get("种植面积")!=null) {
if(eachValue.get(map1.get("种植面积"))==null || eachValue.get(map1.get("种植面积"))=="") {
table.setCropsarea(null);
}else {
table.setCropsarea(Double.valueOf(eachValue.get(map1.get("种植面积"))));
}
}
if(map1.get("1.0")!=null) {
if(eachValue.get(map1.get("1.0"))==null || eachValue.get(map1.get("1.0"))=="") {
table.setOne(null);
}else {
table.setOne(Double.valueOf(eachValue.get(map1.get("1.0"))));
}
}
if(map1.get("2.0")!=null) {
if(eachValue.get(map1.get("2.0"))==null || eachValue.get(map1.get("2.0"))=="") {
table.setTwo(null);
}else {
table.setTwo(Double.valueOf(eachValue.get(map1.get("2.0"))));
}
}
if(map1.get("3.0")!=null) {
if(eachValue.get(map1.get("3.0"))==null || eachValue.get(map1.get("3.0"))=="") {
table.setThree(null);
}else {
table.setThree(Double.valueOf(eachValue.get(map1.get("3.0"))));
}
}
if(map1.get("4.0")!=null) {
if(eachValue.get(map1.get("4.0"))==null || eachValue.get(map1.get("4.0"))=="") {
table.setFour(null);
}else {
table.setFour(Double.valueOf(eachValue.get(map1.get("4.0"))));
}
}
if(map1.get("5.0")!=null) {
if(eachValue.get(map1.get("5.0"))==null || eachValue.get(map1.get("5.0"))=="") {
table.setFive(null);
}else {
table.setFive(Double.valueOf(eachValue.get(map1.get("5.0"))));
}
}
if(map1.get("6.0")!=null) {
if(eachValue.get(map1.get("6.0"))==null || eachValue.get(map1.get("6.0"))=="") {
table.setSix(null);
}else {
table.setSix(Double.valueOf(eachValue.get(map1.get("6.0"))));
}
}
if(map1.get("7.0")!=null) {
if(eachValue.get(map1.get("7.0"))==null || eachValue.get(map1.get("7.0"))=="") {
table.setSeven(null);
}else {
table.setSeven(Double.valueOf(eachValue.get(map1.get("7.0"))));
}
}
if(map1.get("8.0")!=null) {
if(eachValue.get(map1.get("8.0"))==null || eachValue.get(map1.get("8.0"))=="") {
table.setEight(null);
}else {
table.setEight(Double.valueOf(eachValue.get(map1.get("8.0"))));
}
}
if(map1.get("9.0")!=null) {
if(eachValue.get(map1.get("9.0"))==null || eachValue.get(map1.get("9.0"))=="") {
table.setNine(null);
}else {
table.setNine(Double.valueOf(eachValue.get(map1.get("9.0"))));
}
}
if(map1.get("10.0")!=null) {
if(eachValue.get(map1.get("10.0"))==null || eachValue.get(map1.get("10.0"))=="") {
table.setTen(null);
}else {
table.setTen(Double.valueOf(eachValue.get(map1.get("10.0"))));
}
}
} catch (Exception e ) {
e.printStackTrace();
errorList.add(count);
}
}
if(!errorList.isEmpty()) {
String errorCount = StringUtils.join(errorList.toArray(), ",");
info.setCode(300);
info.setData(false);
info.setMessage("导入项目长势Excel");
info.setResult("Excel文件第"+errorCount+"行字段不合法,请检查重新上传!");
return false;
}
//数据分块为1000每批
int batchNum=1000;
//插入
this.batchDeal(result,batchNum,image.getProjectid(),map1,info,fileName,user.getId(),projectMapper);
} catch (Exception e) {
e.printStackTrace();
}
info.setCode(200);
info.setData(true);
info.setResult("成功");
info.setMessage("导入项目长势Excel");
return true;
}
//线程池数据分批导入
private void batchDeal(List<Map<Integer, String>> data, int batchNum,int projectid, Map<String, Integer> map1, JsonResultInfo info, String fileName, Integer id, ProjectMapper projectMapper) throws InterruptedException {
int totalNum = data.size();
int pageNum = totalNum % batchNum == 0 ? totalNum / batchNum : totalNum / batchNum + 1;
ExecutorService executor = Executors.newFixedThreadPool(pageNum);
try {
CountDownLatch countDownLatch = new CountDownLatch(pageNum);
List<Map<Integer, String>> subData = null;
int fromIndex, toIndex;
for (int i = 0; i < pageNum; i++) {
fromIndex = i * batchNum;
toIndex = Math.min(totalNum, fromIndex + batchNum);
subData = data.subList(fromIndex, toIndex);
ImportTask task = new ImportTask(subData, countDownLatch,projectid,map1,info,fileName,id,projectMapper);
executor.execute(task);
}
// 主线程必须在启动其它线程后立即调用CountDownLatch.await()方法，
// 这样主线程的操作就会在这个方法上阻塞，直到其它线程完成各自的任务。
// 计数器的值等于0时，主线程就能通过await()方法恢复执行自己的任务。
countDownLatch.await();
} finally {
// 关闭线程池，释放资源
executor.shutdown();
}
}utils
package com.kero99.utils;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import org.springframework.transaction.interceptor.TransactionAspectSupport;
import com.kero99.mapper.ProjectMapper;
import com.kero99.pojo.InsProjectGrowthtable;
/**
* 线程池分批导入数据
* @author ygc
*
*/
public class ImportTask implements Runnable {
private List<Map<Integer, String>> list;
private CountDownLatch countDownLatch;
private int projectid;
private Map<String, Integer> map1;
private JsonResultInfo info;
private String fileName;
private Integer id;
private ProjectMapper projectMapper;
public ImportTask(List<Map<Integer, String>> data, CountDownLatch countDownLatch,int projectid, Map<String, Integer> map1, JsonResultInfo info, String fileName, Integer id, ProjectMapper projectMapper) {
this.list = data;
this.countDownLatch = countDownLatch;
this.projectid=projectid;
this.map1=map1;
this.info=info;
this.fileName=fileName;
this.id=id;
this.projectMapper=projectMapper;
}
@Override
public void run() {
// int count=1;
if (null != list) {
for (Map<Integer, String> eachValue : list) {
// count++;
try {
InsProjectGrowthtable table = new InsProjectGrowthtable();
table.setProjectid(projectid);
if(map1.get("县")!=null) {
table.setCounty(eachValue.get(map1.get("县")));
}
if(map1.get("镇")!=null) {
table.setTown(eachValue.get(map1.get("镇")));
}
if(map1.get("村")!=null) {
table.setCountry(eachValue.get(map1.get("村")));
}
if(map1.get("地块编号")!=null) {
table.setGrowthnum(eachValue.get(map1.get("地块编号")));
}
if(map1.get("用户")!=null) {
table.setGrowthuser(eachValue.get(map1.get("用户")));
}
if(map1.get("种植面积")!=null) {
if(eachValue.get(map1.get("种植面积"))==null || eachValue.get(map1.get("种植面积"))=="") {
table.setCropsarea(null);
}else {
table.setCropsarea(Double.valueOf(eachValue.get(map1.get("种植面积"))));
}
}
if(map1.get("1.0")!=null) {
if(eachValue.get(map1.get("1.0"))==null || eachValue.get(map1.get("1.0"))=="") {
table.setOne(null);
}else {
table.setOne(Double.valueOf(eachValue.get(map1.get("1.0"))));
}
}
if(map1.get("2.0")!=null) {
if(eachValue.get(map1.get("2.0"))==null || eachValue.get(map1.get("2.0"))=="") {
table.setTwo(null);
}else {
table.setTwo(Double.valueOf(eachValue.get(map1.get("2.0"))));
}
}
if(map1.get("3.0")!=null) {
if(eachValue.get(map1.get("3.0"))==null || eachValue.get(map1.get("3.0"))=="") {
table.setThree(null);
}else {
table.setThree(Double.valueOf(eachValue.get(map1.get("3.0"))));
}
}
if(map1.get("4.0")!=null) {
if(eachValue.get(map1.get("4.0"))==null || eachValue.get(map1.get("4.0"))=="") {
table.setFour(null);
}else {
table.setFour(Double.valueOf(eachValue.get(map1.get("4.0"))));
}
}
if(map1.get("5.0")!=null) {
if(eachValue.get(map1.get("5.0"))==null || eachValue.get(map1.get("5.0"))=="") {
table.setFive(null);
}else {
table.setFive(Double.valueOf(eachValue.get(map1.get("5.0"))));
}
}
if(map1.get("6.0")!=null) {
if(eachValue.get(map1.get("6.0"))==null || eachValue.get(map1.get("6.0"))=="") {
table.setSix(null);
}else {
table.setSix(Double.valueOf(eachValue.get(map1.get("6.0"))));
}
}
if(map1.get("7.0")!=null) {
if(eachValue.get(map1.get("7.0"))==null || eachValue.get(map1.get("7.0"))=="") {
table.setSeven(null);
}else {
table.setSeven(Double.valueOf(eachValue.get(map1.get("7.0"))));
}
}
if(map1.get("8.0")!=null) {
if(eachValue.get(map1.get("8.0"))==null || eachValue.get(map1.get("8.0"))=="") {
table.setEight(null);
}else {
table.setEight(Double.valueOf(eachValue.get(map1.get("8.0"))));
}
}
if(map1.get("9.0")!=null) {
if(eachValue.get(map1.get("9.0"))==null || eachValue.get(map1.get("9.0"))=="") {
table.setNine(null);
}else {
table.setNine(Double.valueOf(eachValue.get(map1.get("9.0"))));
}
}
if(map1.get("10.0")!=null) {
if(eachValue.get(map1.get("10.0"))==null || eachValue.get(map1.get("10.0"))=="") {
table.setTen(null);
}else {
table.setTen(Double.valueOf(eachValue.get(map1.get("10.0"))));
}
}
table.setCreateuser(id);
table.setCreatedate(new Date());
table.setDate(fileName);
int temp = projectMapper.saveExcelProjectGrowthInfo(table);
} catch (Exception e ) {
e.printStackTrace();
// info.setCode(300);
// info.setData(false);
// info.setMessage("导入长势Excel");
// info.setResult("Excel文件第"+count+"行字段不合法,请检查重新上传!");
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();//如果updata2()抛了异常,updata()会回滚,不影响事物正常执行
}
}
}
// 发出线程任务完成的信号
countDownLatch.countDown();
}
}