POI高效百万级别导出-1min

版权声明：原创文章，未经允许不得转载. https://blog.csdn.net/shengqianfeng/article/details/88643222

思路：在数据量达到百万级别的情况下，如果一次性读取并写入Excel会比较慢，如果将数据写入同一个Excel的多个Sheet，并分别通过多个线程分别写Sheet，效率将会提高很多。

经测试：100w行，每行16列的数据导出大概60s到70s之间。

使用工具：

        // 总记录数
        Integer totalRowCount =ipccDao.getSearchSize("customer.selectExportCustNum", params);
 
        // 导出EXCEL文件名称
        String filaName = "客户信息";
 
        // 标题
        String[] titles = {"客户名称", "客户名称", "联系人", "联系方式", "负责人", 
        		"分配时间", "客户意向等级", "最后跟进时间", "通话状态","跟进阶段", 
        		"通话轮次","最近外呼任务","客户来源","创建时间","客户关注点","通话时长"};
        
        // 开始导入
		PoiUtil.exportExcelToWebsite(StrutsUtils.getResponse(), totalRowCount, filaName, titles,
				new WriteExcelDataDelegated() {
					@Override
					public void writeExcelData(SXSSFSheet eachSheet, Integer startRowCount, Integer endRowCount,
							Integer currentPage, Integer pageSize) throws Exception {
						int startRow = (currentPage - 1) * pageSize; // 分页开始行号
						params.put("limit", " LIMIT " + startRow + ", " + pageSize);
						//List<Map<String, Object>> exportCust = customerService.exportCust(params);
						List<Map<String, Object>> exportCust = ipccDao.getSearchList("customer.selectCustomer", params);
						if (!CollectionUtils.isEmpty(exportCust)) {
							for (int i = startRowCount; i <= endRowCount; i++) {
								SXSSFRow eachDataRow = eachSheet.createRow(i);
								if ((i - startRowCount) < exportCust.size()) {
									Map<String, Object> customer = exportCust.get(i - startRowCount);
									eachDataRow.createCell(0).setCellValue(
											customer.get("name") != null ? customer.get("name") + "" : "");
									eachDataRow.createCell(1).setCellValue(customer.get("customer_nbr") != null? customer.get("customer_nbr") + "": "");
									eachDataRow.createCell(2).setCellValue(customer.get("contacts_name") != null? customer.get("contacts_name") + "": "");
									eachDataRow.createCell(3).setCellValue(customer.get("contacts_nbr")!=null?customer.get("contacts_nbr")+ "":"");
									eachDataRow.createCell(4).setCellValue(customer.get("leader")!=null?customer.get("leader") + "":"");
									eachDataRow.createCell(5).setCellValue(customer.get("distribution_date")!=null?customer.get("distribution_date") + "":"");
									eachDataRow.createCell(6).setCellValue(customer.get("cust_type")!=null?customer.get("cust_type") + "":"");
									eachDataRow.createCell(7).setCellValue(customer.get("last_contact_date")!=null?customer.get("last_contact_date") + "":"");
									changeCallStatusToMsg(customer);
									eachDataRow.createCell(8).setCellValue(customer.get("status") + "");
									changeContactGradation(customer);
									eachDataRow.createCell(9).setCellValue(customer.get("contact_gradation") + "");
									eachDataRow.createCell(10).setCellValue(customer.get("talk_times")!=null?customer.get("talk_times") + "":"");
									eachDataRow.createCell(11).setCellValue(customer.get("task_name")!=null?customer.get("task_name") + "":"");
									eachDataRow.createCell(12).setCellValue(customer.get("finder") !=null?customer.get("finder") + "":"");
									eachDataRow.createCell(13).setCellValue(customer.get("create_date")!=null?customer.get("create_date") + "":"");
									eachDataRow.createCell(14).setCellValue(customer.get("result_label")!=null?customer.get("result_label") + "":"");
									eachDataRow.createCell(15).setCellValue(customer.get("tel_time")!=null?customer.get("tel_time") + "":"");
								}
							}
						}

					}
				});

POIUtil.java工具类

package com.admin.util;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import javax.servlet.http.HttpServletResponse;

import org.apache.poi.xssf.streaming.SXSSFCell;
import org.apache.poi.xssf.streaming.SXSSFRow;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.admin.task.ExportExcelCallable;
 
/**
 * @author qjwyss
 * @date 2018/9/18
 * @description POI导出工具类
 */
public class PoiUtil {
	
	/**
     * 每个sheet存储的记录数10W
     */
    public static Integer PER_SHEET_ROW_COUNT;
 
    /**
     * 每次向EXCEL写入的记录数(查询每页数据大小) 5W
     */
    public static Integer PER_WRITE_ROW_COUNT;
    
    static{
    	PER_SHEET_ROW_COUNT = SysConfig.getInstance().getPropertyInt("PER_SHEET_ROW_COUNT",200000);
    	PER_WRITE_ROW_COUNT = SysConfig.getInstance().getPropertyInt("PER_WRITE_ROW_COUNT",100000);
    }
 
 
    /**
     * 每个sheet的写入次数 2
     */
    public static final Integer PER_SHEET_WRITE_COUNT = PER_SHEET_ROW_COUNT / PER_WRITE_ROW_COUNT;
 
    private final static Logger logger = LoggerFactory.getLogger(PoiUtil.class);
    
    
    /**
     * 初始化EXCEL(sheet个数和标题)
     *
     * @param totalRowCount 总记录数
     * @param titles        标题集合
     * @return XSSFWorkbook对象
     */
    public static SXSSFWorkbook initExcel(Integer totalRowCount, String[] titles) {
    	logger.info("PER_SHEET_ROW_COUNT：{}，PER_WRITE_ROW_COUNT：{}",PER_SHEET_ROW_COUNT,PER_WRITE_ROW_COUNT);
        // 在内存当中保持 100 行 , 超过的数据放到硬盘中在内存当中保持 100 行 , 超过的数据放到硬盘中
        SXSSFWorkbook wb = new SXSSFWorkbook(100);
 
        Integer sheetCount = ((totalRowCount % PER_SHEET_ROW_COUNT == 0) ?
                (totalRowCount / PER_SHEET_ROW_COUNT) : (totalRowCount / PER_SHEET_ROW_COUNT + 1));
 
        // 根据总记录数创建sheet并分配标题
        for (int i = 0; i < sheetCount; i++) {
            SXSSFSheet sheet = wb.createSheet("sheet" + (i + 1));
            sheet.setRandomAccessWindowSize(-1);
            SXSSFRow headRow = sheet.createRow(0);
 
            for (int j = 0; j < titles.length; j++) {
                SXSSFCell headRowCell = headRow.createCell(j);
                headRowCell.setCellValue(titles[j]);
            }
        }
 
        return wb;
    }
 
 
    /**
     * 下载EXCEL到本地指定的文件夹
     *
     * @param wb         EXCEL对象SXSSFWorkbook
     * @param exportPath 导出路径
     */
    public static void downLoadExcelToLocalPath(SXSSFWorkbook wb, String exportPath) {
        FileOutputStream fops = null;
        try {
            fops = new FileOutputStream(exportPath);
            wb.write(fops);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (null != wb) {
                try {
                    wb.dispose();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            if (null != fops) {
                try {
                    fops.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
 
 
    /**
     * 下载EXCEL到浏览器
     *
     * @param wb       EXCEL对象XSSFWorkbook
     * @param response
     * @param fileName 文件名称
     * @throws IOException
     */
    public static void downLoadExcelToWebsite(SXSSFWorkbook wb, HttpServletResponse response, String fileName) throws IOException {
 
        response.setHeader("Content-disposition", "attachment; filename="
                + new String((fileName + ".xlsx").getBytes("utf-8"), "ISO8859-1"));//设置下载的文件名
 
        OutputStream outputStream = null;
        try {
            outputStream = response.getOutputStream();
            wb.write(outputStream);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (null != wb) {
                try {
                    wb.dispose();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            if (null != outputStream) {
                try {
                    outputStream.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
 
 
    /**
     * 导出Excel到本地指定路径
     *
     * @param totalRowCount           总记录数
     * @param titles                  标题
     * @param exportPath              导出路径
     * @param writeExcelDataDelegated 向EXCEL写数据/处理格式的委托类 自行实现
     * @throws Exception
     */
    public static final void exportExcelToLocalPath(Integer totalRowCount, String[] titles, String exportPath, WriteExcelDataDelegated writeExcelDataDelegated) throws Exception {
 
        logger.info("开始导出：" + formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
 
        // 初始化EXCEL
        SXSSFWorkbook wb = PoiUtil.initExcel(totalRowCount, titles);
 
        // 调用委托类分批写数据
        int sheetCount = wb.getNumberOfSheets();
        for (int i = 0; i < sheetCount; i++) {
            SXSSFSheet eachSheet = wb.getSheetAt(i);
 
            for (int j = 1; j <= PER_SHEET_WRITE_COUNT; j++) {
 
                int currentPage = i * PER_SHEET_WRITE_COUNT + j;
                int pageSize = PER_WRITE_ROW_COUNT;
                int startRowCount = (j - 1) * PER_WRITE_ROW_COUNT + 1;
                int endRowCount = startRowCount + pageSize - 1;
 
 
                writeExcelDataDelegated.writeExcelData(eachSheet, startRowCount, endRowCount, currentPage, pageSize);
 
            }
        }
 
 
        // 下载EXCEL
        PoiUtil.downLoadExcelToLocalPath(wb, exportPath);
 
        logger.info("导出完成：" + formatDate(new Date(),"yyyy-MM-dd HH:mm:ss"));
    }
 
    
    /**
     * 将日期转换为字符串
     *
     * @param date   DATE日期
     * @param format 转换格式
     * @return 字符串日期
     */
    public static String formatDate(Date date, String format) {
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat(format);
        return simpleDateFormat.format(date);
    }
    

    public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss";
 
    /**
     * 导出Excel到浏览器
     *
     * @param response
     * @param totalRowCount           总记录数
     * @param fileName                文件名称
     * @param titles                  标题
     * @param writeExcelDataDelegated 向EXCEL写数据/处理格式的委托类 自行实现
     * @throws Exception
     */
    public static final void exportExcelToWebsite(HttpServletResponse response, Integer totalRowCount, String fileName, String[] titles, WriteExcelDataDelegated writeExcelDataDelegated) throws Exception {
 
        logger.info("开始导出：" + formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
        long start = System.currentTimeMillis();
        // 初始化EXCEL
        SXSSFWorkbook wb = PoiUtil.initExcel(totalRowCount, titles);
 
 
        // 调用委托类分批写数据
        int sheetCount = wb.getNumberOfSheets();
        ExecutorService exportSheetPool = Executors.newFixedThreadPool(sheetCount);//工头
        ArrayList<Future<String>> results = new ArrayList<Future<String>>();
        for (int i = 0; i < sheetCount; i++) {
            SXSSFSheet eachSheet = wb.getSheetAt(i);
            for (int j = 1; j <= PER_SHEET_WRITE_COUNT; j++) {
                int currentPage = i * PER_SHEET_WRITE_COUNT + j;
                int pageSize = PER_WRITE_ROW_COUNT;
                int startRowCount = (j - 1) * PER_WRITE_ROW_COUNT + 1;
                int endRowCount = startRowCount + pageSize - 1;
                
                ExportExcelCallable exportExcelCallable =  new ExportExcelCallable(writeExcelDataDelegated, eachSheet, startRowCount, endRowCount, currentPage, pageSize);
                results.add(exportSheetPool.submit(exportExcelCallable));//submit返回一个Future，代表了即将要返回的结果
            }
        }
        boolean noFinishflag = true;
        while(noFinishflag){
        	for (Future<String> future : results) {
        		//首先校验是否结束，没有结束直接跳出for循环进行下一次while
        		if(!future.isDone()){
        			noFinishflag = true;
        			break;
        		}else{
        			noFinishflag = false;
        		}
        	}
        }
        logger.info("下载前："+formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
        // 下载EXCEL
        PoiUtil.downLoadExcelToWebsite(wb, response, fileName);
        logger.info("下载后："+formatDate(new Date(), YYYY_MM_DD_HH_MM_SS));
        long end = System.currentTimeMillis();
        logger.info("导出完成：" + formatDate(new Date(), YYYY_MM_DD_HH_MM_SS)+"总消耗："+(end-start));
    }

}

package com.admin.task;

import java.util.concurrent.Callable;

import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.admin.constant.Consts;
import com.admin.util.WriteExcelDataDelegated;

/**
 * 
 * 导出Excel异步处理类
 * 根据多个sheet创建多个线程分别去写不同的sheet，平衡最大处理量和最快处理时间。
 *
 * @author jeffSheng
 * 2019年3月18日
 */
public class ExportExcelCallable implements Callable<String>{
	private static final Logger logger = LoggerFactory.getLogger(ExportExcelCallable.class);
	private WriteExcelDataDelegated writeExcelDataDelegated;
	private SXSSFSheet eachSheet;
	private Integer startRowCount;
	private Integer endRowCount;
	private Integer currentPage;
	private Integer pageSize;
	
	
	public ExportExcelCallable(WriteExcelDataDelegated writeExcelDataDelegated,SXSSFSheet eachSheet,Integer startRowCount,Integer endRowCount,Integer currentPage,Integer pageSize){
		this.writeExcelDataDelegated = writeExcelDataDelegated;
		this.eachSheet = eachSheet;
		this.startRowCount = startRowCount;
		this.endRowCount = endRowCount;
		this.currentPage=currentPage;
		this.pageSize = pageSize;
	}
	
	/**
	 *  eachSheet 指定SHEET
		startRowCount 开始行
		endRowCount 结束行
		currentPage 分批查询开始页
		pageSize 分批查询数据量
	 */
	@Override
	public String call() throws Exception {
		try {
			writeExcelDataDelegated.writeExcelData(eachSheet, startRowCount, endRowCount, currentPage, pageSize);
			return Consts.Success;
		} catch (Exception e) {
			e.printStackTrace();
			logger.error("导出异常："+e);
		}
		return Consts.Fail;
	}



}

重点说明下一下方法：

 /**
     * 初始化EXCEL(sheet个数和标题)
     *
     * @param totalRowCount 总记录数
     * @param titles        标题集合
     * @return XSSFWorkbook对象
     */
    public static SXSSFWorkbook initExcel(Integer totalRowCount, String[] titles) {
    	logger.info("PER_SHEET_ROW_COUNT：{}，PER_WRITE_ROW_COUNT：{}",PER_SHEET_ROW_COUNT,PER_WRITE_ROW_COUNT);
        // 在内存当中保持 100 行 , 超过的数据放到硬盘中在内存当中保持 100 行 , 超过的数据放到硬盘中
        SXSSFWorkbook wb = new SXSSFWorkbook(100);
 
        Integer sheetCount = ((totalRowCount % PER_SHEET_ROW_COUNT == 0) ?
                (totalRowCount / PER_SHEET_ROW_COUNT) : (totalRowCount / PER_SHEET_ROW_COUNT + 1));
 
        // 根据总记录数创建sheet并分配标题
        for (int i = 0; i < sheetCount; i++) {
            SXSSFSheet sheet = wb.createSheet("sheet" + (i + 1));
            sheet.setRandomAccessWindowSize(-1);
            SXSSFRow headRow = sheet.createRow(0);
 
            for (int j = 0; j < titles.length; j++) {
                SXSSFCell headRowCell = headRow.createCell(j);
                headRowCell.setCellValue(titles[j]);
            }
        }
 
        return wb;
    }
 

  sheet.setRandomAccessWindowSize(-1);//对sheet设置一下无限制访问

这一行如果不添加的话就会报错：

Attempting to write a row[0] in the range [0,0] that is already written to disk.