POI(excel)导出优化

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_36324685/article/details/82527915

寄语：第一次接触百万级别导出，不知道极限，尝试过百度，经历过绝望，内存溢出，等了一万年，不出来文件….

1.要明白 极限：

excel 不同版本的行极限:

    excel 2007 及以上：1048576
          2003 : 65536  行
 不考虑列：因为最少256列够用

2：什么会导致内存溢出：

1.循环量过大
2.创建重复实体过多
3.map 添加多只  
4.实体未释放

3：怎么检测是什么过大：
jconsole

4: 看过无数篇博客，感觉叙述都没用，但其实是业务不对，但其实是能组装业务的
比如说：

1. poi导出100w的excel需要多久：测试了二十列100w行导出 一分半钟就能导出来。
2. 那为什么在系统里会如此慢： 打印日志，查时间，发现mongo 查询 很慢，而且还易报错，比如说16mb的bson，这个会让做大数据很麻烦。但你查询的过程 想到分出去多少万去查询，这个很难，比如说一百万 你分十次 可以，但分五次比十次要测测哪个快，我本人选择的是二十万，因为减少连接，可以快速的拼装数据，而且，利用计算，使流持久化，等我写够一百万在让流释放，如果没写够，让他一直存在。
3. 这样我每二十万就手动控制写出到workbook 里，加快了以后刷出100万的循环。也从而减少了拼装集合。

 /**
     * @param tableID   表id
     * @param tableName 表名称
     * @param request
     * @param response
     * @param query     查询
     * @param isZip     是否是zip
     * @param userName  用户名称  用来生成excelName
     * @return
     * @author liuyu
     * date 2018/8/29
     **/
    private String becomeExcel(String tableID, String tableName,
                               HttpServletRequest request, HttpServletResponse response,
                               Query query, boolean isZip, String userName) throws IOException {
        List<FieldEO> entities = fieldEODao.findByTableID(tableID);
        long count = mongoDB.count(query, tableName);
        String fileName;
        HashMap excelReturnMap = null;
        Workbook workbook = null;
        ZipOutputStream zipOutputStream = null;
        int countIndex = 0;
        if (isZip) {
            fileName = "《" + tableName + "-" + userName + "-" + DateUtils.dateToString(new Date(),
                "yyyy-MM-dd HH-mm-ss") + "》.zip";
            Double countFor = count / 200000.0;
//           每二十万刷新到excel里面
            for (int i = 0; i < countFor.intValue(); i++) {
                List<DBObject> resultList = mongoDB.getList(tableName, new Query(), i * 200000,
                    200000);
                if (excelReturnMap != null) {
                    zipOutputStream = getZipOutputStream(excelReturnMap, zipOutputStream);
                    workbook = getWorkbook(excelReturnMap, workbook);
                    countIndex = getCount(excelReturnMap);
                }
                if (countFor - (i + 1) != 0) {
                    excelReturnMap = ExcelUtils.exportExcel(entities, resultList,
                        fileName, request, response, true, path, i * 200000,
                        true, workbook, zipOutputStream, countIndex);
                } else {
                    excelReturnMap = ExcelUtils.exportExcel(entities, resultList,
                        fileName, request, response, true, path, i * 200000,
                        false, workbook, zipOutputStream, countIndex);
                }
            }
//           刷新
            if (countFor - countFor.intValue() > 0) {
                if (excelReturnMap != null) {
                    zipOutputStream = getZipOutputStream(excelReturnMap, zipOutputStream);
                    workbook = getWorkbook(excelReturnMap, workbook);
                    countIndex = getCount(excelReturnMap);
                }
                List<DBObject> resultListSmaller = mongoDB.getList(tableName, new Query(),
                    countFor.intValue() * 200000, 200000);
                if (resultListSmaller != null && !resultListSmaller.isEmpty()) {
                    workbook = null;
                    ExcelUtils.exportExcel(entities, resultListSmaller, fileName, request, response,
                        true, path, countFor.intValue() * 200000,
                        false, workbook, zipOutputStream, countIndex);
                }
            }
        } else {
//         十万以内直接输出出去
            fileName = "《" + tableName + "-" + DateUtils.dateToString(new Date(),
                "yyyy-MM-dd HH-mm-ss") + "》.xlsx";
            List<DBObject> byQuery = mongoDB.getList(tableName, new Query(), 0, 100000);
            ExcelUtils.exportExcel(entities, byQuery, fileName, request, response, false,
                path, 0, true, workbook, zipOutputStream, countIndex);
            ExcelUtils.exportExcel(entities, byQuery, fileName, request, response, isZip, path, 0,
                true, workbook, zipOutputStream, countIndex);
        }

        return isZip ? path + File.separator + fileName : null;

excel代码：

    /**
     * @param entities     字段集合，用来拆分表头，和解析时间类型
     * @param results      数据集
     * @param excelName    表名
     * @param request      请求头
     * @param response     请求头
     * @param isZip        是否是zip文件
     * @param path         path是导出的路径
     * @param fooIf        是用来判别是第几个excel的
     * @param hasNext      区别是否还需要拼装zip里的一个excel
     * @param workbook     带有未组合好的excel
     * @param zipOutStream 带有未组合好的压缩流
     * @param countIndex   第几个文件名前缀
     * @return
     * @author liuyu
     * date 2018/8/29
     **/
    public static HashMap<String, Object> exportExcel(List<FieldEO> entities, List<DBObject> results,
                                                      String excelName, HttpServletRequest request,
                                                      HttpServletResponse response, boolean isZip, String path,
                                                      int fooIf, boolean hasNext, Workbook workbook,
                                                      ZipOutputStream zipOutStream, int countIndex) throws IOException {
        HashMap<String, Object> returnMap = new HashMap<>();
        if (isZip) {
            File zipFile = null;
            XSSFWorkbook xssfWorkbook = null;
            ByteArrayOutputStream outputStream = null;
            try {
                if (workbook == null) {
                    xssfWorkbook = new XSSFWorkbook();
                    workbook = new SXSSFWorkbook(xssfWorkbook);
                    workbook.createSheet();
                }

                createSheet(entities, results, workbook, fooIf);
                int lastRowNum = workbook.getSheetAt(0).getLastRowNum();
                if (!hasNext || lastRowNum >= 1000000) {
                    zipFile = new File(path + File.separator + excelName);
                    if (!zipFile.exists()) {
                        if (!zipFile.getParentFile().exists()) {
                            boolean mkdirs = zipFile.getParentFile().mkdirs();
                        }
                        boolean newFile = zipFile.createNewFile();
                    }
                    if (zipOutStream == null) {
                        zipOutStream = new ZipOutputStream(new FileOutputStream(zipFile));
                    }

                    outputStream = new ByteArrayOutputStream();
                    try {
                        workbook.write(outputStream);
                    } catch (IOException e) {
                        e.printStackTrace();
                        logger.error(e.getMessage());
                    } finally {
                        ((SXSSFWorkbook) workbook).dispose();
                        workbook.close();
                    }
                    int count;
                    ByteArrayInputStream zipTemp = null;
                    zipTemp = new ByteArrayInputStream(outputStream.toByteArray());

                    try {
                        ZipEntry zipEntry = new ZipEntry(countIndex + ".xlsx");
                        zipOutStream.putNextEntry(zipEntry);
                        while ((count = zipTemp.read()) != -1) {
                            zipOutStream.write(count);
                        }
                    } catch (IOException e) {
                        e.printStackTrace();
                    } finally {
                        zipOutStream.closeEntry();
                        zipTemp.close();
                        outputStream.close();
                    }
                    countIndex++;
                    //先关闭源头
                    xssfWorkbook = new XSSFWorkbook();
                    workbook = new SXSSFWorkbook(xssfWorkbook);
                    workbook.createSheet();
                }
                if (hasNext) {
                    returnMap.put(Constant.EXECEL_EXPORT_WORK_STREAM, workbook);
                    returnMap.put(Constant.EXECEL_EXPORT_ZIP_STREAM, zipOutStream);
                    returnMap.put(Constant.EXECEL_EXPORT_ZIP_COUNT, countIndex);
                    return returnMap;
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                if (!hasNext && workbook != null) {
                    workbook.close();
                }
                if (!hasNext && zipOutStream != null) {
                    zipOutStream.close();
                }
            }
        } else {
            XSSFWorkbook xssfWorkbook = null;
            ServletOutputStream outputStream = null;
            try {
                xssfWorkbook = new XSSFWorkbook();
                workbook = new SXSSFWorkbook(xssfWorkbook);
                workbook.createSheet();
                createSheet(entities, results, workbook, 0);
                String rtn = ContentDispositionUtils.handler(request, excelName);
                response.setCharacterEncoding("UTF-8");
                response.addHeader("Content-disposition", "attachment;" + rtn);
                outputStream = response.getOutputStream();
                workbook.write(outputStream);
                outputStream.flush();
            } catch (IOException e) {
                e.printStackTrace();
                logger.error("exception:" + e);
            } finally {
                if (outputStream != null) {
                    outputStream.close();
                }
                if (workbook != null) {
                    workbook.close();
                }
                if (xssfWorkbook != null) {
                    xssfWorkbook.close();
                }
            }

        }
        return null;
    }

    /**
     * @param entities 字段集合
     * @param results  结果集
     * @param workbook 工作流
     * @param fooIf    用来判定是否需要生成表头
     * @return
     * @author liuyu
     * date 2018/8/29
     **/
    private static void createSheet(List<FieldEO> entities, List<DBObject> results, Workbook workbook, int fooIf) {
        Sheet sheet;
        sheet = workbook.getSheetAt(0);
//        style.setShrinkToFit(true);
        //设置行内居中
//        style.setAlignment(HorizontalAlignment.CENTER);
        //使用这个可以优化公式。
//        XSSFFormulaEvaluator evaluator = new XSSFFormulaEvaluator((XSSFWorkbook) workbook);
        Map<String, String> fieldEOMap = entities.stream().collect(Collectors.toMap(FieldEO::getFieldNameZH,
            FieldEO::getFieldName));
        Map<String, Integer> fieldNameAndType = entities.stream().collect(Collectors.toMap(FieldEO::getFieldName,
            FieldEO::getFieldType));
        if (fooIf == 0 || fooIf % 1000000 == 0) {
            results.add(0, new BasicDBObject(fieldEOMap));
        }
        List<String> fieldList = new ArrayList<>(fieldEOMap.values());
        int lastRowNum = sheet.getLastRowNum();
        int count = lastRowNum;
        if (count != 0) {
            count = count + 1;
        }
        for (int i = 0; i < results.size(); i++) {
            final DBObject paramsMap = results.get(i);
            Row row = sheet.createRow(count + i);
            if ((lastRowNum + i) == 0) {
                Set entries = paramsMap.toMap().entrySet();
                Iterator iterator = entries.iterator();
                for (int j = 0; iterator.hasNext(); j++) {
                    Map.Entry<String, Object> next = (Map.Entry<String, Object>) iterator.next();
                    //设置单元格的值
                    Object value = next.getValue();
                    Object key = next.getKey();
                    Cell cell = row.createCell(j);
                    cell.setCellValue(key + "(" + value + ")\t");
                }
            } else {
                for (int z = 0; z < fieldList.size(); z++) {
                    Cell cell = row.createCell(z);
                    String key = fieldList.get(z);
                    Object value = paramsMap.get(key);
                    if (fieldNameAndType.get(key).equals(FieldTypeEnum.TIME.getId())) {
                        value = DateUtils.dateToString((Date) value, DateUtils.YYYY_MM_DD_HH_MM_SS_EN);
                        cell.setCellType(CellType.STRING);
                    } else if (fieldNameAndType.get(key).equals(FieldTypeEnum.DATE.getId())) {
                        value = DateUtils.dateToString((Date) value, DateUtils.YYYY_MM_DD_EN);
                        cell.setCellType(CellType.STRING);
                    }
                    if (value == null) {
                        value = "";
                    }
                    cell.setCellValue(value + "");
                }
            }
        }
    }