前言
在一个业务中，需要将数据库的一张日志表导出到excel中做统计分析归类，由于单表的数据量特别大，发现在最终导出excel的时候，由于数量太大，导出速度特别慢，想了一些办法，不管使用何种API，单线程始终是操作的瓶颈，因此最终考虑使用多线程进行改善
总体思路：
1、数据总量分段
2、每个线程处理不同分段的数据
3、提交线程池
下面来看具体的代码，为测试方便，这里直接使用一个测试接口进行调用，
1、控制器
/**
* 导出系统日志信息-V2测试
*
* @return
*/
@GetMapping("/log-export/v2")
@ApiOperation(value = "导出系统日志信息V2", notes = "导出系统日志信息V2", produces = "application/json")
public void exportSysLogV2(@RequestParam(name = "userName", required = false) String userName,
@RequestParam(name = "startDate", required = false) String startDate,
@RequestParam(name = "endDate", required = false) String endDate,
@RequestParam(name = "type", required = false) String type,
HttpServletResponse response) {
operLogService.exportSysLogV2(userName, startDate, endDate, type,response);
}
2、业务实现类
@Override
public void exportSysLogV2(String userName, String startDate, String endDate, String type,HttpServletResponse response) {
List<OperLogVO> exportLists = workflowTaskMapper.getOperlogList(userName, startDate, endDate);
List<OperSysVO> handleLists = handleDbLists(exportLists);
List<Map<String, Object>> excelist = ExcelLocalUtils.convertList2Map(handleLists);
//导出
/*XSSFWorkbook xb = new XSSFWorkbook();
ExcelLocalUtils.exportExcel(xb, excelist, EXPORT_TITLES, EXPORT_COLNUMS, SHEET_NAME, 0);
ExcelLocalUtils.Out(xb, response, EXCEL_NAME);*/
try{
MultiWrite.exec(1000,8,excelist,type);
}catch (Exception e){
e.printStackTrace();
}
}
里面的入参可以根据实际的业务需求进行使用，
3、MultiWrite方法
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class MultiWrite {
public static void exec(int max,int threadMax,List<Map<String,Object>> datas,String type) throws Exception {
XSSFWorkbookWrapper workbookWrapper = new XSSFWorkbookWrapper();
workbookWrapper.initTitile(type);
//封装数据库的查询结果
final List <List <String>> values = WriteDataUtils.getValuesOut(datas,max,type);
//将获取到的结果根据线程标记进行分组
List <List <List <String>>> item = WriteDataUtils.groupData(values, threadMax);
Executor executor = new ThreadPoolExecutor(threadMax, threadMax,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>());
AtomicInteger integer = new AtomicInteger(0);
for (int i = 0; i < item.size(); i++) {
final List <List <String>> lists = item.get(i);
int finalI = i * item.get(0).size() + 1;
Runnable runnable = new Runnable() {
@Override
public void run() {
System.out.println(Thread.currentThread().getName() + "执行");
WritePOIUtils utils = new WritePOIUtils();
utils.setWorkbookData(workbookWrapper.getWorkbook(),lists, finalI);
integer.addAndGet(1);
}
};
executor.execute(runnable);
}
while (integer.get() < threadMax) {
}
((ThreadPoolExecutor) executor).shutdown();
System.out.println("数据执行完毕");
WritePOIUtils.writeFile(workbookWrapper.getWorkbook());
System.out.println("执行完毕");
}
}
在本方法中，使用到了几个工具类，下面直接列举出来，
3.1 XSSFWorkbookWrapper
/**
* 创建初始化信息
*/
public class XSSFWorkbookWrapper {
private XSSFWorkbook workbook;
private XSSFSheet sheetAt;
public XSSFWorkbookWrapper() {
workbook = new XSSFWorkbook();
sheetAt = workbook.createSheet("main");
}
//初始化表头
public void initTitile(String type) {
XSSFRow row = sheetAt.createRow(0);
AtomicInteger index = new AtomicInteger(0);
String[] head = null;
if(StringUtils.isNotBlank(type)){
head = new TitlesColumns().getInitTitles().get(type);
}
for(int i=0; i<head.length; i++){
row.createCell(i).setCellValue(head [i]);
}
}
public XSSFWorkbook getWorkbook() {
return workbook;
}
}
3.2 WriteDataUtils 封装查询结果，获取分组数据
public class WriteDataUtils {
public static List<List<String>> getValuesOut(List<Map<String, Object>> lists, int max, String type) {
List<List<String>> rest = new ArrayList<List<String>>();
for (int i = 0; i < lists.size(); i++) {
if (StringUtils.isNotBlank(type)) {
String[] columns = new TitlesColumns().getInitColumns().get(type + "_V");
List<String> item = new ArrayList<String>();
item.add(lists.get(i).get(columns[0]) + "");
item.add(lists.get(i).get(columns[1]) + "");
item.add(lists.get(i).get(columns[2]) + "");
item.add(lists.get(i).get(columns[3]) + "");
item.add(lists.get(i).get(columns[4]) + "");
item.add(lists.get(i).get(columns[5]) + "");
rest.add(item);
}
}
return rest;
}
/**
* 将数据进行分组
*
* @param data
* @param groupNum
* @return
*/
public static <T> List<List<List<T>>> groupData(List<List<T>> data, Integer groupNum) {
int all = data.size();
int other = all % groupNum;
int groupItemNum = all / groupNum;
List<List<List<T>>> runList = new ArrayList<List<List<T>>>();
while (data == null || data.size() > 0) {
if (data.size() <= other + groupItemNum) {
List<List<T>> lists = data.subList(0, data.size());
List<List<T>> item = new ArrayList<List<T>>();
item.addAll(lists);
runList.add(item);
data.removeAll(item);
} else {
List<List<T>> lists = data.subList(0, groupItemNum);
List<List<T>> item = new ArrayList<List<T>>();
item.addAll(lists);
runList.add(item);
data.removeAll(item);
}
}
return runList;
}
}
3.4 WritePOIUtils 将数据写入到具体的excel中
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
public class WritePOIUtils {
private static Object LOCK = new Object();
public static synchronized XSSFRow getRow(XSSFSheet sheetAt,Integer i) {
return sheetAt.getRow(i) == null ? sheetAt.createRow(i) : sheetAt.getRow(i);
}
public static XSSFCell getCell(XSSFRow row,Integer j) {
return row.getCell(j) == null ? row.createCell(j) : row.getCell(j);
}
public void setWorkbookData(XSSFWorkbook workbook,
List<List<String>> data,
Integer startNum) {
XSSFSheet sheetAt = workbook.getSheetAt(0);
Integer endNum = data.size() + startNum;
Integer index = 0;
for (int i = startNum; i < endNum; i++) {
XSSFRow row = getRow(sheetAt,i);
List <String> values = data.get(index);
for (int j = 0; j < values.size(); j++) {
XSSFCell cell = getCell(row,j);
/*cell.setCellValue(test.get(j));*/
String s = values.get(j);
synchronized (LOCK) {
cell.setCellValue(s);
}
}
index++;
}
}
public static void writeFile (XSSFWorkbook workbook) throws IOException {
FileOutputStream out = null;
try {
out = new FileOutputStream("C:\\logs\\multi.xlsx");
//向d://test.xls中写数据
out.flush();
workbook.write(out);
out.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (out != null) {
out.close();
}
}
}
}
3.5 TitlesColumns 表头初始化工具类
import java.util.HashMap;
import java.util.Map;
public class TitlesColumns {
public static final String[] EXPORT_TITLES = {
"操作人员", "登录IP", "模块标题", "业务类型", "操作状态", "操作时间"
};
public static final String[] EXPORT_COLNUMS = {
"operName", "operIp", "title", "businessType", "status", "operTime"
};
public static Map<String,String[]> initTitles = new HashMap<>() ;
public static Map<String,String[]> initColumns = new HashMap<>();
public TitlesColumns(){
initTitles = new HashMap<>();
initTitles.put("001",EXPORT_TITLES);
initColumns.put("001_V",EXPORT_COLNUMS);
}
public Map<String, String[]> getInitTitles() {
return initTitles;
}
public Map<String, String[]> getInitColumns() {
return initColumns;
}
}
启动程序，通过接口调用一下，观察执行效果，可以看到，5000多条数据，大概花了2秒的时间全部导出来
下面仍然使用这个程序，我们使用单线程的方式做一下导出，看一下效果，
4.1 修改业务实现类
@Override
public void exportSysLogV2(String userName, String startDate, String endDate, String type,HttpServletResponse response) {
List<OperLogVO> exportLists = workflowTaskMapper.getOperlogList(userName, startDate, endDate);
List<OperSysVO> handleLists = handleDbLists(exportLists);
List<Map<String, Object>> excelist = ExcelLocalUtils.convertList2Map(handleLists);
//导出
XSSFWorkbook xb = new XSSFWorkbook();
ExcelLocalUtils.exportExcel(xb, excelist, EXPORT_TITLES, EXPORT_COLNUMS, SHEET_NAME, 0);
ExcelLocalUtils.Out(xb, response, EXCEL_NAME);
/*try{
MultiWrite.exec(1000,8,excelist,type);
}catch (Exception e){
e.printStackTrace();
}*/
}
本方法中要使用到的工具类直接列出来
import com.google.common.base.Charsets;
import com.sx.common.utils.StringUtils;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import javax.servlet.http.HttpServletResponse;
import java.beans.BeanInfo;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ExcelLocalUtils {
public static void exportExcel(XSSFWorkbook xb, List<Map<String, Object>> excelist, String[] fieldTitle, String[] tableColnum, String sheetName, int sheetNumber) {
creatTableSheet(xb, excelist, fieldTitle, tableColnum, sheetName);
}
public static Map<String,String> getSplitMap(int totalSize,int threaCount){
Map<String, String> numMap = new HashMap<>();
if (totalSize % threaCount == 0) {
//能够整除,直接切分数据
//将每个线程要处理的数据进行封装
int meanValue = totalSize / threaCount;
int start = 0;
int end = 0;
for (int i = 1; i <= threaCount; i++) {
if (i == 1) {
start = 1;
end = meanValue;
numMap.put(String.valueOf(i), start + ":" + end);
} else {
start = end + 1;
end = meanValue * i;
numMap.put(String.valueOf(i), start + ":" + end);
}
}
System.out.println(numMap);
}else {
//如果不能整除
//先找出能整除的分量，剩下的继续组装
int lastValue = totalSize%threaCount;
int meanValue = totalSize/threaCount;
int start = 0;
int end = 0;
for(int i=1;i<=threaCount;i++){
if (i == 1) {
start = 1;
end = meanValue;
numMap.put(String.valueOf(i), start + ":" + end);
} else if(i>1 && i<threaCount){
start = end + 1;
end = meanValue * i;
numMap.put(String.valueOf(i), start + ":" + end);
} else {
start = end + 1;
end = meanValue * i + lastValue;
numMap.put(String.valueOf(i), start + ":" + end);
}
}
System.out.println(numMap);
}
return numMap;
}
public static void main(String[] args) {
getSplitMap(1000,9);
}
public static void creatTableSheet(XSSFWorkbook xb, List<Map<String, Object>> excelist, String[] fieldTitle, String[] tableColnum, String sheetName) {
Sheet sheet = xb.createSheet(sheetName);
Row row = sheet.createRow(0);
Cell cell = null;
if (StringUtils.isNotEmpty(fieldTitle) && StringUtils.isNotEmpty(tableColnum)) {
int j;
for (j = 0; j < fieldTitle.length; ++j) {
cell = row.createCell(j);
CellStyle styles = xb.createCellStyle();
Font fonts = xb.createFont();
fonts.setFontName("黑体");
styles.setFont(fonts);
cell.setCellValue(fieldTitle[j]);
cell.setCellStyle(styles);
sheet.autoSizeColumn(j, true);
}
for (j = 0; j < excelist.size(); ++j) {
row = sheet.createRow(j + 1);
for (int k = 0; k < fieldTitle.length; ++k) {
cell = row.createCell(k);
if (StringUtils.isNotNull(((Map) excelist.get(j)).get(tableColnum[k]))) {
cell.setCellValue(((Map) excelist.get(j)).get(tableColnum[k]).toString());
}
}
}
}
}
public static void Out(XSSFWorkbook xb, HttpServletResponse response, String excelName) {
try {
OutputStream toClient = response.getOutputStream();
response.reset();
response.addHeader("Content-Disposition", "attachment;filename=" + new String(excelName.getBytes("utf-8"), Charsets.ISO_8859_1));
response.setContentType("application/octet-stream");
xb.write(toClient);
toClient.flush();
toClient.close();
} catch (FileNotFoundException var4) {
var4.printStackTrace();
} catch (IOException var5) {
var5.printStackTrace();
}
}
public static List<Map<String, Object>> convertList2Map(List list) {
List<Map<String, Object>> excelist = new ArrayList<>();
if (list != null && !list.isEmpty()) {
list.forEach((l) -> {
try {
Map<String, Object> map = objectToMap(l);
excelist.add(map);
} catch (Exception e) {
e.printStackTrace();
}
});
}
return excelist;
}
public static Map<String, Object> objectToMap(Object obj) throws Exception {
if (obj == null) {
return null;
} else {
Map<String, Object> map = new HashMap();
BeanInfo beanInfo = Introspector.getBeanInfo(obj.getClass());
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
PropertyDescriptor[] var4 = propertyDescriptors;
int var5 = propertyDescriptors.length;
for (int var6 = 0; var6 < var5; ++var6) {
PropertyDescriptor property = var4[var6];
String key = property.getName();
if (key.compareToIgnoreCase("class") != 0) {
Method getter = property.getReadMethod();
Object value = getter != null ? getter.invoke(obj) : null;
map.put(key, value);
}
}
return map;
}
}
}
下面测试一下效果，直接调用接口即可，
由于这里数据量还不算大，时间相差不算明显，我本地测试了5万多条数据的一张表，结果相差了5秒多，可见一斑，本篇主要讲述了一下使用多线程导出excel多数据的案例，希望对看到的伙伴有用，最后感谢观看！
