POI读取excle多线程批量插入到数据库

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_33229669/article/details/84258714

开发环境

• JDK 1.6
• POI 3.9
• 框架 struts2 spring mybatis

model

public class ConsultGovDataModel extends ModelBase {
    private static final long serialVersionUID = 1L;
    /**
     * 政务咨询数据主键
     */
    private String govdataId;
    /**
     * 项目名称
     */
    private String projectName;
    /**
     * 项目地点
     */
    private String projectAddress;
    /**
     *行业分类
     */
    private String projectIndustry;
    /**
     * 发布日期
     */
    private String publishDate;
    /**
     * 行业
     */
    private String industry;
    /**
     * 产品
     */
    private String product;
    /**
     * 导入日期
     */
    private String importDate;
    /**
     * 创建时间开始 检索条件
     */
    private String start_time;
    /**
     * 创建时间结束 检索条件
     */
    private String end_time;
	
     Get .... set ...

}

Action

public class ConsultGovDataAction extends ActionBase implements ModelDriven<ConsultGovDataModel> {

    private static final long serialVersionUID = 1L;

    /**
     * 上传文件
     */
    private File upload;
    /**
     * 上传文件名称
     */
    private String uploadFileName;
    /**
     * 上传文件的mimeType类型
     */
    private String uploadContentType;

    ConsultGovDataService consultGovDataService;

    ConsultGovDataModel model = new ConsultGovDataModel();
     
Get set ... 

@Override
    public ConsultGovDataModel getModel() {
      return model;
    }


/**
 * 方法名: importExcle
 * 方法描述: 导入政务咨询的excle数据
 * 参数 []
 * 返回类型 java.lang.String
 * @throws
 */
public String importExcle() throws Exception {
    String type = getRequest().getParameter("type");
    String fileEnd="xls";
    //获取文件名,判断是xls还是xlsx
    Workbook workbook=null;
    ArrayList<ConsultGovDataModel> list = new ArrayList<ConsultGovDataModel>();
    if (StringUtils.isNotBlank(uploadFileName)) {
        if (uploadFileName.endsWith(fileEnd)) {
            //03版本的xls
            workbook= new HSSFWorkbook(new FileInputStream(upload));
        }else {
            //07版本的xlsx
            workbook= new XSSFWorkbook(new FileInputStream(upload));
        }
        //获取一共有几个sheet
        int numberOfSheets = workbook.getNumberOfSheets();
        for (int i=0;i<numberOfSheets;i++) {
            //遍历循环sheet 导入 获取每一个sheet
            Sheet sheet = workbook.getSheetAt(i);
            //遍历sheet中的每一行
            for (Row row : sheet) {
                if (row.getRowNum() == 0) {
                    //跳过每一个sheet第一行的标题行
                    continue;
                }
                //读取到末尾了或者到空行了,跳过空行. (当读取到最后一行时,会结束循环  )
                if (row.getCell(0) == null || StringUtils.isBlank(row.getCell(0).getStringCellValue())) {
                    // 读取到末尾了, 保存当前sheet页的数据到数据库
                    consultGovDataService.saveBatch(list);
                    list.removeAll(list);
                    break;
                }
                ConsultGovDataModel govDataModel = new ConsultGovDataModel();
                if (row.getCell(0)!=null) {
                    govDataModel.setProjectName(row.getCell(0).getStringCellValue());
                }
                if (row.getCell(1)!=null) {
                    govDataModel.setProjectAddress(row.getCell(1).getStringCellValue());
                }
                if (row.getCell(2)!=null) {
                    govDataModel.setProjectIndustry(row.getCell(2).getStringCellValue());
                }
                if (row.getCell(3)!=null) {
                    govDataModel.setPublishDate(row.getCell(3).getStringCellValue());
                }
                //转换导入的日期到时分秒
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                String format = sdf.format(new Date());
                govDataModel.setImportDate(format);
                if ("product".equals(type)) {
                    //选择的是产品,那么就把sheet名,存入产品列中
                    govDataModel.setProduct(sheet.getSheetName());
                } else if ("industry".equals(type)) {
                    //选择的是行业,那么就把sheet名, 存入行业列中
                    govDataModel.setIndustry(sheet.getSheetName());
                }
                list.add(govDataModel);
            }
            //一个sheet读取完毕了, 放入dao层进行插入
            consultGovDataService.saveBatch(list);
            list.removeAll(list);
        }
    }
    return "success";
}

Service

public class ConsultGovDataServiceImpl implements ConsultGovDataService {
    ConsultGovdataDao consultGovdataDao;

    public ConsultGovdataDao getConsultGovdataDao() {
        return consultGovdataDao;
    }

    public void setConsultGovdataDao(ConsultGovdataDao consultGovdataDao) {
        this.consultGovdataDao = consultGovdataDao;
    }

       @Override
    public void saveBatch(ArrayList<ConsultGovDataModel> list) throws InterruptedException{
        //一个线程处理200条数据
        int count = 200;
        //数据集合大小
        int listSize = list.size();
        //开启的线程数
        int runSize = (listSize / count) + 1;
        //存放每个线程的执行数据
        List<ConsultGovDataModel> newlist = null;

        //创建一个线程池，数量和开启线程的数量一样
        //Executors 的写法
       // ExecutorService executor = Executors.newFixedThreadPool(runSize);
        
        //ThreadPoolExecutor的写法
        ThreadPoolExecutor executor = new ThreadPoolExecutor(runSize, runSize, 1,
                TimeUnit. SECONDS, new ArrayBlockingQueue<Runnable>(3),
                new ThreadPoolExecutor.DiscardOldestPolicy());
                
        //创建两个个计数器
        CountDownLatch begin = new CountDownLatch(1);
        CountDownLatch end = new CountDownLatch(runSize);
        //循环创建线程
        for (int i = 0; i < runSize; i++) {
            //计算每个线程执行的数据
            if ((i + 1) == runSize) {
                int startIndex = (i * count);
                int endIndex = list.size();
                newlist = list.subList(startIndex, endIndex);
            } else {
                int startIndex = (i * count);
                int endIndex = (i + 1) * count;
                newlist = list.subList(startIndex, endIndex);
            }
            //线程类
            ImportThread mythead = new ImportThread(newlist, begin, end);
            //这里执行线程的方式是调用线程池里的executor.execute(mythead)方法。
            executor.execute(mythead);
        }
        begin.countDown();
        end.await();
        //执行完关闭线程池
        executor.shutdown();
        //consultGovdataDao.saveBatch(list);
    }

Dao

public interface ConsultGovdataDao {

    void saveBatch(List<ConsultGovDataModel> list);
}

Mapper

<!--批量插入导入的数据-->
<insert id="saveBatch" parameterType="java.util.List">
    insert all
    <foreach item="item" collection="list" separator="">
        INTO CONSULT_GOVDATA
        (
        PROJECT_NAME, PROJECT_ADDRESS, PROJECT_INDUSTRY, PUBLISH_DATE, INDUSTRY,
        PRODUCT,IMPORT_DATE
        )
        values
        (
        #{item.projectName,jdbcType=VARCHAR},
        #{item.projectAddress,jdbcType=VARCHAR},
        #{item.projectIndustry,jdbcType=VARCHAR},
        to_date(#{item.publishDate,jdbcType=DATE},'yyyy-mm-dd'),
        #{item.industry,jdbcType=VARCHAR},
        #{item.product,jdbcType=VARCHAR},
        to_date(#{item.importDate,jdbcType=DATE},'yyyy-mm-dd hh24:mi:ss')
        )
    </foreach>
    select * from dual
</insert>

Thread类

public class ImportThread implements Runnable {
    public ImportThread() {
    }

    ConsultGovdataDao consultGovdataDao;

    public ConsultGovdataDao getConsultGovdataDao() {
        return consultGovdataDao;
    }

    public void setConsultGovdataDao(ConsultGovdataDao consultGovdataDao) {
        this.consultGovdataDao = consultGovdataDao;
    }

    private List<ConsultGovDataModel> list;
    private CountDownLatch begin;
    private CountDownLatch end;

    /**
     * 方法名: ImportThread
     * 方法描述: 创建个构造函数初始化 list,和其他用到的参数
     * @throws
     */
    public ImportThread(List<ConsultGovDataModel> list, CountDownLatch begin, CountDownLatch end) {
        this.list = list;
        this.begin = begin;
        this.end = end;
    }

    @Override
    public void run() {
        try {
            //执行完让线程直接进入等待
            ConsultGovdataDao consultGovdataDao = (ConsultGovdataDao) ServiceFactory.getService("consultGovdataDao");
            consultGovdataDao.saveBatch(list);
            begin.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            //这里要主要了，当一个线程执行完 了计数要减一不然这个线程会被一直挂起
            //这个方法就是直接把计数器减一的
            end.countDown();
        }
    }
}

测试结果

经测试, 2万条数据插入, 批量插入200到300一次最佳.
原来要四分钟, 现在为41秒.

需要优化的地方

创建线程池的方式为Executors.newFixedThreadPool
需要改进为
ThreadPoolExecutor 的方式创建线程
20181120已经优化