摘要
另外一篇文章写了如何运用工厂方法加模版方法，文中的例子是依靠记录写了当时的应用，感觉不够清晰，今天再次运用，给出全新的，更加容易理解的常见。
背景
需求是导入白名单，一个excel文件，再完成之后，又需要导入一个通知名单，也是一个excel文件。为了复用代码，利用工厂方法加模版方法重构原来的代码，隔离变与不变，把变化的逻辑由子类继承去实现。
类图
- CommonImportServiceImpl里面全都是通用的逻辑流程，比如分析excel单元格类型，去除重复行，按step值分割数据等等
- 所有业务相关的不同逻辑定义抽象方法，依靠子类去实现，比如selectAllDataInDB，插入数据库等等。
代码
首先是commonimportservice，这个类主要是处理流程已经公共逻辑，代码如下：
package com.puhui.flowplatform.manage.service.impl;
import com.google.common.base.Strings;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.puhui.flowplatform.common.utils.DateUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.PostConstruct;
import java.io.InputStream;
import java.math.BigDecimal;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 2 * @Author: kerry
* 3 * @Date: 2018/8/27 14:53
* 4
*/
public abstract class CommonImportServiceImpl {
private static final Logger log = LoggerFactory.getLogger(CommonImportServiceImpl.class);
protected AtomicInteger importedRow;
protected Integer totalCount;
protected ExecutorService pool;
@PostConstruct
public void initializeThreadPool(){
ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()
.setNameFormat("import-pool-%d").build();
pool = new ThreadPoolExecutor(50, 200,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(1024), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());
}
/**
* 验证数据合法性
*
* @param row
* @param r
*/
protected void validateDataType(Row row, int r) {
String mobile = getCellValue(row.getCell(0));
if (Strings.isNullOrEmpty(mobile)) {
log.error("Failed to import(" + (r + 1) + "row,because the mobile is empty");
}
String idNo = getCellValue(row.getCell(1));
if (Strings.isNullOrEmpty(idNo)) {
log.error("Failed to import(" + (r + 1) + "row,because the idNo is empty");
}
String name = getCellValue(row.getCell(2));
if (Strings.isNullOrEmpty(name)) {
log.error("Failed to import(" + (r + 1) + "row,because the name is empty");
}
String batchNumber = getCellValue(row.getCell(6));
if (Strings.isNullOrEmpty(batchNumber)) {
log.error("Failed to import(" + (r + 1) + "row,because the source is empty");
}
}
/**
* 去除重复记录
* @param sheet
*/
private void removeDuplicateRow(Sheet sheet){
//内部重复则删除
Set<String> set=new HashSet<>();
Iterator<Row> rowIterator = sheet.iterator();
while (rowIterator.hasNext()) {
Row row = rowIterator.next();
String mobile = getCellValue(row.getCell(0));
String idNo = getCellValue(row.getCell(1));
String batchNumber = getCellValue(row.getCell(6));
if (set.contains(mobile+idNo+batchNumber)){
log.info("Remove duplicate row and row number is {} and mobile is {}",row.getRowNum(),mobile);
rowIterator.remove();
}
else{
set.add(mobile+idNo+batchNumber);
}
}
}
/**
* 判断cell值类型
* @param cell
* @return
*/
protected String getCellValue(Cell cell) {
String cellValue = "";
if (cell != null) {
switch (cell.getCellTypeEnum()) {
case NUMERIC:
if (DateUtil.isCellDateFormatted(cell)) {
Date date = cell.getDateCellValue();
cellValue = DateUtils.dateToStr(DateUtils.format(date,DateUtils.FORMAT_8),DateUtils.FORMAT_8);
} else {
double value = cell.getNumericCellValue();
int intValue = (int) value;
cellValue = value - intValue == 0 ? String.valueOf(intValue) : String.valueOf(value);
}
break;
case STRING:
cellValue = cell.getStringCellValue();
break;
case BOOLEAN:
cellValue = String.valueOf(cell.getBooleanCellValue());
break;
case FORMULA:
cellValue = String.valueOf(cell.getCellFormula());
break;
case BLANK:
cellValue = "";
break;
case ERROR:
cellValue = "";
break;
default:
cellValue = cell.toString().trim();
break;
}
}
return cellValue.trim();
}
/**
* 初始化excel的sheet对象
* @param file
* @return
* @throws Exception
*/
private Sheet initializeSheet(MultipartFile file) throws Exception{
String fileName = file.getOriginalFilename();
boolean notNull = false;
if (!fileName.matches("^.+\\.(?i)(xls)$") && !fileName.matches("^.+\\.(?i)(xlsx)$")) {
throw new Exception("The file format is not supported");
}
Workbook wb = null;
boolean isExcel2003 = true;
if (fileName.matches("^.+\\.(?i)(xlsx)$")) {
isExcel2003 = false;
}
try {
InputStream is = file.getInputStream();
if (isExcel2003) {
wb = new HSSFWorkbook(is);
} else {
wb = new XSSFWorkbook(is);
}
}catch (Exception e){
log.error("Failed to create work book",e);
}
return wb.getSheetAt(0);
}
public void importList(MultipartFile file) throws Exception {
importedRow=new AtomicInteger(0);
pool.execute(()->{
Sheet sheet=null;
try {
sheet = initializeSheet(file);
}catch (Exception e){
log.error("Failed to create sheet object {}",e);
}
if(null==sheet){
return;
}
log.info("begin to remove duplicate element {}",DateUtils.getCurrentTime());
removeDuplicateRow(sheet);
log.info("The duplicate remove operation is done {}",DateUtils.getCurrentTime());
totalCount = sheet.getLastRowNum();
int step = 1000;
int totalTasks = (totalCount % step == 0 ? totalCount/step : (totalCount/step + 1));
log.info("The total task number is {}",totalTasks);
Map<String,Long> allDataDbMap=selectAllDataInDB();
for(int i=1;i<=totalCount;i+=step){
//储存每个小段，并发入库
List<Object> subStepList=getSubStepList();
int start=i;
int perCount;
if (totalCount < step) {
perCount = totalCount;
} else {
perCount = (totalCount - start + 1) < step ? (totalCount - start + 1) : step;
}
log.info("The current percount is {}", perCount);
log.info(" start == " + start + " , count" + perCount);
caculateDataEachStep(start,perCount,sheet,subStepList);
processEachStepCurrency(subStepList,allDataDbMap);
}
log.info("All sub thread is started ");
});
}
public Double getImportProgress() {
if (importedRow!=null&&importedRow.get() != 0) {
log.info("get import progress and the imported row is {}", importedRow);
log.info("get import progress and the remaining count is {}", totalCount - importedRow.intValue());
double f1 = new BigDecimal((float) importedRow.intValue() / totalCount).setScale(2, BigDecimal.ROUND_HALF_UP).multiply(new BigDecimal("100")).doubleValue();
if (f1==0){
return 1D;
}
return f1;
}
return 1D;
}
/**
* 并发处理分段数据
* @param subStepList
* @param allDataDbMap
*/
abstract void processEachStepCurrency(List<Object> subStepList,Map<String,Long> allDataDbMap);
/**
* 获取存储数据段容器类型
* @return
*/
abstract List<Object> getSubStepList();
/**
* 拆分数据段
* @param start
* @param perCount
* @param sheet
* @param subStepList
*/
abstract void caculateDataEachStep(int start,int perCount,Sheet sheet,List<Object> subStepList);
/**
* 读取db所有数据
* @return
*/
abstract Map<String,Long> selectAllDataInDB();
}
白名单导入类
package com.puhui.flowplatform.manage.service.impl;
import com.google.common.base.Strings;
import com.puhui.flowplatform.common.dao.goosecard.UserImportWhiteListMapper;
import com.puhui.flowplatform.common.model.goosecard.UserImportWhiteList;
import com.puhui.flowplatform.common.model.goosecard.UserImportWhiteListExample;
import com.puhui.flowplatform.common.utils.CommonUtils;
import com.puhui.flowplatform.common.utils.DateUtils;
import com.puhui.flowplatform.manage.service.WhiteListService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.math.BigDecimal;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
/**
* 2 * @Author: kerry
* 3 * @Date: 2018/8/27 14:53
* 4
*/
@Service
public class WhiteListServiceImpl extends CommonImportServiceImpl implements WhiteListService {
private static final Logger log = LoggerFactory.getLogger(WhiteListServiceImpl.class);
@Autowired
private UserImportWhiteListMapper userImportWhiteListMapper;
@Override
public List getSubStepList() {
return new CopyOnWriteArrayList<UserImportWhiteList>();
}
/**
* 分段数据存入容器
* @param start
* @param perCount
* @param sheet
* @param subStepList
*/
@Override
public void caculateDataEachStep(int start,int perCount,Sheet sheet,List subStepList){
for (int n = 0; n <perCount; n++) {
Row row = sheet.getRow(start + n);
if (row == null) {
log.info("The row is null and maybe deleted by duplicate remove operation {}",start+n);
continue;
}
int r = row.getRowNum();
validateDataType(row, r);
String mobile = getCellValue(row.getCell(0));
String idNo = getCellValue(row.getCell(1));
String name = getCellValue(row.getCell(2));
String amount = getCellValue(row.getCell(3));
String source = getCellValue(row.getCell(4));
String state = getCellValue(row.getCell(5));
if (Strings.isNullOrEmpty(state)){
state="1";
}
String batchNumber = getCellValue(row.getCell(6));;
UserImportWhiteList userImportWhiteList = new UserImportWhiteList();
userImportWhiteList.setMobile(CommonUtils.encrypt(mobile.trim()));
userImportWhiteList.setIdNo(CommonUtils.encrypt(idNo.trim()));
userImportWhiteList.setName(name);
userImportWhiteList.setAmount(new BigDecimal(amount));
userImportWhiteList.setSource(source);
userImportWhiteList.setState(Integer.valueOf(state));
userImportWhiteList.setBatchNumber(batchNumber);
Date date = DateUtils.getCurrentTime();
userImportWhiteList.setCreateTime(date);
userImportWhiteList.setUpdateTime(date);
subStepList.add(userImportWhiteList);
}
}
/***
* 查询所有db数据，判断进行更新or插入
* @return
*/
@Override
public Map<String,Long> selectAllDataInDB(){
Map<String,Long> map=new HashMap<>();
List<UserImportWhiteList> userImportWhiteListList=userImportWhiteListMapper.selectByExample(new UserImportWhiteListExample());
if (CollectionUtils.isNotEmpty(userImportWhiteListList)){
userImportWhiteListList.forEach(userImportWhiteList -> {
String mobile= CommonUtils.decrypt(userImportWhiteList.getMobile());
String idNo=CommonUtils.decrypt(userImportWhiteList.getIdNo());
String batchNo=userImportWhiteList.getBatchNumber();
map.put(mobile+idNo+batchNo,userImportWhiteList.getId());
});
}
return map;
}
/**
* 每个数据段并发入库
* @param subStepList
* @param allDataDbMap
*/
@Override
public void processEachStepCurrency(List subStepList,Map allDataDbMap){
if (CollectionUtils.isNotEmpty(subStepList)) {
pool.execute(()->{
log.info("Start thread to insert sub step and the size is {}", subStepList.size());
subStepList.forEach(object -> {
UserImportWhiteList userImportWhiteList=(UserImportWhiteList)object;
String mobile= CommonUtils.decrypt(userImportWhiteList.getMobile());
String idNo=CommonUtils.decrypt(userImportWhiteList.getIdNo());
String batchNo=userImportWhiteList.getBatchNumber();
String key=mobile+idNo+batchNo;
if(allDataDbMap.containsKey(key)){
//update
userImportWhiteList.setId((Long)allDataDbMap.get(key));
userImportWhiteListMapper.updateByPrimaryKey(userImportWhiteList);
}
else{
//insert
userImportWhiteListMapper.insert(userImportWhiteList);
}
importedRow.getAndIncrement();
});
log.info("Sub thread inserted successfully and size is {}", subStepList.size());
});
}
}
}
另外一个名单导入类：
package com.puhui.flowplatform.manage.service.impl;
import com.google.common.base.Strings;
import com.puhui.flowplatform.common.dao.goosecard.UserTaskMessageReturnGoodsPassEntryMapper;
import com.puhui.flowplatform.common.model.goosecard.UserImportWhiteList;
import com.puhui.flowplatform.common.model.goosecard.UserTaskMessageReturnGoodsPassEntry;
import com.puhui.flowplatform.common.model.goosecard.UserTaskMessageReturnGoodsPassEntryExample;
import com.puhui.flowplatform.common.utils.DateUtils;
import com.puhui.flowplatform.manage.service.NotifyListService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.math.BigDecimal;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
/**
* 2 * @Author: kerry
* 3 * @Date: 2018/8/27 14:53
* 4
*/
@Service
public class NotifyListServiceImpl extends CommonImportServiceImpl implements NotifyListService {
private static final Logger log = LoggerFactory.getLogger(NotifyListServiceImpl.class);
@Autowired
private UserTaskMessageReturnGoodsPassEntryMapper userTaskMessageReturnGoodsPassEntryMapper;
@Override
public List getSubStepList() {
return new CopyOnWriteArrayList<UserImportWhiteList>();
}
/**
* 分段数据存入容器
* @param start
* @param perCount
* @param sheet
* @param subStepList
*/
@Override
public void caculateDataEachStep(int start,int perCount,Sheet sheet,List subStepList){
for (int n = 0; n <perCount; n++) {
Row row = sheet.getRow(start + n);
if (row == null) {
log.info("The row is null and maybe deleted by duplicate remove operation {}",start+n);
continue;
}
int r = row.getRowNum();
String cardNo = getCellValue(row.getCell(0));
String tradeNo = getCellValue(row.getCell(1));
String originalTradeNo = getCellValue(row.getCell(2));
String amount = getCellValue(row.getCell(3));
String stageTime = getCellValue(row.getCell(4));
String state = getCellValue(row.getCell(5));
if (Strings.isNullOrEmpty(state)){
state="1";
}
UserTaskMessageReturnGoodsPassEntry userTaskMessageReturnGoodsPassEntry = new UserTaskMessageReturnGoodsPassEntry();
userTaskMessageReturnGoodsPassEntry.setCardNo(cardNo);
userTaskMessageReturnGoodsPassEntry.setOutTradeNo(tradeNo);
userTaskMessageReturnGoodsPassEntry.setOriginalOutTradeNo(originalTradeNo);
userTaskMessageReturnGoodsPassEntry.setTradeAmount(new BigDecimal(amount));
userTaskMessageReturnGoodsPassEntry.setTrustState(Integer.valueOf(state));
userTaskMessageReturnGoodsPassEntry.setReturnType(4);
userTaskMessageReturnGoodsPassEntry.setStageTime(DateUtils.StringToDate(stageTime,DateUtils.FORMAT_8));
Date date = DateUtils.getCurrentTime();
userTaskMessageReturnGoodsPassEntry.setCreateTime(date);
userTaskMessageReturnGoodsPassEntry.setUpdateTime(date);
subStepList.add(userTaskMessageReturnGoodsPassEntry);
}
}
/***
* 查询所有db数据，判断进行更新or插入
* @return
*/
@Override
public Map<String,Long> selectAllDataInDB(){
Map<String,Long> map=new HashMap<>();
List<UserTaskMessageReturnGoodsPassEntry> userTaskMessageReturnGoodsPassEntryList=userTaskMessageReturnGoodsPassEntryMapper.selectByExample(new UserTaskMessageReturnGoodsPassEntryExample());
if (CollectionUtils.isNotEmpty(userTaskMessageReturnGoodsPassEntryList)){
userTaskMessageReturnGoodsPassEntryList.forEach(userTaskMessageReturnGoodsPassEntry -> {
String outTradeNo= userTaskMessageReturnGoodsPassEntry.getOutTradeNo();
String originalTradeNo=userTaskMessageReturnGoodsPassEntry.getOriginalOutTradeNo();
String key=outTradeNo+originalTradeNo;
map.put(key,userTaskMessageReturnGoodsPassEntry.getId());
});
}
return map;
}
/**
* 每个数据段并发入库
* @param subStepList
* @param allDataDbMap
*/
@Override
public void processEachStepCurrency(List subStepList,Map allDataDbMap){
if (CollectionUtils.isNotEmpty(subStepList)) {
pool.execute(()->{
log.info("Start thread to insert sub step and the size is {}", subStepList.size());
subStepList.forEach(object -> {
UserTaskMessageReturnGoodsPassEntry userTaskMessageReturnGoodsPassEntry=(UserTaskMessageReturnGoodsPassEntry)object;
String outTradeNo= userTaskMessageReturnGoodsPassEntry.getOutTradeNo();
String originalTradeNo=userTaskMessageReturnGoodsPassEntry.getOriginalOutTradeNo();
String key=outTradeNo+originalTradeNo;
if(allDataDbMap.containsKey(key)){
//update
userTaskMessageReturnGoodsPassEntry.setId((Long)allDataDbMap.get(key));
userTaskMessageReturnGoodsPassEntryMapper.updateByPrimaryKey(userTaskMessageReturnGoodsPassEntry);
}
else{
//insert
userTaskMessageReturnGoodsPassEntryMapper.insert(userTaskMessageReturnGoodsPassEntry);
}
importedRow.getAndIncrement();
});
log.info("Sub thread inserted successfully and size is {}", subStepList.size());
});
}
}
}