数据迁移--分组处理

表数据迁移案例：

分组工具代码：

/**
	 * 功能描述: <br>
	 * 将list进行分割
	 *
	 * @param source
	 * @param groupSize
	 * @return
	 */
	public static <T> List<T>[] split(List<T> source, int groupSize) {
		if (CollectionUtils.isEmpty(source)) {
			return null;
		}
		int size = source.size();
		groupSize = groupSize < 1 ? 1 : groupSize;
		groupSize = groupSize > size ? size : groupSize;
		List<?>[] listArr = new List<?>[groupSize];
		int sizeOfGroup = size / groupSize;
		int groupNum = -1;
		List<T> temp = null;
		for (int i = 0; i < size; i++) {
			if ((groupNum + 1) < groupSize && (i) % sizeOfGroup == 0) {
				groupNum++;
				temp = new ArrayList<T>();
				listArr[groupNum] = temp;
			}
			temp.add(source.get(i));
		}
		return (List<T>[]) listArr;
	}

多线程迁移数据：

import com.xxl.job.core.biz.model.ReturnT;
import com.xxl.job.core.handler.IJobHandler;
import com.xxl.job.core.handler.annotation.JobHandler;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.task.TaskRejectedException;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * @author watson
 * @Description: 回款历史数据导入
 * @date 18/6/21上午10:29
 */
@JobHandler(value = "hisRedeemImportJobHandler")
@Component
public class HisRedeemImportJobHandler extends IJobHandler {
    private static final Logger LOGGER = LoggerFactory.getLogger(HisRedeemImportJobHandler.class);
    public static String[] TABLE_NAME_ARR = {"0000", "0004", "0008", "0012", "0016", "0020", "0024", "0028", "0032", "0036", "0040", "0044", "0048", "0052", "0056", "0060"};

    @Resource
    private RedeemManagerService redeemManagerService;

    @Resource
    private TzSubjectRedeemPlanService tzSubjectRedeemPlanService;

    @Resource(name = "tempTaskExecutor")
    private ThreadPoolTaskExecutor taskExecutor;

    private static final int GROUP_SIZE = 6;
    private static Integer pageSize = 1000;
    private ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() + 1, 20, 5, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());


    /**
     * {tableName:"0000,0004,0008","timeSlice":"","pageSize":"5000"}
     *
     * @param param
     * @return
     * @throws Exception
     */
    @Override
    public ReturnT<String> execute(String param) throws Exception {
        LOGGER.info("开始执行导入回款历史数据job, 参数：{}", param);
        JSONObject jo = JSONObject.parseObject(param);
        validate(jo);
        String time = jo.getString("timeSlice");
        String pageSizeAuto = jo.getString("pageSize");
        String inputTableNames = jo.getString("tableName");

        String[] inputTableNameArr = inputTableNames.split(",");
        TABLE_NAME_ARR = inputTableNameArr;

        if (StringUtil.isNotBlank(pageSizeAuto)) {
            pageSize = Integer.valueOf(pageSizeAuto);
        }

        // 获取指定时间段的放款数据
        TimeSlice ts = JobUtils.parseParam(time);
        Assert.assertNotNull("非法参数:" + time, ts);
        LOGGER.info("指定参数，开始获取指定时间段的代扣数据:{}", param);

        for (String tableName : TABLE_NAME_ARR) {
            LOGGER.info(" the 表名序号: {} 加入线程池.", tableName);
            TableTask tableTask = new TableTask(tableName, ts);
            threadPoolExecutor.submit(tableTask);
        }

        return SUCCESS;

    }

    /**
     * 数据解析
     *
     * @param
     */
    private void analyseRedeemData(String tableName, TimeSlice ts) {
        TzSubjectRedeemPlanQuery query = new TzSubjectRedeemPlanQuery();
        query.setCreateTimeFrom(ts.getStart());
        query.setCreateTimeTo(ts.getEnd());
        query.setLimit(pageSize);
        Long startTime = System.currentTimeMillis();


        int totalCount = tzSubjectRedeemPlanService.countRecordByConditionAndPageParam(query, tableName);
        int totalPage = (totalCount / pageSize) + 1;
        LOGGER.info("== 表序号:{} count 总回款统计: {} ,总页数:" + totalPage + "==", tableName, totalCount);
        if (totalCount == 0) {
            return;
        }

        List<Future> futures = Lists.newArrayList();
        while (true) {
            //分组处理
            List<TzSubjectRedeemPlan> list = tzSubjectRedeemPlanService.getAllRecordByConditionAndPageParam(query, tableName);
            List<TzSubjectRedeemPlan>[] group = null;
            if (list.size() > GROUP_SIZE) {
                group = ListUtil.split(list, GROUP_SIZE);
            }

            if (group == null) {
                redeemManagerService.saveHisRedeemDetails(list);
            } else {
                Arrays.stream(group).forEach(x -> {
                    try {
                        Future<Long> future = taskExecutor.submit(new Callable<Long>() {
                            @Override
                            public Long call() {
                                redeemManagerService.saveHisRedeemDetails(x);
                                return Thread.currentThread().getId();
                            }
                        });

                        futures.add(future);
                    } catch (TaskRejectedException e) {
                        LOGGER.info("队列已满，改为同步！");
                        redeemManagerService.saveHisRedeemDetails(x);
                    }
                });
            }

            if (list.size() < pageSize) {
                break;
            } else {
                query.setOffset(query.getOffset() + pageSize);
            }
        }

        //等待结果
        Long threadId = null;
        for (Future future : futures) {
            try {
                threadId = (Long) future.get();
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
            LOGGER.info("== 线程ID: {} 执行完毕!==", threadId);
        }

        Long endTime = System.currentTimeMillis();
        LOGGER.info("==时间范围: {}--{} 的回款数据迁移任务执行完毕! 总耗时: " + (endTime - startTime) + " ms ==", DateUtil.getChineseDateString(ts.getStart()), DateUtil.getChineseDateString(ts.getEnd()));
    }

    class TableTask implements Runnable {
        private String tableName;
        private TimeSlice ts;

        public TableTask(String tableName, TimeSlice ts) {
            this.tableName = tableName;
            this.ts = ts;
        }

        @Override
        public void run() {
            try {
                analyseRedeemData(tableName, ts);
            } catch (Exception e) {
                LOGGER.error("表序号:{} 数据迁移异常:{}", tableName, e);
            }
        }
    }

    private void validate(JSONObject jo) {
        String time = jo.getString("timeSlice");
        Assert.assertNotNull("时间不可为空", time);
        String tableNames = jo.getString("tableName");
        Assert.assertNotNull("表名数组不可为空", tableNames);
    }
}