版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/eumenides_/article/details/82587412
分表背景
我们项目有个很重要的功能就是群发短信，我接盘之前是一个大佬负责的(现已离职)，业务抱怨短信日志明细查询太慢，我接手后看了下大佬居然用mysql一张sms_log表存的日志，随着业务的飞速发展，每天记的日志越来越多，我去线上mysql看了下，以及快2000w数据了，而且日志表的字段又很多，看得我隐隐蛋疼不以。我以前记得以前看到过mysql单表数据最好不要超过500w，一般维持在300w以下，具体哪里看到的已经不知道了。。。
分表计划
- 选用nosql数据库mongodb按月份保存日志，每个月就是一个集合，每条日志就是一个文档
- 由于不定时的会有平台返回的状态报告更新，所以为了避免频繁crud同一个集合就采用读写分离的思想，同一个月份创建两个集合，预日志和全量日志，主要逻辑是:
保存日志到预日志中->接收状态报告查询预日志->更新预日志->保存全量日志->删除预日志
这样做就能保证预日志集合稳步减少，全量日志集合稳步增加 - 数据同步，做好这些事还需要把mysql里面的老数据同步到mongodb里
这里不具体展示数据同步的逻辑和代码以及如何保存日志，本篇只介绍一下我如何根据按月份分表后分页查询的。
分表分页查询规则
- 由于从单表mysql迁移到多集合的mongodb所以原来支持的自定义字段排序无法支持了，当然非要支持的话也不是办不到，不过好在业务上主要也只是按照短信的创建时间倒叙而已，所以这块可以写死的sql里
- 分页查询是核心功能，不可或缺，但是由于检索条件的创建时间，所以时间参数会给分页查询带来较大难度。这里目前限制只能查同一年的数据，不支持跨年查询，尽管如此也会产生同月、临月、跨月数据3种情况，因此具体查询逻辑需要先分析时间参数在做具体查询和拼接。
- 同事提到过可以把所有数据查好，放到内存里排序好然后截取部分数据，这个办法一开始就被我否定了，这样数据量太大，处理速度太慢，及占内存，随着翻页越大数据量越大，很可能内存溢出，如果我没记错mycat就是这么做的。那么我是怎么做的呢，我是根据页码来定位数据处于哪个集合中，如果该集合取数后不够一页的份量那么继续从下个集合再取，当然极端情况可能多个集合都取不够，需要把按照条件符合的集合依次遍历下去取到够为止或者取到完。
既然逻辑我们理清楚了，就开始看下代码吧！
public Map<String, Object> listByParamsFromMongodb(SendmsgLog sendmsgLog, Integer page, Integer rows, String beginTime, String endTime,Integer logType) throws ParseException {
Map<String, Object> resultMap = new HashMap<>();
List<SendmsgLog> sendmsgLogList = new ArrayList<>();
int total = 0;
if (StringUtils.isBlank(beginTime) || StringUtils.isBlank(endTime) || logType == null){
resultMap.put("sendmsgLogList",sendmsgLogList);
resultMap.put("total",total);
return resultMap;
}
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date startDate = sdf.parse(beginTime);
Date endDate = sdf.parse(endTime);
Calendar cs = Calendar.getInstance();
Calendar ce = Calendar.getInstance();
cs.setTime(startDate);
ce.setTime(endDate);
int startYear = cs.get(Calendar.YEAR);
int startMonth = cs.get(Calendar.MONTH) + 1;
int endYear = ce.get(Calendar.YEAR);
int endMonth = ce.get(Calendar.MONTH) + 1;
int difYear = endYear - startYear;
int difMonth = endMonth - startMonth;
String baseCollectionName = logType == 0 ? MarketConstant.WAIT_STATUS_REPORT_COLLECTION_NAME : MarketConstant.SEND_SMS_LOG_COLLECTION_NAME;
if(difYear == 0 && difMonth == 0){
//同年同月份查询
String collectionName = startYear + "_" + baseCollectionName + "_" + (startMonth < 10 ? "0" + startMonth : startMonth);
sendmsgLogList = searchSameMonthFromMongoDb(sendmsgLog,(page - 1) * rows,rows,beginTime,endTime,collectionName);
total = getCollectionEleCount(sendmsgLog,beginTime,endTime,collectionName);
}else if((difYear == 0 && difMonth == 1) || (difYear == 1 && difMonth == -11)){
//同年相邻月份查询 || 跨年相邻月份查询
String startCollectionName = startYear + "_" + baseCollectionName + "_" + (startMonth < 10 ? "0" + startMonth : startMonth);
String endCollectionName = endYear + "_" + baseCollectionName + "_" + (endMonth < 10 ? "0" + endMonth : endMonth);
sendmsgLogList = searchAdjoinMonthFromMongoDb(sendmsgLog,page,rows,beginTime,endTime,startCollectionName,endCollectionName);
int startTotal = getCollectionEleCount(sendmsgLog,beginTime,endTime,startCollectionName);
int endTotal = getCollectionEleCount(sendmsgLog,beginTime,endTime,endCollectionName);
total = startTotal + endTotal;
}else if (difYear == 0 && difMonth > 1){
//同年跨多月份查询
sendmsgLogList = searchMultiMonthFromMongoDb(sendmsgLog,page,rows,beginTime,endTime,startYear,startMonth,endMonth,baseCollectionName);
String currentCollectionName;
for (int i = endMonth; i>=startMonth; i--){
currentCollectionName = startYear + "_" + baseCollectionName + "_" + (i < 10 ? "0" + i : i);
total += getCollectionEleCount(sendmsgLog,beginTime,endTime,currentCollectionName);
}
}else{
//跨年跨多月份查询[暂不支持,返回空结果]
resultMap.put("sendmsgLogList",sendmsgLogList);
resultMap.put("total",total);
return resultMap;
}
resultMap.put("sendmsgLogList",sendmsgLogList);
resultMap.put("total",total);
return resultMap;
}
/**
* 查询同年同月数据
* @param sendmsgLog
* @param skipCount
* @param rows
* @param beginTime
* @param endTime
* @param collectionName
* @return
* @throws ParseException
*/
public List<SendmsgLog> searchSameMonthFromMongoDb(SendmsgLog sendmsgLog, Integer skipCount, Integer rows, String beginTime, String endTime,String collectionName) throws ParseException {
Criteria criteria = buildSearchParams(sendmsgLog,beginTime,endTime);
Query query = new Query();
query.addCriteria(criteria).with(new Sort(new Sort.Order(Sort.Direction.DESC,"createTime"))).skip(skipCount).limit(rows);
return mongoTemplate.find(query,SendmsgLog.class,collectionName);
}
/**
* 获得集合中过滤后的元素个数
* @param sendmsgLog
* @param beginTime
* @param endTime
* @param collectionName
* @return
* @throws ParseException
*/
public int getCollectionEleCount(SendmsgLog sendmsgLog,String beginTime,String endTime,String collectionName) throws ParseException {
Criteria criteria = buildSearchParams(sendmsgLog,beginTime,endTime);
Query query = new Query();
query.addCriteria(criteria).with(new Sort(new Sort.Order(Sort.Direction.DESC,"createTime")));
return (int) mongoTemplate.count(query,collectionName);
}
/**
* 查询同年相邻月数据
* @param sendmsgLog
* @param page
* @param rows
* @param beginTime
* @param endTime
* @param startCollectionName
* @param endCollectionName
* @return
* @throws ParseException
*/
public List<SendmsgLog> searchAdjoinMonthFromMongoDb(SendmsgLog sendmsgLog, Integer page, Integer rows, String beginTime, String endTime,String startCollectionName,String endCollectionName) throws ParseException {
long endMonthCount = getCollectionEleCount(sendmsgLog,beginTime,endTime,endCollectionName);
int skipCount = (page - 1) * rows;
if (endMonthCount - skipCount >= rows){
//当前页数据全部处于结束月集合中
return searchSameMonthFromMongoDb(sendmsgLog,skipCount,rows,beginTime,endTime,endCollectionName);
}else{
//当前页数据全部处于开始月集合中 || 处于结束月和开始月集合中
//从结束月中获取数据
List<SendmsgLog> endMonthList = searchSameMonthFromMongoDb(sendmsgLog,page,rows,beginTime,endTime,endCollectionName);
//结束月中无数据
if (CollectionUtils.isEmpty(endMonthList)){
//当前页数据全部处于开始月集合中
skipCount = (page - 1) * rows - (int)endMonthCount;
return searchSameMonthFromMongoDb(sendmsgLog,skipCount,rows,beginTime,endTime,startCollectionName);
}else{
//当前页数据处于结束月和开始月集合中
if (rows - endMonthList.size() >= 0){
skipCount = 0;
rows = rows - endMonthList.size();
List<SendmsgLog> startMonthList = searchSameMonthFromMongoDb(sendmsgLog,skipCount,rows,beginTime,endTime,startCollectionName);
endMonthList.addAll(startMonthList);
return endMonthList;
}else{
//rows - endMonthList.size() = 0,此时有新数据插入endMonth刚好够rows数量
return endMonthList;
}
}
}
}
/**
* 同年跨多月份查询
* @param year
* @param startMonth
* @param endMonth
* @param baseCollectionName
* @return
*/
public List<SendmsgLog> searchMultiMonthFromMongoDb(SendmsgLog sendmsgLog, Integer page, Integer rows, String beginTime, String endTime, int year,int startMonth,int endMonth,String baseCollectionName) throws ParseException {
int skipCount = (page - 1) * rows;
int currentCollectionEleCount;
int sumCollectionEleCount = 0;
String currentCollectionName;
List<String> waitFindCollectionNames = new ArrayList<>();
for (int i = endMonth; i>=startMonth; i--){
currentCollectionName = year + "_" + baseCollectionName + "_" + (i < 10 ? "0" + i : i);
currentCollectionEleCount = getCollectionEleCount(sendmsgLog,beginTime,endTime,currentCollectionName);
sumCollectionEleCount += currentCollectionEleCount;
if (currentCollectionEleCount >= skipCount){
waitFindCollectionNames.add(currentCollectionName);
}
if (sumCollectionEleCount >= (skipCount + rows)){
break;
}
}
List<SendmsgLog> logList;
List<SendmsgLog> resultList = new ArrayList<>();
int size = rows;
for (String collectionName : waitFindCollectionNames){
if (resultList.size() >= size) {
break;
}else{
logList = searchSameMonthFromMongoDb(sendmsgLog,skipCount,rows,beginTime,endTime,collectionName);
resultList.addAll(logList);
}
skipCount = 0;
rows = size - resultList.size();
}
return resultList;
}
public Criteria buildSearchParams(SendmsgLog sendmsgLog, String beginTime, String endTime) throws ParseException{
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Criteria criteria = new Criteria();
//发送时间开始和发送时间结束
if (StringUtils.isNotBlank(beginTime) && StringUtils.isNotBlank(endTime)) {
criteria.and("createTime").gte(sdf.parse(beginTime)).lte(sdf.parse(endTime));
}
//发送时间开始
if (StringUtils.isNotBlank(beginTime) && StringUtils.isBlank(endTime)) {
criteria.and("createTime").gte(sdf.parse(beginTime));
}
//发送时间结束
if (StringUtils.isBlank(beginTime) && StringUtils.isNotBlank(endTime)) {
criteria.and("createTime").lte(sdf.parse(endTime));
}
//任务编号
if (StringUtils.isNotBlank(sendmsgLog.getBatchNum())){
criteria.and("batchNum").is(sendmsgLog.getBatchNum());
}
//登录名
if (StringUtils.isNotBlank(sendmsgLog.getLoginName())){
criteria.and("loginName").is(sendmsgLog.getLoginName());
}
//通道简称
if (StringUtils.isNotBlank(sendmsgLog.getSmswayName()) && !sendmsgLog.getSmswayName().equals("-1")) {
criteria.and("smswayId").is(sendmsgLog.getSmswayName());
}
//发送方式
if (StringUtils.isNotBlank(sendmsgLog.getType()) && !sendmsgLog.getType().equals("-1")){
criteria.and("type").is(sendmsgLog.getType());
}
//短信类型
if (StringUtils.isNotBlank(sendmsgLog.getSmsType()) && !sendmsgLog.getSmsType().equals("-1")){
criteria.and("smsType").is(sendmsgLog.getSmsType());
}
if ((StringUtils.isNotBlank(sendmsgLog.getState()) && !sendmsgLog.getState().equals("-1")) && (StringUtils.isNotBlank(sendmsgLog.getSentStatus()) && !sendmsgLog.getSentStatus().equals("-1"))) {
if (sendmsgLog.getSentStatus().equals("1") && sendmsgLog.getState().equals("1")){
criteria.orOperator(Criteria.where("state").is(sendmsgLog.getState()),Criteria.where("sentStatus").is(sendmsgLog.getSentStatus()));
}else {
criteria.and("state").is(sendmsgLog.getState());
criteria.and("sentStatus").is(sendmsgLog.getSentStatus());
}
}else if (StringUtils.isNotBlank(sendmsgLog.getState()) && !sendmsgLog.getState().equals("-1")) {
//发送状态
criteria.and("state").is(sendmsgLog.getState());
}else if (StringUtils.isNotBlank(sendmsgLog.getSentStatus()) && !sendmsgLog.getSentStatus().equals("-1")) {
//下发状态
criteria.and("sentStatus").is(sendmsgLog.getSentStatus());
}else{
//均为空的情况前面拦截掉了
}
return criteria;
}