百万级数据自定义分页查询效率飞升
一.慢的原因？
offset+limit方式的分页查询，当数据表超过100w条记录，性能会很差。
主要原因是offset limit的分页方式是从头开始查询，然后舍弃前offset个记录，所以offset偏移量越大，查询速度越慢。
mybatis-plus 的分页插件也许存在这个问题。
二.如何优化？
方式1:基于索引再排序，利用MySQL支持ORDER操作可以利用索引快速定位部分元组,避免全表扫描
#大于上一页最后一个id
SELECT * FROM table WHERE id>=10000 ORDER BY id ASC LIMIT 0,20
方式2:利用子查询/连接+索引快速定位元组的位置,然后再读取元组
SELECT a.* FROM table a JOIN (select id from table limit 100000, 20) b ON a.id = b.id
三.上代码(博主用的方式2)
1.自定义Page
@Data
public class MyPage<T> implements Serializable {
private List<T> records; //结果集
private Integer total; //符合条件总条数
private Integer size; //每页记录数
private Integer current; //当前处于第几页
private List<T> orders;
private final boolean optimizeCountSql = true;
private final boolean hitCount = false;
private Integer countId;
private Integer maxLimit;
private final boolean searchCount = true;
private Integer pages; // 符合条件总页数
}
2.ServiceImpl 实现类
public MyPage<XXXPageVo> getPageXXX(XXXDto dto) {
QueryWrapper<XXX> qw = new QueryWrapper<>();
qw.lambda() //以下条件是我的业务条件，自行变更自己业务条件
.eq(ObjectUtils.isNotEmpty(dto.getOrderId()),XXX::getOrderId,dto.getOrderId())
.eq(ObjectUtils.isNotEmpty(dto.getOrderNo()) && dto.getOrderNo().length() == 14,XXX::getNewSysOrderNo,dto.getOrderNo())
.eq(ObjectUtils.isNotEmpty(dto.getOrderNo()) && dto.getOrderNo().length() > 14,XXX::getOrderNo,dto.getOrderNo())
.eq(ObjectUtils.isNotEmpty(dto.getMainDriverId()),XXX::getMainDriverId,dto.getMainDriverId())
.like(ObjectUtils.isNotEmpty(dto.getMainDriverName()),XXX::getMainDriverName,dto.getMainDriverName())
.like(ObjectUtils.isNotEmpty(dto.getMainDriverVehicleNo()),XXX::getMainDriverVehicleNo,dto.getMainDriverVehicleNo())
.eq(ObjectUtils.isNotEmpty(dto.getKmDriverId()),XXX::getKmDriverId,dto.getKmDriverId())
.like(ObjectUtils.isNotEmpty(dto.getKmDriverName()),XXX::getKmDriverName,dto.getKmDriverName())
.like(ObjectUtils.isNotEmpty(dto.getKmDriverVehicleNo()),XXX::getKmDriverVehicleNo,dto.getKmDriverVehicleNo())
.eq(ObjectUtils.isNotEmpty(dto.getLineGroupId()),XXX::getLineGroupId,dto.getLineGroupId())
.eq(ObjectUtils.isNotEmpty(dto.getTotalLineGroupId()),XXX::getTotalLineGroupId,dto.getTotalLineGroupId())
.eq(ObjectUtils.isNotEmpty(dto.getPassengerMobile()),XXX::getPassengerMobile,dto.getPassengerMobile())
.eq(ObjectUtils.isNotEmpty(dto.getKmDriverScore()),XXX::getKmDriverScore,dto.getKmDriverScore())
.eq(ObjectUtils.isNotEmpty(dto.getZtDriverScore()),XXX::getZtDriverScore,dto.getZtDriverScore())
.eq(ObjectUtils.isNotEmpty(dto.getUid()),XXX::getUid,dto.getUid())
.eq(ObjectUtils.isNotEmpty(dto.getStatus()),XXX::getStatus,dto.getStatus());
if(ObjectUtils.isNotEmpty(dto.getStartArrangeRunTime()) && ObjectUtils.isNotEmpty(dto.getEndArrangeRunTime())){
long sl = dto.getStartArrangeRunTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
long el = dto.getEndArrangeRunTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
qw.lambda().between(XXX::getArrangeRunTime,sl,el);
}
if(ObjectUtils.isNotEmpty(dto.getStartUpdateTime()) && ObjectUtils.isNotEmpty(dto.getEndUpdateTime())){
long slu = dto.getStartUpdateTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
long elu = dto.getEndUpdateTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
qw.lambda().between(XXX::getUpdateTime,slu,elu);
}
// CurrentSize 自定义字段 例如: 现在分页处于第3页，每页条数是10条,那么第四页需要从
// 第31条开始查询，抛弃前面30条数据,也就有了下面的这行代码
dto.setCurrentSize(dto.getCurrent() * dto.getSize() - dto.getSize());
Integer total = baseMapper.selectCount(qw); // 关键点(查询出符合条件的总条数)
List<XXXPageVo> resList = baseMapper.pageComments(dto);
//以下是自定义Page拼装
MyPage<XXXPageVo> page = new MyPage<>();
int pages = (int) Math.ceil((double) total / dto.getSize());
page.setTotal(total);
page.setPages(pages);
page.setRecords(resList);
page.setSize(dto.getSize());
page.setCurrent(dto.getCurrent());
return page;
}
3.Mapper.xml
<select id="pageComments" parameterType="包名.XXXDTO" resultType="包名.XXXPageVO">
SELECT
a.id,
uniacid,
order_no,
order_id,
total_line_group_id,
total_line_group_name,
line_group_id,
zt_driver_label_ids,
zt_driver_score,
zt_driver_other_comment,
zt_driver_label_str,
main_driver_vehicle_no,
km_driver_label_ids,
km_driver_score,
km_driver_label_str,
description,
main_driver_id,
km_driver_other_comment,
main_driver_name,
km_driver_id,
km_driver_name,
km_driver_vehicle_no,
other_comment_content,
order_name,
from_unixtime( arrange_run_time, '%Y-%m-%d %H:%i:%s' ) AS arrange_run_time,
passenger_name,
passenger_mobile,
uid,
wx_name,
new_sys_order_no,
recovery_time,
from_unixtime( update_time, '%Y-%m-%d %H:%i:%s' ) AS update_time,
update_user_id,
update_user_name,
is_del,
STATUS,
from_type
FROM
table a
INNER JOIN
(
SELECT
id
FROM
table
where
is_del = 0
<if test="XXXDTO.orderId != null and XXXDTO.orderId != '' ">
AND order_id = #{XXXDTO.orderId}
</if>
<if test="XXXDTO.orderNo != null and XXXDTO.orderNo != '' and XXXDTO.orderNo.length == 14 ">
AND new_sys_order_no = #{XXXDTO.orderNo}
</if>
<if test="XXXDTO.orderNo != null and XXXDTO.orderNo != '' and XXXDTO.orderNo.length > 14 ">
AND order_no = #{XXXDTO.orderNo}
</if>
<if test="XXXDTO.mainDriverId != null and XXXDTO.mainDriverId != '' ">
AND main_driver_id = #{XXXDTO.mainDriverId}
</if>
<if test="XXXDTO.mainDriverName != null and XXXDTO.mainDriverName != '' ">
AND main_driver_name like CONCAT('%',CONCAT(#{XXXDTO.mainDriverName},'%'))
</if>
<if test="XXXDTO.mainDriverVehicleNo != null and XXXDTO.mainDriverVehicleNo != '' ">
AND main_driver_vehicle_no like CONCAT('%',CONCAT(#{XXXDTO.mainDriverVehicleNo},'%'))
</if>
<if test="XXXDTO.kmDriverId != null and XXXDTO.kmDriverId != '' ">
AND km_driver_id = #{XXXDTO.kmDriverId}
</if>
<if test="XXXDTO.kmDriverName != null and XXXDTO.kmDriverName != '' ">
AND km_driver_name like CONCAT('%',CONCAT(#{XXXDTO.kmDriverName},'%'))
</if>
<if test="XXXDTO.kmDriverVehicleNo != null and XXXDTO.kmDriverVehicleNo != '' ">
AND km_driver_vehicle_no like CONCAT('%',CONCAT(#{XXXDTO.kmDriverVehicleNo},'%'))
</if>
<if test="XXXDTO.lineGroupId != null and XXXDTO.lineGroupId != '' ">
AND line_group_id = #{XXXDTO.lineGroupId}
</if>
<if test="XXXDTO.totalLineGroupId != null and XXXDTO.totalLineGroupId != '' ">
AND total_line_group_id = #{XXXDTO.totalLineGroupId}
</if>
<if test="XXXDTO.passengerMobile != null and XXXDTO.passengerMobile != '' ">
AND passenger_mobile = #{XXXDTO.passengerMobile}
</if>
<if test="XXXDTO.kmDriverScore != null and XXXDTO.kmDriverScore != '' ">
AND km_driver_score = #{XXXDTO.kmDriverScore}
</if>
<if test="XXXDTO.ztDriverScore != null and XXXDTO.ztDriverScore != '' ">
AND zt_driver_score = #{XXXDTO.ztDriverScore}
</if>
<if test="XXXDTO.uid != null and XXXDTO.uid != '' ">
AND uid = #{XXXDTO.uid}
</if>
<if test="XXXDTO.status != null and XXXDTO.status != '' ">
AND status = #{XXXDTO.status}
</if>
<if test="XXXDTO.startArrangeRunTime != null and XXXDTO.endArrangeRunTime != null ">
AND arrange_run_time BETWEEN unix_timestamp(#{XXXDTO.startArrangeRunTime}) AND unix_timestamp(#{XXXDTO.endArrangeRunTime})
</if>
<if test="XXXDTO.startUpdateTime != null and XXXDTO.endUpdateTime != null">
AND update_time BETWEEN unix_timestamp(#{XXXDTO.startUpdateTime}) AND unix_timestamp(#{XXXDTO.endUpdateTime})
</if>
ORDER BY update_time DESC
LIMIT #{XXXDTO.currentSize},
#{XXXDTO.size}
) b on a.id = b.id
</select>
关于效果图也就不截图了，大多数博客大概也用这两种方式，博主只是整合了一下逻辑，避一避坑，并用于线上项目，效率是真的提升了几十个倍，如果再加入数据库索引辅助的话会更快