The efficiency of custom paging query for millions of data has soared
1. The reason for the slowness?
Offset+limit paging query, when the data table exceeds 1 million records, the performance will be very poor.
The main reason is that the paging method of offset limit is to query from the beginning and then discard the first offset records, so the larger the offset offset, the slower the query speed.
The paging plug-in of mybatis-plus may have this problem.
2. How to optimize?
Method 1: Reordering based on index, using MySQL to support ORDER operation, you can use the index to quickly locate some tuples and avoid full table scan
#大于上一页最后一个id
SELECT * FROM table WHERE id>=10000 ORDER BY id ASC LIMIT 0,20
Method 2: Use subquery/join + index to quickly locate the tuple, and then read the tuple
SELECT a.* FROM table a JOIN (select id from table limit 100000, 20) b ON a.id = b.id
3. Upload the code (method 2 used by bloggers)
1. Customize Page
@Data
public class MyPage<T> implements Serializable {
private List<T> records; //结果集
private Integer total; //符合条件总条数
private Integer size; //每页记录数
private Integer current; //当前处于第几页
private List<T> orders;
private final boolean optimizeCountSql = true;
private final boolean hitCount = false;
private Integer countId;
private Integer maxLimit;
private final boolean searchCount = true;
private Integer pages; // 符合条件总页数
}
2.ServiceImpl implementation class
public MyPage<XXXPageVo> getPageXXX(XXXDto dto) {
QueryWrapper<XXX> qw = new QueryWrapper<>();
qw.lambda() //以下条件是我的业务条件，自行变更自己业务条件
.eq(ObjectUtils.isNotEmpty(dto.getOrderId()),XXX::getOrderId,dto.getOrderId())
.eq(ObjectUtils.isNotEmpty(dto.getOrderNo()) && dto.getOrderNo().length() == 14,XXX::getNewSysOrderNo,dto.getOrderNo())
.eq(ObjectUtils.isNotEmpty(dto.getOrderNo()) && dto.getOrderNo().length() > 14,XXX::getOrderNo,dto.getOrderNo())
.eq(ObjectUtils.isNotEmpty(dto.getMainDriverId()),XXX::getMainDriverId,dto.getMainDriverId())
.like(ObjectUtils.isNotEmpty(dto.getMainDriverName()),XXX::getMainDriverName,dto.getMainDriverName())
.like(ObjectUtils.isNotEmpty(dto.getMainDriverVehicleNo()),XXX::getMainDriverVehicleNo,dto.getMainDriverVehicleNo())
.eq(ObjectUtils.isNotEmpty(dto.getKmDriverId()),XXX::getKmDriverId,dto.getKmDriverId())
.like(ObjectUtils.isNotEmpty(dto.getKmDriverName()),XXX::getKmDriverName,dto.getKmDriverName())
.like(ObjectUtils.isNotEmpty(dto.getKmDriverVehicleNo()),XXX::getKmDriverVehicleNo,dto.getKmDriverVehicleNo())
.eq(ObjectUtils.isNotEmpty(dto.getLineGroupId()),XXX::getLineGroupId,dto.getLineGroupId())
.eq(ObjectUtils.isNotEmpty(dto.getTotalLineGroupId()),XXX::getTotalLineGroupId,dto.getTotalLineGroupId())
.eq(ObjectUtils.isNotEmpty(dto.getPassengerMobile()),XXX::getPassengerMobile,dto.getPassengerMobile())
.eq(ObjectUtils.isNotEmpty(dto.getKmDriverScore()),XXX::getKmDriverScore,dto.getKmDriverScore())
.eq(ObjectUtils.isNotEmpty(dto.getZtDriverScore()),XXX::getZtDriverScore,dto.getZtDriverScore())
.eq(ObjectUtils.isNotEmpty(dto.getUid()),XXX::getUid,dto.getUid())
.eq(ObjectUtils.isNotEmpty(dto.getStatus()),XXX::getStatus,dto.getStatus());
if(ObjectUtils.isNotEmpty(dto.getStartArrangeRunTime()) && ObjectUtils.isNotEmpty(dto.getEndArrangeRunTime())){
long sl = dto.getStartArrangeRunTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
long el = dto.getEndArrangeRunTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
qw.lambda().between(XXX::getArrangeRunTime,sl,el);
}
if(ObjectUtils.isNotEmpty(dto.getStartUpdateTime()) && ObjectUtils.isNotEmpty(dto.getEndUpdateTime())){
long slu = dto.getStartUpdateTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
long elu = dto.getEndUpdateTime().toInstant(ZoneOffset.ofHours(8)).toEpochMilli()/1000;
qw.lambda().between(XXX::getUpdateTime,slu,elu);
}
// CurrentSize 自定义字段 例如: 现在分页处于第3页，每页条数是10条,那么第四页需要从
// 第31条开始查询，抛弃前面30条数据,也就有了下面的这行代码
dto.setCurrentSize(dto.getCurrent() * dto.getSize() - dto.getSize());
Integer total = baseMapper.selectCount(qw); // 关键点(查询出符合条件的总条数)
List<XXXPageVo> resList = baseMapper.pageComments(dto);
//以下是自定义Page拼装
MyPage<XXXPageVo> page = new MyPage<>();
int pages = (int) Math.ceil((double) total / dto.getSize());
page.setTotal(total);
page.setPages(pages);
page.setRecords(resList);
page.setSize(dto.getSize());
page.setCurrent(dto.getCurrent());
return page;
}
3.Mapper.xml
<select id="pageComments" parameterType="包名.XXXDTO" resultType="包名.XXXPageVO">
SELECT
a.id,
uniacid,
order_no,
order_id,
total_line_group_id,
total_line_group_name,
line_group_id,
zt_driver_label_ids,
zt_driver_score,
zt_driver_other_comment,
zt_driver_label_str,
main_driver_vehicle_no,
km_driver_label_ids,
km_driver_score,
km_driver_label_str,
description,
main_driver_id,
km_driver_other_comment,
main_driver_name,
km_driver_id,
km_driver_name,
km_driver_vehicle_no,
other_comment_content,
order_name,
from_unixtime( arrange_run_time, '%Y-%m-%d %H:%i:%s' ) AS arrange_run_time,
passenger_name,
passenger_mobile,
uid,
wx_name,
new_sys_order_no,
recovery_time,
from_unixtime( update_time, '%Y-%m-%d %H:%i:%s' ) AS update_time,
update_user_id,
update_user_name,
is_del,
STATUS,
from_type
FROM
table a
INNER JOIN
(
SELECT
id
FROM
table
where
is_del = 0
<if test="XXXDTO.orderId != null and XXXDTO.orderId != '' ">
AND order_id = #{XXXDTO.orderId}
</if>
<if test="XXXDTO.orderNo != null and XXXDTO.orderNo != '' and XXXDTO.orderNo.length == 14 ">
AND new_sys_order_no = #{XXXDTO.orderNo}
</if>
<if test="XXXDTO.orderNo != null and XXXDTO.orderNo != '' and XXXDTO.orderNo.length > 14 ">
AND order_no = #{XXXDTO.orderNo}
</if>
<if test="XXXDTO.mainDriverId != null and XXXDTO.mainDriverId != '' ">
AND main_driver_id = #{XXXDTO.mainDriverId}
</if>
<if test="XXXDTO.mainDriverName != null and XXXDTO.mainDriverName != '' ">
AND main_driver_name like CONCAT('%',CONCAT(#{XXXDTO.mainDriverName},'%'))
</if>
<if test="XXXDTO.mainDriverVehicleNo != null and XXXDTO.mainDriverVehicleNo != '' ">
AND main_driver_vehicle_no like CONCAT('%',CONCAT(#{XXXDTO.mainDriverVehicleNo},'%'))
</if>
<if test="XXXDTO.kmDriverId != null and XXXDTO.kmDriverId != '' ">
AND km_driver_id = #{XXXDTO.kmDriverId}
</if>
<if test="XXXDTO.kmDriverName != null and XXXDTO.kmDriverName != '' ">
AND km_driver_name like CONCAT('%',CONCAT(#{XXXDTO.kmDriverName},'%'))
</if>
<if test="XXXDTO.kmDriverVehicleNo != null and XXXDTO.kmDriverVehicleNo != '' ">
AND km_driver_vehicle_no like CONCAT('%',CONCAT(#{XXXDTO.kmDriverVehicleNo},'%'))
</if>
<if test="XXXDTO.lineGroupId != null and XXXDTO.lineGroupId != '' ">
AND line_group_id = #{XXXDTO.lineGroupId}
</if>
<if test="XXXDTO.totalLineGroupId != null and XXXDTO.totalLineGroupId != '' ">
AND total_line_group_id = #{XXXDTO.totalLineGroupId}
</if>
<if test="XXXDTO.passengerMobile != null and XXXDTO.passengerMobile != '' ">
AND passenger_mobile = #{XXXDTO.passengerMobile}
</if>
<if test="XXXDTO.kmDriverScore != null and XXXDTO.kmDriverScore != '' ">
AND km_driver_score = #{XXXDTO.kmDriverScore}
</if>
<if test="XXXDTO.ztDriverScore != null and XXXDTO.ztDriverScore != '' ">
AND zt_driver_score = #{XXXDTO.ztDriverScore}
</if>
<if test="XXXDTO.uid != null and XXXDTO.uid != '' ">
AND uid = #{XXXDTO.uid}
</if>
<if test="XXXDTO.status != null and XXXDTO.status != '' ">
AND status = #{XXXDTO.status}
</if>
<if test="XXXDTO.startArrangeRunTime != null and XXXDTO.endArrangeRunTime != null ">
AND arrange_run_time BETWEEN unix_timestamp(#{XXXDTO.startArrangeRunTime}) AND unix_timestamp(#{XXXDTO.endArrangeRunTime})
</if>
<if test="XXXDTO.startUpdateTime != null and XXXDTO.endUpdateTime != null">
AND update_time BETWEEN unix_timestamp(#{XXXDTO.startUpdateTime}) AND unix_timestamp(#{XXXDTO.endUpdateTime})
</if>
ORDER BY update_time DESC
LIMIT #{XXXDTO.currentSize},
#{XXXDTO.size}
) b on a.id = b.id
</select>
Regarding the renderings, I won’t take screenshots. Most blogs probably use these two methods. The blogger just integrated the logic to avoid pitfalls and used it for online projects. The efficiency was really improved by dozens of times. It will be faster if we add database index assistance.