EasyExcel implements file import in batches

EasyExcel

EasyExcel is a Java-based, fast and concise Excel processing tool that solves memory overflow of large files.
It allows you to quickly complete Excel's reading, writing and other functions without considering performance, memory and other factors.

Introduce dependencies

 	<dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>easyexcel</artifactId>
        <version>3.1.3</version> 
    </dependency>

Introduce dozermapper dependency for subsequent type conversion between objects

    <dependency>
        <groupId>com.github.dozermapper</groupId>
        <artifactId>dozer-spring-boot-starter</artifactId>
        <version>6.5.0</version>
    </dependency>

Table Structure

Student table

CREATE TABLE `student`  (
  `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '学生表ID',
  `sname` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '学生姓名',
  `sno` bigint NOT NULL COMMENT '学号',
  `sex` char(2) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '性别',
  `age` int NOT NULL COMMENT '年龄',
  `is_deleted` tinyint UNSIGNED NOT NULL DEFAULT 0 COMMENT '是否删除',
  PRIMARY KEY (`id`)
);

Project structure

DozerUtils tool class

public class DozerUtils {
    
    
    public static <T,S> List<T> mapList(final Mapper mapper, List<S> sourceList, Class<T> targetObjectClass){
    
    
        List<T> targetList=new ArrayList<T>();
        for(S s:sourceList){
    
    
            targetList.add(mapper.map(s,targetObjectClass));
        }
        return targetList;
    }
}

Entity class Student

@ExcelProperty(value = "The header name of this field in Excel")
@ExcelIgnore: Ignore this field when parsing

@Data
public class Student {
    
    

    /**主键ID**/
    @ExcelIgnore
    private Long id;
    /**学生姓名**/
    @ExcelProperty(value = "姓名")
    private String sname;
    /**学号**/
    @ExcelProperty(value = "学号")
    private Long sno;
    /**性别**/
    @ExcelProperty(value = "性别")
    private String sex;
    /**年龄**/
    @ExcelProperty(value = "年龄")
    private Integer age;
    /**是否删除**/
    @ExcelIgnore
    private Integer isDeleted;

}

Controller

@RestController
@RequestMapping("/easyExcel")
public class EasyExcelController {
    
    

    @Resource
    private IStudentService studentService;

    @PostMapping("/studentInput")
    public void studentInput(MultipartFile file){
    
    
        studentService.studentInput(file);
    }

}

Monitoring class

This time, it is set to 1000 pieces of data for a batch import. It can be adjusted to within 3000 according to the actual situation.
This import does not set the import logic. If you need to set the logic, call the service method added according to the logic in the saveDate method of the listening class.

@Data
public class StudentListener extends AnalysisEventListener<Student> {
    
    
    /**
     *单次缓存量为1000
     */
    private final int BATCH_SIZE = 1000;

    /**
     * 临时存储List
     */
    List<Student> cacheData = new ArrayList<Student>();
    private IStudentService studentService;
    private Mapper dozerMapper;

    public StudentListener(IStudentService studentService, Mapper dozerMapper){
    
    
        this.studentService = studentService;
        this.dozerMapper = dozerMapper;
    }
    @Override
    public void invoke(Student data, AnalysisContext analysisContext) {
    
    
        cacheData.add(data);
        if (cacheData.size() >= BATCH_SIZE){
    
    
            saveData();
            //每批存储完成后清空list
            cacheData.clear();
        }
    }

    @Override
    public void doAfterAllAnalysed(AnalysisContext analysisContext) {
    
    
        if (cacheData.size() > 0){
    
    
            saveData();
        }
    }

    /**
     * 加入数据库
     */
    private void saveData(){
    
    
        List<Student> students = DozerUtils.mapList(dozerMapper,cacheData,Student.class);
        studentService.saveBatch(students);
    }
}

Service

public interface IStudentService extends IService<Student> {
    
    
    void studentInput(MultipartFile file);
}

ServiceImpl

@Service
public class StudentServiceImpl extends ServiceImpl<StudentMapper, Student> implements IStudentService {
    
    
    
    @Resource
    private IStudentService studentService;
    @Resource
    private Mapper dozerMapper;
    
    @Override
    @Transactional(rollbackFor = Throwable.class)
    public void studentInput(MultipartFile file) {
    
    
        try {
    
    
            InputStream inputStream = file.getInputStream();
            EasyExcel.read(inputStream, Student.class,
                    new StudentListener(studentService, dozerMapper)).sheet().doRead();
        }catch (Exception e){
    
    
            e.printStackTrace();
        }
    }
}

mapper

public interface StudentMapper extends BaseMapper<Student> {
    
    
}

Startup project

test

Test Data

Note that the column name text is the same as the text in the corresponding field annotation of the entity class.

PostMan test

Set the format to file in form-data

Select the test file to be uploaded

After completion, check the database

and the import is successful.