EasyExcel implements batch import of multiple sheet files

EasyExcel

EasyExcel is a Java-based, fast and concise Excel processing tool that solves memory overflow of large files.
It allows you to quickly complete Excel's reading, writing and other functions without considering performance, memory and other factors.

Introduce dependencies

 	<dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>easyexcel</artifactId>
        <version>3.1.3</version> 
    </dependency>

Introduce dozermapper dependency for subsequent type conversion between objects

    <dependency>
        <groupId>com.github.dozermapper</groupId>
        <artifactId>dozer-spring-boot-starter</artifactId>
        <version>6.5.0</version>
    </dependency>

Table Structure

Student table

CREATE TABLE `student`  (
  `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '学生表ID',
  `sname` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '学生姓名',
  `sno` bigint NOT NULL COMMENT '学号',
  `sex` char(2) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '性别',
  `age` int NOT NULL COMMENT '年龄',
  `is_deleted` tinyint UNSIGNED NOT NULL DEFAULT 0 COMMENT '是否删除',
  PRIMARY KEY (`id`)
);

Class Schedule

CREATE TABLE `course`  (
  `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '课程表ID',
  `cname` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '课程名',
  `cno` bigint NOT NULL COMMENT '课程号',
  `tno` bigint NOT NULL COMMENT '任教教师编码',
  `is_deleted` tinyint(1) UNSIGNED NOT NULL DEFAULT 0 COMMENT '是否删除',
  PRIMARY KEY (`id`)
);

teacher list

CREATE TABLE `teacher`  (
  `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '教师表ID',
  `tname` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '教师姓名',
  `tno` bigint NOT NULL COMMENT '教师编号',
  `sex` char(2) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '性别',
  `age` int NOT NULL COMMENT '年龄',
  `is_deleted` tinyint(1) UNSIGNED NOT NULL DEFAULT 0 COMMENT '是否删除',
  PRIMARY KEY (`id`)
);

Project structure

DozerUtils tool class

public class DozerUtils {
    
    
    public static <T,S> List<T> mapList(final Mapper mapper, List<S> sourceList, Class<T> targetObjectClass){
    
    
        List<T> targetList=new ArrayList<T>();
        for(S s:sourceList){
    
    
            targetList.add(mapper.map(s,targetObjectClass));
        }
        return targetList;
    }
}

Entity class

Student

@Data
public class Student {
    
    

    /**主键ID**/
    @ExcelIgnore
    private Long id;
    /**学生姓名**/
    @ExcelProperty(value = "姓名")
    private String sname;
    /**学号**/
    @ExcelProperty(value = "学号")
    private Long sno;
    /**性别**/
    @ExcelProperty(value = "性别")
    private String sex;
    /**年龄**/
    @ExcelProperty(value = "年龄")
    private Integer age;
    /**是否删除**/
    @ExcelIgnore
    private Integer isDeleted;

}

Course

@Data
public class Course {
    
    
    /**主键ID**/
    @ExcelIgnore
    private Long id;
    /**课程名**/
    @ExcelProperty(value = "课程名")
    private String cname;
    /**课程号**/
    @ExcelProperty(value = "课程号")
    private Long cno;
    /**任教教师编号**/
    @ExcelProperty(value = "任教教师编号")
    private Long tno;
    /**是否删除**/
    @ExcelIgnore
    private Integer isDeleted;
}

Teacher

@Data
public class Teacher {
    
    
    /**主键ID**/
    @ExcelIgnore
    private Long id;
    /**教师姓名**/
    @ExcelProperty(value = "姓名")
    private String tname;
    /**教师编号**/
    @ExcelProperty(value = "教师编号")
    private Long tno;
    /**性别**/
    @ExcelProperty(value = "性别")
    private String sex;
    /**年龄**/
    @ExcelProperty(value = "年龄")
    private Integer age;
    /**是否删除**/
    @ExcelIgnore
    private Integer isDeleted;
}

Controller

@RestController
@RequestMapping("/easyExcel")
public class EasyExcelController {
    
    

	@Resource
    private IEasyExcelService easyExcelService;

    @PostMapping("excelInput")
    public void excelInput(MultipartFile file){
    
    
        easyExcelService.excelInput(file);
    }

}

Monitoring class

This time, it is set to 1000 pieces of data for a batch import. It can be adjusted to within 3000 according to the actual situation.
This import does not set the import logic. If you need to set the logic, call the service method added according to the logic in the saveDate method of the listening class.

StudentListener

public class StudentListener extends AnalysisEventListener<Student> {
    
    
    /**
     *单次缓存量为1000
     */
    private final int BATCH_SIZE = 1000;

    /**
     * 临时存储List
     */
    List<Student> cacheData = new ArrayList<Student>();
    private IStudentService studentService;
    private Mapper dozerMapper;

    public StudentListener(IStudentService studentService, Mapper dozerMapper){
    
    
        this.studentService = studentService;
        this.dozerMapper = dozerMapper;
    }
    @Override
    public void invoke(Student data, AnalysisContext analysisContext) {
    
    
        cacheData.add(data);
        if (cacheData.size() >= BATCH_SIZE){
    
    
            saveData();
            //每批存储完成后清空list
            cacheData.clear();
        }
    }

    @Override
    public void doAfterAllAnalysed(AnalysisContext analysisContext) {
    
    
        if (cacheData.size() > 0){
    
    
            saveData();
        }
    }

    /**
     * 加入数据库
     */
    private void saveData(){
    
    
        List<Student> students = DozerUtils.mapList(dozerMapper,cacheData,Student.class);
        studentService.saveBatch(students);
    }
}

CourseListener

public class CourseListener extends AnalysisEventListener<Course> {
    
    
    /**
     *单次缓存量为1000
     */
    private final int BATCH_SIZE = 1000;

    /**
     * 临时存储List
     */
    List<Course> cacheData = new ArrayList<Course>();
    private ICourseService courseService;
    private Mapper dozerMapper;

    public CourseListener(ICourseService courseService, Mapper dozerMapper){
    
    
        this.courseService = courseService;
        this.dozerMapper = dozerMapper;
    }
    @Override
    public void invoke(Course data, AnalysisContext analysisContext) {
    
    
        cacheData.add(data);
        if (cacheData.size() >= BATCH_SIZE){
    
    
            saveData();
            //每批存储完成后清空list
            cacheData.clear();
        }
    }

    @Override
    public void doAfterAllAnalysed(AnalysisContext analysisContext) {
    
    
        if (cacheData.size() > 0){
    
    
            saveData();
        }
    }

    /**
     * 加入数据库
     */
    private void saveData(){
    
    
        List<Course> courses = DozerUtils.mapList(dozerMapper,cacheData,Course.class);
        courseService.saveBatch(courses);
    }
}

TeacherListener

public class TeacherListener extends AnalysisEventListener<Teacher> {
    
    
    /**
     *单次缓存量为1000
     */
    private final int BATCH_SIZE = 1000;

    /**
     * 临时存储List
     */
    List<Teacher> cacheData = new ArrayList<Teacher>();
    private ITeacherService teacherService;
    private Mapper dozerMapper;

    public TeacherListener(ITeacherService teacherService, Mapper dozerMapper){
    
    
        this.teacherService = teacherService;
        this.dozerMapper = dozerMapper;
    }
    @Override
    public void invoke(Teacher data, AnalysisContext analysisContext) {
    
    
        cacheData.add(data);
        if (cacheData.size() >= BATCH_SIZE){
    
    
            saveData();
            //每批存储完成后清空list
            cacheData.clear();
        }
    }

    @Override
    public void doAfterAllAnalysed(AnalysisContext analysisContext) {
    
    
        if (cacheData.size() > 0){
    
    
            saveData();
        }
    }

    /**
     * 加入数据库
     */
    private void saveData(){
    
    
        List<Teacher> teachers = DozerUtils.mapList(dozerMapper,cacheData,Teacher.class);
        teacherService.saveBatch(teachers);
    }
}

Service

EasyExcelService

public interface IEasyExcelService {
    
    
    void excelInput(MultipartFile file);
}

StudentService

public interface IStudentService extends IService<Student> {
    
    
}

CourseService

public interface ICourseService extends IService<Course> {
    
    
}

TeacherService

public interface ITeacherService extends IService<Teacher> {
    
    
}

ServiceImpl

EasyExcelServiceImpl

@Service
public class EasyExcelServiceImpl implements IEasyExcelService {
    
    

    @Resource
    private ICourseService courseService;
    @Resource
    private ITeacherService teacherService;
    @Resource
    private IStudentService studentService;
    @Resource
    private Mapper dozerMapper;

    @Override
    @Transactional(rollbackFor = Throwable.class)
    public void excelInput(MultipartFile file) {
    
    
        try {
    
    
            InputStream inputStream = file.getInputStream();
            ExcelReader excelReader = EasyExcel.read(inputStream).build();

            ReadSheet sheet = EasyExcel.readSheet("student").head(Student.class)
                    .registerReadListener(new StudentListener(studentService,dozerMapper))
                    .build();
            ReadSheet sheet1 = EasyExcel.readSheet("course").head(Course.class)
                    .registerReadListener(new CourseListener(courseService,dozerMapper))
                    .build();
            ReadSheet sheet2 = EasyExcel.readSheet("teacher").head(Teacher.class)
                    .registerReadListener(new TeacherListener(teacherService,dozerMapper))
                    .build();
            excelReader.read(sheet);
            excelReader.read(sheet1);
            excelReader.read(sheet2);
            excelReader.finish();
        }catch (Exception e){
    
    
            e.printStackTrace();
        }
    }
}

StudentServiceImpl

@Service
public class StudentServiceImpl extends ServiceImpl<StudentMapper, Student> implements IStudentService {
    
    
}

CourseServiceImpl

@Service
public class CourseServiceImpl extends ServiceImpl<CourseMapper, Course> implements ICourseService {
    
    
}

TeacherServiceImpl

@Service
public class TeacherServiceImpl extends ServiceImpl<TeacherMapper, Teacher> implements ITeacherService {
    
    
}

mapper

StudentMapper

public interface StudentMapper extends BaseMapper<Student> {
    
    
}

CourseMapper

public interface CourseMapper extends BaseMapper<Course> {
    
    
}

TeacherMapper

public interface TeacherMapper extends BaseMapper<Teacher> {
    
    
}

Startup project

test

Test Data

Create three sheet test files. Sheetming and their respective internal column names are set as required.

PostMan test

Set the format to file in form-data

Select the test file to be uploaded

and view the database after completion

Import successful