ilustrar

Registre o aprendizado do MySQL para adicionar milhões de dados em lotes. Guarde para mais tarde.
O ambiente local mysql está instalado com a versão 5.7, o projeto usa a versão jdk1.8 e a versão do driver mysql usada pelo projeto é a versão 8.0.

Endereço do código de demonstração do projeto

SpringBoot usa mybatis para adicionar em lote 5 milhões de dados ao banco de dados mysql Demo

Diretório do projeto

Insira a descrição da imagem aqui

instrução de criação de tabela correspondente do mysql

CREATE TABLE `order_info` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
  `period` int(11) NOT NULL COMMENT '账期月份',
  `amount` decimal(20,2) NOT NULL COMMENT '金额',
  `user_name` varchar(20) NOT NULL COMMENT '下单人',
  `phone` varchar(11) NOT NULL COMMENT '手机号',
  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `creator` varchar(20) NOT NULL COMMENT '创建人',
  `modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间',
  `modifier` varchar(20) NOT NULL COMMENT '修改人',
  PRIMARY KEY (`id`) USING BTREE,
  KEY `idx_period` (`period`),
  KEY `idx_modified` (`modified`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC COMMENT='订单信息表';

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.7.3</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.example</groupId>
    <artifactId>batching</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>batching</name>
    <description>batching</description>
    <properties>
        <java.version>1.8</java.version>
        <!--下列版本都是2022/04/16最新版本,都是父项目的基本依赖,用来子项目继承父项目依赖-->
        <pagehelper-starter.version>1.4.2</pagehelper-starter.version>
        <mybatis.version>3.5.9</mybatis.version>
        <mysql-connector.version>8.0.28</mysql-connector.version>
        <druid.version>1.2.9</druid.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <!--MyBatis分页插件1.4.2版本才支持spring-boot2.6.6-->
        <!--pagehelper分页官网:https://mvnrepository.com/artifact/com.github.pagehelper/pagehelper-spring-boot-starter/-->
        <dependency>
            <groupId>com.github.pagehelper</groupId>
            <artifactId>pagehelper-spring-boot-starter</artifactId>
            <version>${
    
    pagehelper-starter.version}</version>
        </dependency>
        <!-- MyBatis就是用来创建数据库连接进行增删改查等操作,提供了原生JDBC,如Connection,Statement,ResultSet这些底层-->
        <!-- MyBatis官网:https://mybatis.org/mybatis-3/zh/dependency-info.html-->
        <dependency>
            <groupId>org.mybatis</groupId>
            <artifactId>mybatis</artifactId>
            <version>${
    
    mybatis.version}</version>
        </dependency>
        <!--Mysql数据库驱动-->
        <!--Mysql驱动官网:https://mvnrepository.com/artifact/mysql/mysql-connector-java/-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>${
    
    mysql-connector.version}</version>
        </dependency>
        <!--集成druid连接池-->
        <!--druid版本官网:https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>${
    
    druid.version}</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

</project>

classe de configuração application.yml

server:
  port: 8080

mybatis:
  mapper-locations:
    - classpath:mapper/*.xml #找到mybatis位置,自定义sql语句
  #当查询语句中resultType="java.util.HashMap"时,如果返回的字段值为null时,设置如下参数为true,让它返回
  configuration:
    call-setters-on-nulls: true

spring:
  datasource:
    #mysql批量新增需要在url后面添加rewriteBatchedStatements=true才能生效
    url: jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
    driverClassName: com.mysql.cj.jdbc.Driver #mysql8.0驱动,mysql5.7驱动是com.mysql.jdbc.Driver
    username: 你自己数据库的用户名
    password: 你自己数据库的密码
    druid:
      initial-size: 3 #连接池初始大小
      min-idle: 5 #最小空闲连接数
      max-active: 20 #最大空闲连接数
      web-stat-filter:
        exclusions: "*.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/*" #不统计这些请求数据
      stat-view-servlet: #访问监控网页的用户名和密码
        #默认为true,内置监控页面首页/druid/index.html
        enabled: true
        login-username: druid
        login-password: druid

Código da classe de inicialização

package com.example.batching;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class BatchingApplication {
    
    

    public static void main(String[] args) {
    
    
        SpringApplication.run(BatchingApplication.class, args);
    }

}

Classe de entidade OrderInfo

package com.example.batching.entity;

import java.math.BigDecimal;

public class OrderInfo {
    
    

    private int id;
    private int period;//账期月份
    private BigDecimal amount;//金额
    private String userName;//下单人
    private String phone;//手机号
    private String created;//创建时间
    private String creator;//创建人
    private String modified;//修改时间
    private String modifier;//修改人

    public int getId() {
    
    
        return id;
    }

    public void setId(int id) {
    
    
        this.id = id;
    }

    public int getPeriod() {
    
    
        return period;
    }

    public void setPeriod(int period) {
    
    
        this.period = period;
    }

    public BigDecimal getAmount() {
    
    
        return amount;
    }

    public void setAmount(BigDecimal amount) {
    
    
        this.amount = amount;
    }

    public String getUserName() {
    
    
        return userName;
    }

    public void setUserName(String userName) {
    
    
        this.userName = userName;
    }

    public String getPhone() {
    
    
        return phone;
    }

    public void setPhone(String phone) {
    
    
        this.phone = phone;
    }

    public String getCreated() {
    
    
        return created;
    }

    public void setCreated(String created) {
    
    
        this.created = created;
    }

    public String getCreator() {
    
    
        return creator;
    }

    public void setCreator(String creator) {
    
    
        this.creator = creator;
    }

    public String getModified() {
    
    
        return modified;
    }

    public void setModified(String modified) {
    
    
        this.modified = modified;
    }

    public String getModifier() {
    
    
        return modifier;
    }

    public void setModifier(String modifier) {
    
    
        this.modifier = modifier;
    }
}

Camada de controle TestController

package com.example.batching.controller;

import com.example.batching.entity.OrderInfo;
import com.example.batching.service.TestService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;

@RestController
@RequestMapping(value = "/order")
public class TestController {
    
    

    @Autowired
    private TestService testService;

    //批量新增数据处理
    @PostMapping(value = "/batchSave")
    public String batchSave() {
    
    
        //随机生成电话号码
        String[] start = {
    
    "130", "131", "132", "133", "134", "150", "151", "155", "158", "166", "180", "181", "184", "185", "188"};
        List<OrderInfo> orderInfoList=new ArrayList<>();
        //生成500万数据批量新增到mysql数据库里面
        for(int i=1;i<=5000000;i++){
    
    
            OrderInfo orderInfo=new OrderInfo();
            orderInfo.setPeriod(202206);
            orderInfo.setAmount(new BigDecimal(i));
            orderInfo.setUserName("用户"+i);
            orderInfo.setPhone(start[(int) (Math.random() * start.length)]+(10000000+(int)(Math.random()*(99999999-10000000+1))));
            orderInfo.setCreator("用户"+i);
            orderInfo.setModifier("用户"+i);
            orderInfoList.add(orderInfo);
            //每一万条数据进行批量新增
            if(i%10000==0){
    
    
                testService.batchSave(orderInfoList);
                //新增完成后清空list集合防止内存溢出
                orderInfoList.clear();
                System.out.println("当前已新增完数据："+i+"行");
            }
        }
        return "成功";
    }
}

Camada de interface TestService

package com.example.batching.service;

import com.example.batching.entity.OrderInfo;

import java.util.List;

public interface TestService {
    
    
    void batchSave(List<OrderInfo> orderInfoList);
}

Camada de implementação TestServiceImpl

package com.example.batching.service.impl;

import com.example.batching.dao.TestDao;
import com.example.batching.entity.OrderInfo;
import com.example.batching.service.TestService;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.List;

@Service
public class TestServiceImpl implements TestService {
    
    

    @Resource
    private TestDao testDao;
    @Resource
    private SqlSessionFactory sqlSessionFactory;

    @Override
    public void batchSave(List<OrderInfo> orderInfoList) {
    
    
        //批量新增处理,需要在jdbc连接那里添加rewriteBatchedStatements=true属性，批量新增才能生效
        // ExecutorType.SIMPLE: 这个执行器类型不做特殊的事情。它为每个语句的执行创建一个新的预处理语句。自动提交不关闭的前提下，默认设置是这个
        // ExecutorType.REUSE: 这个执行器类型会复用预处理语句。
        // ExecutorType.BATCH: 这个执行器会批量执行所有更新语句,如果 SELECT 在它们中间执行还会标定它们是 必须的,来保证一个简单并易于理解的行为。
        //如果自动提交设置为true,将无法控制提交的条数，改为最后统一提交
        SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
        try {
    
    
            TestDao testMapper = sqlSession.getMapper(TestDao.class);
            orderInfoList.stream().forEach(orderInfo -> testMapper.batchSave(orderInfo));
            //提交数据
            sqlSession.commit();
        } catch (Exception e) {
    
    
            sqlSession.rollback();
        } finally {
    
    
            sqlSession.close();
        }
    }
}

Camada de interface de dados TestDao

package com.example.batching.dao;

import com.example.batching.entity.OrderInfo;
import org.apache.ibatis.annotations.Mapper;

@Mapper
public interface TestDao {
    
    
    void batchSave(OrderInfo orderInfo);
}

camada dao corresponde à instrução sql personalizada mapper.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<!-- namespace必须指向Dao接口 -->
<mapper namespace="com.example.batching.dao.TestDao">

    <insert id="batchSave" parameterType="com.example.batching.entity.OrderInfo">
        INSERT INTO order_info
        <trim prefix="(" suffix=")" suffixOverrides=",">
            <if test="period != null">
                period,
            </if>
            <if test="amount != null">
                amount,
            </if>
            <if test="userName != null">
                user_name,
            </if>
            <if test="phone != null">
                phone,
            </if>
            <if test="creator != null">
                creator,
            </if>
            <if test="modifier != null">
                modifier,
            </if>
        </trim>
        <trim prefix="values (" suffix=")" suffixOverrides=",">
            <if test="period != null">
                #{
    
    period},
            </if>
            <if test="amount != null">
                #{
    
    amount},
            </if>
            <if test="userName != null">
                #{
    
    userName},
            </if>
            <if test="phone != null">
                #{
    
    phone},
            </if>
            <if test="creator != null">
                #{
    
    creator},
            </if>
            <if test="modifier != null">
                #{
    
    modifier},
            </if>
        </trim>
    </insert>
</mapper>

Os resultados do teste são os seguintes

A interface de chamada do Postman retorna da seguinte forma:
Insira a descrição da imagem aqui
O console em segundo plano é impresso da seguinte forma:

A interface de execução do banco de dados é a seguinte: