sharding-jdbc use caseCustom primary key generation strategyCustom table partition strategyCustom library partition strategy

Sharding-JDBC

Perform operations such as sharding the database in accordance with the rules in the form of configuration

Hello World

Implement horizontal table splitting through Sharding-JDBC: use user_1 and user_2 in the database to split horizontally through the primary key, the id is an odd number into the user_1 table, and the id is an even number into the user_2 table

1 Create a database and table

create database sharding character set = utf8;
use sharding;

create table user_1(
 id int primary key auto_increment,
 name varchar(20),
 age int,
 birthday datetime,
 cmd varchar(200)
);

create table user_2(
 id int primary key auto_increment,
 name varchar(20),
 age int,
 birthday datetime,
 cmd varchar(200)
);

2 Create a SpringBoot project

Sharding-jdbc provides us with a startup class that integrates springBoot, which is very simple to configure

<!-- 数据库连接驱动 -->
<dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <scope>runtime</scope>
</dependency>

<!-- mybatis-plus -->
<dependency>
    <groupId>com.baomidou</groupId>
    <artifactId>mybatis-plus-boot-starter</artifactId>
    <version>3.4.1</version>
</dependency>


<!-- sharding-jdbc 数据库分库分表 -->
<dependency>
    <groupId>org.apache.shardingsphere</groupId>
    <artifactId>sharding-jdbc-spring-boot-starter</artifactId>
    <version>4.1.1</version>
</dependency>

<!-- 数据库连接池-->
<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>druid</artifactId>
    <version>1.1.8</version>
</dependency>
<!-- reids 依赖(这里主要用于生成主键) -->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>

3 configure the application.properties file

##基本配置(正常操作)
server.port=8998

# mybatis-plus 配置
mybatis-plus.mapper-locations=classpath*:/mapper/**/*.xml
mybatis-plus.type-aliases-package=com.mt.bean
mybatis-plus.configuration.map-underscore-to-camel-case=true
mybatis-plus.configuration.log-impl=org.apache.ibatis.logging.stdout.StdOutImpl

spring.redis.host=120.25.227.88
spring.redis.password=123456


#sharding-jdbc的配置 ps:官网有详细的配置文件介绍:https://shardingsphere.apache.org/document/legacy/4.x/document/cn/manual/sharding-jdbc/configuration/config-spring-boot/

#声明一个数据库(虚拟的)
spring.shardingsphere.datasource.names=db1

#声明虚拟数据库对应的连接,驱动,用户名,密码,连接池等信息
spring.shardingsphere.datasource.db1.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db1.url=jdbc:mysql://localhost:3306/sharding?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db1.username=root
spring.shardingsphere.datasource.db1.password=root
spring.shardingsphere.datasource.db1.type=com.alibaba.druid.pool.DruidDataSource


#声明 表存放在对应的数据库 $->{0..1} 就是行内表达式
#这里的意思是db1库内有user虚拟表指向 user_1和user_2
spring.shardingsphere.sharding.tables.user.actual-data-nodes=db1.user_$->{1..2}

#设置主键字段
spring.shardingsphere.sharding.tables.user.key-generator.column=id
# 设置主键生成策略 可选内置的 SNOWFLAKE(雪花算法)/UUID 
#也可以自定义(实现ShardingKeyGenerator,并配置META-INF/services/org.apache.shardingsphere.spi.keygen.ShardingKeyGenerator) 
spring.shardingsphere.sharding.tables.user.key-generator.type=SNOWFLAKE

#设置 根据哪个字段进行分片
spring.shardingsphere.sharding.tables.user.table-strategy.inline.sharding-column=id
#分片规则奇数存入user_1  偶数存入user_2
spring.shardingsphere.sharding.tables.user.table-strategy.inline.algorithm-expression=user_$->{id % 2 != 0 ? 1:2}

spring.shardingsphere.props.sql.show=true

4 write the corresponding object

bean:

@Data
@NoArgsConstructor
@AllArgsConstructor
@TableName("user")
public class User {
    
    

    @TableId(type = IdType.AUTO)
    private Long id;
    private String name;
    private Integer age;
    private Date birthday;
    private String cmd;
}

mapper:

public interface UserMapper extends BaseMapper<User> {
    
    }

Test class:

@SpringBootTest
class ShardingSphereApplicationTests {
    
    

    @Autowired
    private UserMapper userMapper;

    @Test
    void contextLoads() {
    
    
        User user1 = new User();
        user1.setAge(10);
        user1.setName("张三");
        user1.setBirthday(new Date());
        user1.setCmd("张三今年10岁");
        
        User user2 = new User();
        user2.setAge(20);
        user2.setName("李四");
        user2.setBirthday( new Date() );
        user2.setCmd("李四今年20岁");
        
        userMapper.insert( user2 );
    }

}

After testing, user Zhang San was added to user_1, and Li Si was added to user_2

Custom primary key generation strategy

Sharding-jdbc provides us with 2 default generation strategies:

• SNOWFLAKE: Snowflake algorithm (corresponding to the database bigint type and java Long type)
• UUID: uuid generation strategy (varchar and String types)

These two types alone cannot meet our needs, so sharding-jdbc provides us with an ShardingKeyGeneratorinterface to allow us to customize the primary key generation strategy.

Implementation: Here is an auto-incremented primary key generated by redis

1 Implement the ShardingKeyGenerator interface

@Component
@Slf4j
public class KeyGenerator implements ShardingKeyGenerator, ApplicationContextAware {
    
    
    @Getter
    @Setter
    private Properties properties;

    //必须设置为静态,否则为null
    public static RedisTemplate redisTemplate;


    @Override
    public Comparable<?> generateKey() {
    
    
        ValueOperations valueOp = redisTemplate.opsForValue();
        return  valueOp.increment("id");
    }


    //设置在yaml内的名字
    @Override
    public String getType() {
    
    
        return "auto_increment";
    }


    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
    
    
        redisTemplate = applicationContext.getBean(StringRedisTemplate.class);
    }
}

2 Configure the class just created

We need to create files under resource

META-INF\services\org.apache.shardingsphere.spi.keygen.ShardingKeyGenerator

And configure the host generator we just wrote

com.mt.config.Sharding.KeyGenerator

3 use

Just specify the name of the generator just in the configuration file

spring.shardingsphere.sharding.tables.user.key-generator.type=auto_increment

Custom table fragmentation strategy

Demonstrated above sub-table strategy is sharding-jdbc provides us with the inlinefragmentation of the rules, you can write 行表达式simple table partitioning strategy, for example, according to the modulo id, according to gender classification. But for some more complex partitioning strategies, row expressions may not meet our requirements, so we need to customize the table fragmentation strategy

Case: According to the current year, month and minute, insert data into different tables. For example, there are order_202001, order_202002, etc. in the database. We need to dynamically store the date generated by the order into different tables:

1 Create a database table

create table `order_202001`(
 id int primary key auto_increment,
 date datetime,
 price decimal(10,2),
 cmd varchar(200)
);

create table `order_202002`(
 id int primary key auto_increment,
 date datetime,
 price decimal(10,2),
 cmd varchar(200)
);

create table `order_202003`(
 id int primary key auto_increment,
 date datetime,
 price decimal(10,2),
 cmd varchar(200)
);

2 Create an entity object

@Data
@NoArgsConstructor
@AllArgsConstructor
@TableName("`order`")
public class Order {
    
    

    @TableId(type = IdType.AUTO)
    private Integer id;
    private Date date;
    private BigDecimal price;
    private String cmd;
}

3 Create a sharding logic class

@Component
public class OrderTableShardingAlgorithm implements PreciseShardingAlgorithm<Date> {
    
    

    @Override
    public String doSharding(Collection<String> collection, PreciseShardingValue<Date> preciseShardingValue) {
    
    
        
		//preciseShardingValue就是当前插入的字段值
        //collection 内就是所有的逻辑表
        //获取字段值
        Date time = preciseShardingValue.getValue();

        if(time == null){
    
    
            throw new UnsupportedOperationException("prec is null");
        }

        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM");
        String year =sdf.format(time);

        for (String tableName : collection) {
    
    //循环表名已确定使用哪张表
            String name = tableName.substring(tableName.length() - 6);
            if(year.equals(name)){
    
    
                return tableName;//返回要插入的逻辑表
            }
        }
        return null;
    }

}

#声明虚拟表
spring.shardingsphere.sharding.tables.order.actual-data-nodes=db1.order_$->{2000..2099}0$->{1..9},db1.order_$->{2000..2099}1$->{0..2}
#声明表内的主键
spring.shardingsphere.sharding.tables.order.key-generator.column=id
#声明主键生成策略
spring.shardingsphere.sharding.tables.order.key-generator.type=order_auto_increment

#声明根据哪个字段进行分片
spring.shardingsphere.sharding.tables.order.table-strategy.standard.sharding-column=date
#自定义分片规则类
spring.shardingsphere.sharding.tables.order.table-strategy.standard.precise-algorithm-class-name=com.mt.config.Sharding.OrderTableShardingAlgorithm

4 Add operation through mybatis-plus

@RestController
@RequestMapping("order")
public class OrderController {
    
    

    @Autowired
    private OrderMapper orderMapper;

    @GetMapping("add")
    public String add(Order order){
    
    
        orderMapper.insert(order);
        return "success";
    }
}

Reference blog: https://www.freesion.com/article/3959674242

Sub-library

Splitting the table horizontally through sharding-jdbc has been able to solve most of the problems. However, as the amount of data continues to increase, the performance of the table alone cannot be improved too much, and it must be solved by the database.

Scenario: There are 2 databases: sharding1, sharding2, each database has two tables user_1, user_2, sharding1 stores users younger than 18 years old, sharding2 puts users greater than or equal to 18, and then user_1 stores an odd-numbered id, and user_2 stores an even-numbered id

1 Create a database and table

create table user_1(
	id int primary key auto_increment,
	name varchar(200),
	age int,
	birthday datetime,
	cmd varchar(200),
	sex char(1)
);


create table user_2(
	id int primary key auto_increment,
	name varchar(200),
	age int,
	birthday datetime,
	cmd varchar(200),
	sex char(1)
);

2 Configure application.properties

# 配置逻辑库
spring.shardingsphere.datasource.names=db1,db2


# 配置第 1 个数据源
spring.shardingsphere.datasource.db1.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.db1.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db1.url=jdbc:mysql://localhost:3306/sharding1?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db1.username=root
spring.shardingsphere.datasource.db1.password=root

# 配置第 2 个数据源
spring.shardingsphere.datasource.db2.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.db2.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db2.url=jdbc:mysql://localhost:3306/sharding2?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db2.username=root
spring.shardingsphere.datasource.db2.password=root


# 配置 user 表规则
spring.shardingsphere.sharding.tables.user.actual-data-nodes=db$->{1..2}.user_$->{1..2}

# 配置 user 主键生成策略
spring.shardingsphere.sharding.tables.user.key-generator.column=id
spring.shardingsphere.sharding.tables.user.key-generator.type=user_auto

#配置 表分片策略
spring.shardingsphere.sharding.tables.user.table-strategy.inline.sharding-column=id
spring.shardingsphere.sharding.tables.user.table-strategy.inline.algorithm-expression=user_$->{ (id % 2)+1 }

#配置 库分片策略(主要就是增加了这个配置)
spring.shardingsphere.sharding.tables.user.database-strategy.inline.sharding-column=age
spring.shardingsphere.sharding.tables.user.database-strategy.inline.algorithm-expression=db$->{ (age>18 ? 2:1) }

The rest of the operation remains the same

Customized sub-library strategy

The custom sharding strategy is the same as the custom sharding, you only need to implement the PreciseShardingAlgorithm interface, and then configure the interface

public class UserDatabaseAlgorithm implements PreciseShardingAlgorithm<Integer> {
    
    

    @Override
    public String doSharding(Collection<String> collection, PreciseShardingValue<Integer> preciseShardingValue) {
    
    
		//collection 内是所有的数据库名称
        Integer value = preciseShardingValue.getValue();

        if(value.compareTo(18) >=0){
    
    
            return "db2";
        }else{
    
    
            return "db1";
        }
    }


}

Configuration:

#配置 库分片策略
spring.shardingsphere.sharding.tables.user.database-strategy.standard.sharding-column=age
#这里指向刚刚自定义的类
spring.shardingsphere.sharding.tables.user.database-strategy.standard.precise-algorithm-class-name=com.mt.config.UserDatabaseAlgorithm 

Public table

In some cases, we may return a table with the same name in two databases, and then the data must be consistent. When inserting data, insert data into the two databases at the same time, or both succeed or fail.

Scenario: Create a common table in two databases, sharding1 and sharding2

create table common(
	id int primary key,
	name varchar(20)
);

Configuration:

# 配置真实数据源
spring.shardingsphere.datasource.names=db1,db2



# 配置第 1 个数据源
spring.shardingsphere.datasource.db1.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.db1.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db1.url=jdbc:mysql://localhost:3306/sharding1?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db1.username=root
spring.shardingsphere.datasource.db1.password=root

# 配置第 2 个数据源
spring.shardingsphere.datasource.db2.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.db2.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db2.url=jdbc:mysql://localhost:3306/sharding2?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db2.username=root
spring.shardingsphere.datasource.db2.password=root


#配置2个数据库的公共表(重点)
spring.shardingsphere.sharding.broadcast-tables=common
#配置表主键
spring.shardingsphere.sharding.tables.common.key-generator.column=id
#配置主键生成策略
spring.shardingsphere.sharding.tables.common.key-generator.type=user_auto

use:

Create entity, mapper and service:

@Data
@NoArgsConstructor
@AllArgsConstructor
public class Common {
    
    
    private Integer id;
    private String name;
}


public interface CommonMapper extends BaseMapper<Common> {
    
    
}

@Service
public class CommonService {
    
    
    @Autowired
    private CommonMapper commonMapper;

    //这里必须加事务,否则可能出现一个数据库插入成功另一个不成功的情况
    @Transactional(rollbackFor = Exception.class)
    public void insert(Common common){
    
    
        commonMapper.insert(common);
    }
}

test:

@SpringBootTest
class ShardingJdbcApplicationTests {
    
    

    @Autowired
    private CommonService commonService;

    @Test
    void contextLoads() {
    
    
        commonService.insert(new Common(1,"abcdadfa"));
    }
}

Master-slave replication

Sharding-jdbc provides us with the dynamic switching function of Mysql master-slave replication database, only need to carry out the corresponding configuration to realize the dynamic switching data source under the master-slave replication environment

Scenario: mysql is configured with master-slave replication. The master library is sharding1, and the slave libraries are sharding2 and sharding3. Sharding1 is responsible for adding, deleting and modifying operations, sharding2 and sharding3 only accept query operations

1. Create a database table

create table student(
	id int primary key auto_increment,
	name varchar(200)
);

2. Configuration

# 配置真实数据源
spring.shardingsphere.datasource.names=db1,db2,db3



# 配置第 1 个数据源
spring.shardingsphere.datasource.db1.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.db1.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db1.url=jdbc:mysql://localhost:3306/sharding1?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db1.username=root
spring.shardingsphere.datasource.db1.password=root

# 配置第 2 个数据源
spring.shardingsphere.datasource.db2.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.db2.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db2.url=jdbc:mysql://localhost:3306/sharding2?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db2.username=root
spring.shardingsphere.datasource.db2.password=root

# 配置第 2 个数据源
spring.shardingsphere.datasource.db3.type=com.alibaba.druid.pool.DruidDataSource
spring.shardingsphere.datasource.db3.driver-class-name=com.mysql.cj.jdbc.Driver
spring.shardingsphere.datasource.db3.url=jdbc:mysql://localhost:3306/sharding3?characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
spring.shardingsphere.datasource.db2.username=root
spring.shardingsphere.datasource.db2.password=root

#重点
#配置 虚拟数据库ds, 真实的主库为db1
spring.shardingsphere.sharding.master-slave-rules.ds.master-data-source-name=db1
#配置 虚拟数据库ds, 真实的从库为db2
spring.shardingsphere.sharding.master-slave-rules.ds.slave-data-source-names=db2,db3
#配置student表的真实数据库(注意配置的是ds数据库)
spring.shardingsphere.sharding.tables.student.actual-data-nodes=ds.student

3 test

//编写bean mapper
@Data
@NoArgsConstructor
@AllArgsConstructor
public class Student {
    
    
    private Integer id;
    private String name;
}

public interface StudentMapper extends BaseMapper<Student> {
    
    }


//测试:
@Test
    public void tet2(){
    
    
        System.out.println("=====================插入数据库======================");
        studentMapper.insert(new Student(1,"abc"));

        System.out.println("=====================查询数据库======================");
        studentMapper.selectList(null);

        System.out.println("=====================查询数据库======================");
        studentMapper.selectList(null);


        System.out.println("=====================查询数据库======================");
        studentMapper.selectList(null);
    }