-
- split horizon
1. Horizontal sub-library
1), concept:
Based on the fields and according to a certain strategy, the data in one library is split into multiple libraries.
2), the result
The structure of each library is the same; the data is different;
The union of all libraries is the full amount of data;
2. Horizontal table
1), concept
Based on fields and according to a certain strategy, the data in one table is split into multiple tables.
2), the result
The structure of each table is the same; the data is different;
The union of all tables is the full amount of data;
-
- Shard-jdbc middleware
- architecture diagram
2. Features
1), Sharding-JDBC directly encapsulates the JDBC API, and the cost of old code migration is almost zero.
2), suitable for any Java-based ORM framework, such as Hibernate, Mybatis, etc.
3) It can be based on any third-party database connection pool, such as DBCP, C3P0, BoneCP, Druid, etc.
4) The service is provided in the form of a jar package, without a proxy layer, without additional deployment, and without other dependencies.
5) The sharding strategy is flexible, and can support multi-dimensional sharding such as equal sign, between, in, etc., and can also support multiple sharding keys.
6), SQL parsing function is complete, support aggregation, grouping, sorting, limit, or and other queries.
-
- project demo
- project structure
2. Database configuration
3. Core code block
Data source configuration file
| spring: datasource: # Data source: shard_one dataOne: type: com.alibaba.druid.pool.DruidDataSource druid: driverClassName: com.mysql.jdbc.Driver url: jdbc:mysql://localhost:3306/shard_one?useUnicode=true&characterEncoding=UTF8&zeroDateTimeBehavior=convertToNull&useSSL=false username: root password: 123 initial-size: 10 max-active: 100 min-idle: 10 max-wait: 60000 pool-prepared-statements: true max-pool-prepared-statement-per-connection-size: 20 time-between-eviction-runs-millis: 60000 min-evictable-idle-time-thousands: 300000 max-evictable-idle-time-millis: 60000 validation-query: SELECT 1 FROM DUAL # validation-query-timeout: 5000 test-on-borrow: false test-on-return: false test-while-idle: true connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 # Data source: shard_two dataTwo: type: com.alibaba.druid.pool.DruidDataSource druid: driverClassName: com.mysql.jdbc.Driver url: jdbc:mysql://localhost:3306/shard_two?useUnicode=true&characterEncoding=UTF8&zeroDateTimeBehavior=convertToNull&useSSL=false username: root password: 123 initial-size: 10 max-active: 100 min-idle: 10 max-wait: 60000 pool-prepared-statements: true max-pool-prepared-statement-per-connection-size: 20 time-between-eviction-runs-millis: 60000 min-evictable-idle-time-millis: 300000 max-evictable-idle-time-millis: 60000 validation-query: SELECT 1 FROM DUAL # validation-query-timeout: 5000 test-on-borrow: false test-on-return: false test-while-idle: true connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 # 数据源：shard_three dataThree: type: com.alibaba.druid.pool.DruidDataSource druid: driverClassName: com.mysql.jdbc.Driver url: jdbc:mysql://localhost:3306/shard_three?useUnicode=true&characterEncoding=UTF8&zeroDateTimeBehavior=convertToNull&useSSL=false username: root password: 123 initial-size: 10 max-active: 100 min-idle: 10 max-wait: 60000 pool-prepared-statements: true max-pool-prepared-statement-per-connection-size: 20 time-between-eviction-runs-millis: 60000 min-evictable-idle-time-millis: 300000 max-evictable-idle-time-millis: 60000 validation-query: SELECT 1 FROM DUAL # validation-query-timeout: 5000 test-on-borrow: false test-on-return: false test-while-idle: true connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 |
数据库分库策略
| /** * 数据库映射计算 */ public class DataSourceAlg implements PreciseShardingAlgorithm<String> { private static Logger LOG = LoggerFactory.getLogger(DataSourceAlg.class); @Override public String doSharding(Collection<String> names, PreciseShardingValue<String> value) { LOG.debug("分库算法参数 {},{}",names,value); int hash = HashUtil.rsHash(String.valueOf(value.getValue())); return "ds_" + ((hash % 2) + 2) ; } } |
数据表1分表策略
| /** * 分表算法 */ public class TableOneAlg implements PreciseShardingAlgorithm<String> { private static Logger LOG = LoggerFactory.getLogger(TableOneAlg.class); /** * 该表每个库分5张表 */ @Override public String doSharding(Collection<String> names, PreciseShardingValue<String> value) { LOG.debug("分表算法参数 {},{}",names,value); int hash = HashUtil.rsHash(String.valueOf(value.getValue())); return "table_one_" + (hash % 5+1); } } |
数据表2分表策略
| /** * 分表算法 */ public class TableTwoAlg implements PreciseShardingAlgorithm<String> { private static Logger LOG = LoggerFactory.getLogger(TableTwoAlg.class); /** * 该表每个库分5张表 */ @Override public String doSharding(Collection<String> names, PreciseShardingValue<String> value) { LOG.debug("分表算法参数 {},{}",names,value); int hash = HashUtil.rsHash(String.valueOf(value.getValue())); return "table_two_" + (hash % 5+1); } } |
数据源集成配置
| /** * 数据库分库分表配置 */ @Configuration public class ShardJdbcConfig { // 省略了 druid 配置，源码中有 /** * Shard-JDBC 分库配置 */ @Bean public DataSource dataSource (@Autowired DruidDataSource dataOneSource, @Autowired DruidDataSource dataTwoSource, @Autowired DruidDataSource dataThreeSource) throws Exception { ShardingRuleConfiguration shardJdbcConfig = new ShardingRuleConfiguration(); shardJdbcConfig.getTableRuleConfigs().add(getTableRule01()); shardJdbcConfig.getTableRuleConfigs().add(getTableRule02()); shardJdbcConfig.setDefaultDataSourceName("ds_0"); Map<String,DataSource> dataMap = new LinkedHashMap<>() ; dataMap.put("ds_0",dataOneSource) ; dataMap.put("ds_2",dataTwoSource) ; dataMap.put("ds_3",dataThreeSource) ; Properties prop = new Properties(); return ShardingDataSourceFactory.createDataSource(dataMap, shardJdbcConfig, new HashMap<>(), prop); } /** * Shard-JDBC 分表配置 */ private static TableRuleConfiguration getTableRule01() { TableRuleConfiguration result = new TableRuleConfiguration(); result.setLogicTable("table_one"); result.setActualDataNodes("ds_${2..3}.table_one_${1..5}"); result.setDatabaseShardingStrategyConfig(new StandardShardingStrategyConfiguration("phone", new DataSourceAlg())); result.setTableShardingStrategyConfig(new StandardShardingStrategyConfiguration("phone", new TableOneAlg())); return result; } private static TableRuleConfiguration getTableRule02() { TableRuleConfiguration result = new TableRuleConfiguration(); result.setLogicTable("table_two"); result.setActualDataNodes("ds_${2..3}.table_two_${1..5}"); result.setDatabaseShardingStrategyConfig(new StandardShardingStrategyConfiguration("phone", new DataSourceAlg())); result.setTableShardingStrategyConfig(new StandardShardingStrategyConfiguration("phone", new TableTwoAlg())); return result; } } |
测试代码执行流程
| @RestController public class ShardController { @Resource private ShardService shardService ; /** * 1、建表流程 */ @RequestMapping("/createTable") public String createTable (){ shardService.createTable(); return "success" ; } /** * 2、生成表 table_one 数据 */ @RequestMapping("/insertOne") public String insertOne (){ shardService.insertOne(); return "SUCCESS" ; } /** * 3、生成表 table_two 数据 */ @RequestMapping("/insertTwo") public String insertTwo (){ shardService.insertTwo(); return "SUCCESS" ; } /** * 4、查询表 table_one 数据 */ @RequestMapping("/selectOneByPhone/{phone}") public TableOne selectOneByPhone (@PathVariable("phone") String phone){ return shardService.selectOneByPhone(phone); } /** * 5、查询表 table_one 数据 */ @RequestMapping("/selectTwoByPhone/{phone}") public TableTwo selectTwoByPhone (@PathVariable("phone") String phone){ return shardService.selectTwoByPhone(phone); } } |