cache crash
occur
- The redis host hangs up, and Redis crashes completely
- For example, a large amount of data in the cache expires at the same time
solve
- Redis cache cluster achieves high availability
- master-slave + sentinel
- Redis Cluster
- Ehcache local cache + Hystrix or Ali sentinel current limit & downgrade
- Enable the Redis persistence mechanism aof/rdb to restore the cache cluster as soon as possible
cache penetration
-
what is
- Request to query a record, first redis and then mysql found that the record cannot be queried, but the request will hit the database every time, resulting in a surge in the background database pressure, this phenomenon is called cache penetration, this redis becomes It became a decoration. . . . . .
- Simply put, there is nothing at all, neither in the Redis cache nor in the database
-
harm
- After the first query, we generally have a write-back redis mechanism
- Redis was there when I came to check for the second time, and the occasional penetration phenomenon is generally irrelevant
solve
Scenario 1: Empty Object Cache or Default
- hacking or malicious attack
- Hackers will attack your system and use a non-existent id to query data, which will generate a large number of requests to the database for query. May cause your database to crash due to excessive pressure
- The same id hits your system: the first time you hit mysql, the empty object cache will return null the second time, to avoid mysql being attacked, and you don’t have to go around the database anymore
- The id is different and hits your system: due to the existence of empty object cache and cache writeback (depending on your business is not limited to death), more and more irrelevant keys in redis will be written (remember to set the redis expiration time)
Solution 2: Google Bloom filter Guava solves cache penetration
- The implementation of Bloom filter in Guava is relatively authoritative, so we don’t need to manually implement a Bloom filter in actual projects
- Guava's BloomFilter source code analysis
- Coding in action
-
建Module：bloomfilter_demo
-
Change POM
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-parent</artifactId> <version>2.3.10.RELEASE</version> <relativePath/> <!-- lookup parent from repository --> </parent> <groupId>com.atguigu.redis.bloomfilter</groupId> <artifactId>bloomfilter_demo</artifactId> <version>0.0.1-SNAPSHOT</version> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> <junit.version>4.12</junit.version> <log4j.version>1.2.17</log4j.version> <lombok.version>1.16.18</lombok.version> <mysql.version>5.1.47</mysql.version> <druid.version>1.1.16</druid.version> <mybatis.spring.boot.version>1.3.0</mybatis.spring.boot.version> </properties> <dependencies> <!--guava Google 开源的 Guava 中自带的布隆过滤器--> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>23.0</version> </dependency> <!-- redisson --> <dependency> <groupId>org.redisson</groupId> <artifactId>redisson</artifactId> <version>3.13.4</version> </dependency> <!--web+actuator--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-actuator</artifactId> </dependency> <!--SpringBoot与Redis整合依赖--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-redis</artifactId> </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-pool2</artifactId> </dependency> <!-- jedis --> <dependency> <groupId>redis.clients</groupId> <artifactId>jedis</artifactId> <version>3.1.0</version> </dependency> <!-- springboot-aop 技术--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-aop</artifactId> </dependency> <!--Mysql数据库驱动--> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.47</version> </dependency> <!--集成druid连接池--> <dependency> <groupId>com.alibaba</groupId> <artifactId>druid-spring-boot-starter</artifactId> <version>1.1.10</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>druid</artifactId> <version>${druid.version}</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>druid</artifactId> <version>${druid.version}</version> </dependency> <!--mybatis和springboot整合--> <dependency> <groupId>org.mybatis.spring.boot</groupId> <artifactId>mybatis-spring-boot-starter</artifactId> <version>${mybatis.spring.boot.version}</version> </dependency> <!-- 添加springboot对amqp的支持 --> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-amqp</artifactId> </dependency> <!--通用基础配置--> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>${junit.version}</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-devtools</artifactId> <scope>runtime</scope> <optional>true</optional> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>${log4j.version}</version> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <version>${lombok.version}</version> <optional>true</optional> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> </plugin> </plugins> </build> </project> -
write YML
server.port=6666 # ========================alibaba.druid相关配置===================== spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.url=jdbc:mysql://localhost:3306/db2021?useUnicode=true&characterEncoding=utf-8&useSSL=false spring.datasource.username=root spring.datasource.password=123456 spring.datasource.druid.test-while-idle=false # ========================redis相关配置===================== # Redis数据库索引（默认为0） spring.redis.database=0 # Redis服务器地址 spring.redis.host=192.168.111.147 # Redis服务器连接端口 spring.redis.port=6379 # Redis服务器连接密码（默认为空） spring.redis.password= # 连接池最大连接数（使用负值表示没有限制） 默认 8 spring.redis.lettuce.pool.max-active=8 # 连接池最大阻塞等待时间（使用负值表示没有限制） 默认 -1 spring.redis.lettuce.pool.max-wait=-1 # 连接池中的最大空闲连接 默认 8 spring.redis.lettuce.pool.max-idle=8 # 连接池中的最小空闲连接 默认 0 spring.redis.lettuce.pool.min-idle=0 -
main boot
-
business class
package com.learn.bloomfilter; import com.google.common.hash.BloomFilter; import com.google.common.hash.Funnels; import java.util.ArrayList; import java.util.List; /** * @author YSK * @since 2023/5/31 14:25 */ public class GuavaBloomFilterDemo { public static final int _1W = 10000; //布隆过滤器里预计要插入多少数据 public static int size = 100 * _1W; //误判率,它越小误判的个数也就越少(思考，是不是可以设置的无限小，没有误判岂不更好) //默认是0.03 public static double fpp = 0.01; /** * helloworld入门 */ public void bloomFilter() { // 创建布隆过滤器对象 BloomFilter<Integer> filter = BloomFilter.create(Funnels.integerFunnel(), 100); // 判断指定元素是否存在 System.out.println(filter.mightContain(1)); System.out.println(filter.mightContain(2)); // 将元素添加进布隆过滤器 filter.put(1); filter.put(2); System.out.println(filter.mightContain(1)); System.out.println(filter.mightContain(2)); } /** * 误判率演示+源码分析 */ public void bloomFilter2() { // 构建布隆过滤器 BloomFilter<Integer> bloomFilter = BloomFilter.create(Funnels.integerFunnel(), size, fpp); //1 先往布隆过滤器里面插入100万的样本数据 for (int i = 0; i < size; i++) { bloomFilter.put(i); } /* List<Integer> listSample = new ArrayList<>(size); //2 这100万的样本数据，是否都在布隆过滤器里面存在？ for (int i = 0; i < size; i++) { if (bloomFilter.mightContain(i)) { listSample.add(i); continue; } } System.out.println("存在的数量：" + listSample.size());*/ //3 故意取10万个不在过滤器里的值，看看有多少个会被认为在过滤器里,误判率演示 List<Integer> list = new ArrayList<>(10 * _1W); for (int i = size + 1; i < size + 100000; i++) { if (bloomFilter.mightContain(i)) { System.out.println(i + "\t" + "被误判了."); list.add(i); } } System.out.println("误判的数量：" + list.size()); } public static void main(String[] args) { new GuavaBloomFilterDemo().bloomFilter(); } }
-
- Bloom filter description
Solution 3: Redis bloom filter solves cache penetration
- Disadvantages of Guava: The implementation of the Bloom filter provided by Guava is still very good (if you want to know more about it, you can look at its source code implementation), but it has a major flaw that it can only be used on a single machine, and now the Internet is generally available. It is a distributed scene.
Case: whitelist filter
- Whitelist Architecture Description
- Misjudgment problem, but the probability is small and acceptable, and cannot be deleted from the Bloom filter
- All legal keys need to be put into the filter + redis, otherwise the data will return null
- code
package com.learn.bloomfilter;
import org.redisson.Redisson;
import org.redisson.api.RBloomFilter;
import org.redisson.api.RBucket;
import org.redisson.api.RedissonClient;
import org.redisson.client.codec.StringCodec;
import org.redisson.config.Config;
import org.springframework.data.redis.core.RedisTemplate;
import javax.annotation.Resource;
import java.util.concurrent.TimeUnit;
/**
* @author YSK
* @since 2023/5/31 15:48
*/
public class RedissonBloomFilterDemo {
public static final int _1W = 10000;
//布隆过滤器里预计要插入多少数据
public static int size = 100 * _1W;
//误判率,它越小误判的个数也就越少
public static double fpp = 0.03;
static RedissonClient redissonClient = null;//jedis
static RBloomFilter rBloomFilter = null;//redis版内置的布隆过滤器
@Resource
RedisTemplate redisTemplate;
static {
Config config = new Config();
config.useSingleServer().setAddress("redis://192.168.111.147:6379").setDatabase(0);
//构造redisson
redissonClient = Redisson.create(config);
//通过redisson构造rBloomFilter
rBloomFilter = redissonClient.getBloomFilter("phoneListBloomFilter", new StringCodec());
rBloomFilter.tryInit(size, fpp);
// 1测试 布隆过滤器有+redis有
//rBloomFilter.add("10086");
//redissonClient.getBucket("10086",new StringCodec()).set("chinamobile10086");
// 2测试 布隆过滤器有+redis无
//rBloomFilter.add("10087");
//3 测试 ，布隆过滤器无+redis无
}
private static String getPhoneListById(String IDNumber) {
String result = null;
if (IDNumber == null) {
return null;
}
//1 先去布隆过滤器里面查询
if (rBloomFilter.contains(IDNumber)) {
//2 布隆过滤器里有，再去redis里面查询
RBucket<String> rBucket = redissonClient.getBucket(IDNumber, new StringCodec());
result = rBucket.get();
if (result != null) {
return "i come from redis: " + result;
} else {
result = getPhoneListByMySQL(IDNumber);
if (result == null) {
return null;
}
// 重新将数据更新回redis
redissonClient.getBucket(IDNumber, new StringCodec()).set(result);
}
return "i come from mysql: " + result;
}
return result;
}
private static String getPhoneListByMySQL(String IDNumber) {
return "chinamobile" + IDNumber;
}
public static void main(String[] args) {
//String phoneListById = getPhoneListById("10086");
//String phoneListById = getPhoneListById("10087"); //请测试执行2次
String phoneListById = getPhoneListById("10088");
System.out.println("------查询出来的结果： " + phoneListById);
//暂停几秒钟线程
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
redissonClient.shutdown();
}
}
- Summarize
cache breakdown
what is
- When a large number of requests query a key at the same time, the key just fails at this time, which will cause a large number of requests to be sent to the database
- To put it simply, the hotspot key suddenly fails and beats mysql violently
- Harm: It will cause excessive database requests at a certain time, and the pressure will increase sharply.
solve
- For frequently accessed hotspot keys, simply do not set the expiration time
- Mutex exclusive lock prevents breakdown
- Multiple threads query this data in the database at the same time, then we can use a mutex to lock it on the first request to query data.
- When the other threads reach this point, they wait until the first thread gets the data and then cache it if they can't get the lock. Later threads come in and find that there is already a cache, so they go directly to the cache.
The high-concurrency Taobao Juhuasuan case landed
Analysis process
- 100% high concurrency, absolutely cannot be achieved with mysql
- First extract the data of participating activities in mysql into redis, and generally use timer scanning to decide whether to go online or go offline to cancel.
- Support paging function, 20 records per page
redis data type selection
Springboot+redis realizes high-concurrency Taobao Juhuasuan business
package com.atguigu.redis.service;
import cn.hutool.core.date.DateUtil;
import com.atguigu.redis.entities.Product;
import com.atguigu.redis.util.Constants;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
* @auther zzyy
* @create 2021-05-09 14:47
*/
@Service
@Slf4j
public class JHSTaskService
{
@Autowired
private RedisTemplate redisTemplate;
@PostConstruct
public void initJHS(){
log.info("启动定时器淘宝聚划算功能模拟.........."+DateUtil.now());
new Thread(() -> {
//模拟定时器，定时把数据库的特价商品，刷新到redis中
while (true){
//模拟从数据库读取100件特价商品，用于加载到聚划算的页面中
List<Product> list=this.products();
//采用redis list数据结构的lpush来实现存储
this.redisTemplate.delete(Constants.JHS_KEY);
//lpush命令
this.redisTemplate.opsForList().leftPushAll(Constants.JHS_KEY,list);
//间隔一分钟 执行一遍
try {
TimeUnit.MINUTES.sleep(1); } catch (InterruptedException e) {
e.printStackTrace(); }
log.info("runJhs定时刷新..............");
}
},"t1").start();
}
/**
* 模拟从数据库读取100件特价商品，用于加载到聚划算的页面中
*/
public List<Product> products() {
List<Product> list=new ArrayList<>();
for (int i = 1; i <=20; i++) {
Random rand = new Random();
int id= rand.nextInt(10000);
Product obj=new Product((long) id,"product"+i,i,"detail");
list.add(obj);
}
return list;
}
}
package com.atguigu.redis.controller;
import com.atguigu.redis.entities.Product;
import com.atguigu.redis.util.Constants;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @auther zzyy
* @create 2021-05-09 14:56
*/
@RestController
@Slf4j
@Api(description = "聚划算商品列表接口")
public class JHSProductController
{
@Autowired
private RedisTemplate redisTemplate;
/**
* 分页查询：在高并发的情况下，只能走redis查询，走db的话必定会把db打垮
* http://localhost:5555/swagger-ui.html#/jhs-product-controller/findUsingGET
*/
@RequestMapping(value = "/pruduct/find",method = RequestMethod.GET)
@ApiOperation("按照分页和每页显示容量，点击查看")
public List<Product> find(int page, int size) {
List<Product> list=null;
long start = (page - 1) * size;
long end = start + size - 1;
try {
//采用redis list数据结构的lrange命令实现分页查询
list = this.redisTemplate.opsForList().range(Constants.JHS_KEY, start, end);
if (CollectionUtils.isEmpty(list)) {
//TODO 走DB查询
}
log.info("查询结果：{}", list);
} catch (Exception ex) {
//这里的异常，一般是redis瘫痪 ，或 redis网络timeout
log.error("exception:", ex);
//TODO 走DB查询
}
return list;
}
}
Bug and hidden danger description
- After 1000 QPS leads to terrible cache breakdown
Further upgrade and reinforcement cases
- Timing polling, mutually exclusive update, differential expiration time
package com.atguigu.redis.service;
import cn.hutool.core.date.DateUtil;
import com.atguigu.redis.entities.Product;
import com.atguigu.redis.util.Constants;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
* @auther zzyy
* @create 2021-05-09 15:54
*/
@Service
@Slf4j
public class JHSABTaskService
{
@Autowired
private RedisTemplate redisTemplate;
@PostConstruct
public void initJHSAB(){
log.info("启动AB定时器计划任务淘宝聚划算功能模拟.........."+DateUtil.now());
new Thread(() -> {
//模拟定时器，定时把数据库的特价商品，刷新到redis中
while (true){
//模拟从数据库读取100件特价商品，用于加载到聚划算的页面中
List<Product> list=this.products();
//先更新B缓存
this.redisTemplate.delete(Constants.JHS_KEY_B);
this.redisTemplate.opsForList().leftPushAll(Constants.JHS_KEY_B,list);
this.redisTemplate.expire(Constants.JHS_KEY_B,20L,TimeUnit.DAYS);
//再更新A缓存
this.redisTemplate.delete(Constants.JHS_KEY_A);
this.redisTemplate.opsForList().leftPushAll(Constants.JHS_KEY_A,list);
this.redisTemplate.expire(Constants.JHS_KEY_A,15L,TimeUnit.DAYS);
//间隔一分钟 执行一遍
try {
TimeUnit.MINUTES.sleep(1); } catch (InterruptedException e) {
e.printStackTrace(); }
log.info("runJhs定时刷新..............");
}
},"t1").start();
}
/**
* 模拟从数据库读取100件特价商品，用于加载到聚划算的页面中
*/
public List<Product> products() {
List<Product> list=new ArrayList<>();
for (int i = 1; i <=20; i++) {
Random rand = new Random();
int id= rand.nextInt(10000);
Product obj=new Product((long) id,"product"+i,i,"detail");
list.add(obj);
}
return list;
}
}
package com.atguigu.redis.controller;
import com.atguigu.redis.entities.Product;
import com.atguigu.redis.util.Constants;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @auther zzyy
* @create 2021-05-09 15:58
*/
@RestController
@Slf4j
@Api(description = "聚划算商品列表接口AB")
public class JHSABProductController
{
@Autowired
private RedisTemplate redisTemplate;
@RequestMapping(value = "/pruduct/findab",method = RequestMethod.GET)
@ApiOperation("按照分页和每页显示容量，点击查看AB")
public List<Product> findAB(int page, int size) {
List<Product> list=null;
long start = (page - 1) * size;
long end = start + size - 1;
try {
//采用redis list数据结构的lrange命令实现分页查询
list = this.redisTemplate.opsForList().range(Constants.JHS_KEY_A, start, end);
if (CollectionUtils.isEmpty(list)) {
log.info("=========A缓存已经失效了，记得人工修补，B缓存自动延续5天");
//用户先查询缓存A(上面的代码)，如果缓存A查询不到（例如，更新缓存的时候删除了），再查询缓存B
this.redisTemplate.opsForList().range(Constants.JHS_KEY_B, start, end);
}
log.info("查询结果：{}", list);
} catch (Exception ex) {
//这里的异常，一般是redis瘫痪 ，或 redis网络timeout
log.error("exception:", ex);
//TODO 走DB查询
}
return list;
}
}
Summarize
