前言
前几天公司生产环境一个服务由于流量上升触发了Sentinel的流控机制,然后用户反馈访问慢,定位发现是task定时任务导致,后面task优化之后发布,流量恢复正常。
这是一个再正常不过的生产问题,可能大部分同学都经历过,经历过的大多数是解决问题之后就不了了之,导致事故还有再次发生的可能,最终对用户造成了不好的体验。所以我觉得所有的生产问题都需要进行复盘,当然复盘的目的不是为了追责,而是防止下次再发生同样的错误。那我们就简单分析一下这个问题,首先肯定是业务层面的疏漏导致task发出不合理的大量请求,其二我们的流控只是简单粗暴的流控,没有更好的预警措施,导致影响到用户之后我们才知晓(即流控或熔断已经触发)。
那我们的解决方案呢?首先肯定是业务层面的预防,但这不是本文要说的重点,这里不展开讨论了。其次就是预警,就是我们能否在快要触发流控之前知晓,然后报警到相关负责人提前介入处理,防止触发流控熔断。当然也不能完全避免,但是总比流控或熔断触发之后在报警要好得多。
由于之前流控用的阿里的Sentinel,所以本文介绍的具体实现是用Sentinel的自定义slot功能,这个自定义slot卡槽在Sentinel官方文档里面就一句话带过,然后加上一个demo代码,我在使用的过程中也遇到过不少坑,所以分享一下结果给大家。
如果大家对Sentinel不是很了解,可以先去github先了解简单试用一下在阅读本文。github地址:https://github.com/alibaba/Sentinel
如果想熟悉自定义slot功能建议了解一下Sentinel的工作原理:https://github.com/alibaba/Sentinel/wiki/Sentinel%E5%B7%A5%E4%BD%9C%E4%B8%BB%E6%B5%81%E7%A8%8B
还有源码中的demo对于自定义slot的写法:https://github.com/alibaba/Sentinel/tree/master/sentinel-demo/sentinel-demo-slot-chain-spi
具体实现
下面介绍下Sentinel预警功能的相关实现,使用的前提是你的系统已经在用Sentinel的流控或熔断等功能。
- 自定义CustomSlotChainBuilder实现SlotChainBuilder接口,这里主要是把我们自定义的Slot加到SlotChain这个链中
import com.alibaba.csp.sentinel.slotchain.ProcessorSlotChain; import com.alibaba.csp.sentinel.slotchain.SlotChainBuilder; import com.alibaba.csp.sentinel.slots.DefaultSlotChainBuilder; import com.qiaofang.tortoise.gateway.component.ApplicationContextUtil; import com.qiaofang.tortoise.gateway.config.SentinelProperties; import org.springframework.stereotype.Component; import javax.annotation.Resource; /** * 自定义slot * * @author chenhao */ public class CustomSlotChainBuilder implements SlotChainBuilder { @Override public ProcessorSlotChain build() { ProcessorSlotChain chain = new DefaultSlotChainBuilder().build(); SentinelProperties sentinelProperties = (SentinelProperties) ApplicationContextUtil.getContext().getBean("sentinelProperties"); chain.addLast(new FlowEarlyWarningSlot(sentinelProperties)); chain.addLast(new DegradeEarlyWarningSlot(sentinelProperties)); return chain; } }
2.自定义FlowEarlyWarningSlot、DegradeEarlyWarningSlot 流控熔断2个预警slot
import com.alibaba.csp.sentinel.context.Context; import com.alibaba.csp.sentinel.node.DefaultNode; import com.alibaba.csp.sentinel.slotchain.AbstractLinkedProcessorSlot; import com.alibaba.csp.sentinel.slotchain.ResourceWrapper; import com.alibaba.csp.sentinel.slots.block.flow.FlowRule; import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleChecker; import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager; import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleUtil; import com.alibaba.csp.sentinel.util.AssertUtil; import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; import java.util.List; import java.util.Map; import java.util.stream.Collectors; /** * 流控预警slot * * @author chenhao */ public class FlowEarlyWarningSlot2 extends AbstractLinkedProcessorSlot<DefaultNode> { /** * log */ private Logger logger = LoggerFactory.getLogger(this.getClass()); private final FlowRuleChecker checker; public FlowEarlyWarningSlot2() { this(new FlowRuleChecker()); } /** * Package-private for test. * * @param checker flow rule checker * @since 1.6.1 */ FlowEarlyWarningSlot2(FlowRuleChecker checker) { AssertUtil.notNull(checker, "flow checker should not be null"); this.checker = checker; } private List<FlowRule> getRuleProvider(String resource) { // Flow rule map should not be null. List<FlowRule> rules = FlowRuleManager.getRules(); List<FlowRule> earlyWarningRuleList = Lists.newArrayList(); for (FlowRule rule : rules) { FlowRule earlyWarningRule = new FlowRule(); BeanUtils.copyProperties(rule, earlyWarningRule); /** * 这里是相当于把规则阈值改成原来的80%,达到提前预警的效果, * 这里建议把0.8做成配置 */ earlyWarningRule.setCount(rule.getCount() * 0.8); earlyWarningRuleList.add(earlyWarningRule); } Map<String, List<FlowRule>> flowRules = FlowRuleUtil.buildFlowRuleMap(earlyWarningRuleList); return flowRules.get(resource); } /** * get origin rule * * @param resource * @return */ private FlowRule getOriginRule(String resource) { List<FlowRule> originRule = FlowRuleManager.getRules().stream().filter(flowRule -> flowRule.getResource().equals(resource)).collect(Collectors.toList()); if (CollectionUtils.isEmpty(originRule)) { return null; } return originRule.get(0); } @Override public void entry(Context context, ResourceWrapper resourceWrapper, DefaultNode node, int count, boolean prioritized, Object... args) throws Throwable { String resource = context.getCurEntry().getResourceWrapper().getName(); List<FlowRule> rules = getRuleProvider(resource); if (rules != null) { for (FlowRule rule : rules) { //这里取到的规则都是配置阈值的80%,这里如果检查到阈值了,说明就是到了真实阈值的80%,既可以发报警给对应负责人了 if (!checker.canPassCheck(rule, context, node, count, prioritized)) { FlowRule originRule = getOriginRule(resource); String originRuleCount = originRule == null ? "未知" : String.valueOf(originRule.getCount()); logger.info("FlowEarlyWarning:服务{}目前的流量指标已经超过{},接近配置的流控阈值:{},", resource, rule.getCount(), originRuleCount); //TODO 报警功能自行实现 break; } } } fireEntry(context, resourceWrapper, node, count, prioritized, args); } @Override public void exit(Context context, ResourceWrapper resourceWrapper, int count, Object... args) { fireExit(context, resourceWrapper, count, args); } }
import com.alibaba.csp.sentinel.context.Context; import com.alibaba.csp.sentinel.node.DefaultNode; import com.alibaba.csp.sentinel.slotchain.AbstractLinkedProcessorSlot; import com.alibaba.csp.sentinel.slotchain.ResourceWrapper; import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRule; import com.alibaba.csp.sentinel.slots.block.degrade.DegradeRuleManager; import com.alibaba.csp.sentinel.util.AssertUtil; import com.google.common.collect.Lists; import com.qiaofang.tortoise.gateway.config.SentinelProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; import java.util.List; import java.util.stream.Collectors; /** * 熔断预警slot * * @author chenhao */ public class DegradeEarlyWarningSlot2 extends AbstractLinkedProcessorSlot<DefaultNode> { /** * log */ private Logger logger = LoggerFactory.getLogger(this.getClass()); /** * 与流控基本一致 就是取原规则的方式不一样 * @param resource * @return */ private List<DegradeRule> getRuleProvider(String resource) { // Flow rule map should not be null. List<DegradeRule> rules = DegradeRuleManager.getRules(); List<DegradeRule> earlyWarningRuleList = Lists.newArrayList(); for (DegradeRule rule : rules) { DegradeRule earlyWarningRule = new DegradeRule(); BeanUtils.copyProperties(rule, earlyWarningRule); earlyWarningRule.setCount(rule.getCount() * 0.8); earlyWarningRuleList.add(earlyWarningRule); } return earlyWarningRuleList.stream().filter(rule -> resource.equals(rule.getResource())).collect(Collectors.toList()); } /** * get origin rule * * @param resource * @return */ private DegradeRule getOriginRule(String resource) { List<DegradeRule> originRule = DegradeRuleManager.getRules().stream().filter(rule -> rule.getResource().equals(resource)).collect(Collectors.toList()); if (CollectionUtils.isEmpty(originRule)) { return null; } return originRule.get(0); } @Override public void entry(Context context, ResourceWrapper resourceWrapper, DefaultNode node, int count, boolean prioritized, Object... args) throws Throwable { String resource = context.getCurEntry().getResourceWrapper().getName(); List<DegradeRule> rules = getRuleProvider(resource); if (rules != null) { for (DegradeRule rule : rules) { if (!rule.passCheck(context, node, count)) { DegradeRule originRule = getOriginRule(resource); String originRuleCount = originRule == null ? "未知" : String.valueOf(originRule.getCount()); logger.info("DegradeEarlyWarning:服务{}目前的熔断指标已经超过{},接近配置的熔断阈值:{},", resource, rule.getCount(), originRuleCount); break; } } } fireEntry(context, resourceWrapper, node, count, prioritized, args); } @Override public void exit(Context context, ResourceWrapper resourceWrapper, int count, Object... args) { fireExit(context, resourceWrapper, count, args); }
3.在resources文件夹下面新增META-INF.services文件夹,新增文件com.alibaba.csp.sentinel.slotchain.SlotChainBuilder(文件名无所谓) 内容如下
# 这里写你CustomSlotChainBuilder的完整包路径 com.xxx.sentinel.CustomSlotChainBuilder
到这里基本上就可以了,用的过程中还是遇到挺多坑的,简单列举几个吧
- 直接改FlowRule的count属性是不行的,因为底层验证规则的时候用的是FlowRule的controller属性,这个属性又是私有的,所以直接先拿到原始的配置后通过FlowRuleUtil重新生成
- 调试过程中,DefaultNode里面很多方法的值是都是1s内有效,从方法A debug到方法B 可能值就没了,当时一脸懵逼
写在最后
本人很少写这种技术博客,所以有什么问题,或者不严谨的地方,大家可以提出来,求轻点喷我哈哈哈