一 引言
最近项目在线上运行出现了一些难以复现的bug需要定位相应api的日志，通过nginx提供的api请求日志难以实现，于是在gateway通过全局过滤器记录api请求日志。
二 实现逻辑
- 接受到用户请求后，经过全局过滤器，检验是否开启相应的日志配置及相应的黑白名单配置
- 在gateway前置处理如记录当前请求开始时间，请求url，请求方法等
- 在gateway后置处理中获得相应的请求结果，响应状态码
- 记录当前请求日志，根据实际需求，通过mq异步处理工具持久化相应的日志（本案列作处理）
- 诊断请求结果，对请求异常，慢api等推送相应的消息给研发人人员
三 代码实现
定义相应的配置类
@Data
@RefreshScope
@Component
@ConfigurationProperties(prefix = LogProperties.PREFIX)
public class LogProperties {
public static final String PREFIX = "config.gateway.log.access";
/**
* 是否开启日志打印
*/
private Boolean enabled = true;
/**
* 忽略的pattern
*/
private List<String> ignoredPatterns;
private ApiAlarmConfiguration fail = new ApiAlarmConfiguration();
private SlowApiAlarmConfiguration slow = new SlowApiAlarmConfiguration();
/**
* 慢API报警配置
*/
@Data
public static class SlowApiAlarmConfiguration {
/**
* 是否开启API慢日志打印
*/
private boolean alarm = true;
/**
* 报警阈值 （单位：毫秒）
*/
private long threshold = 500;
}
/**
* API异常报警(根据http状态码判定）
*/
@Data
public static class ApiAlarmConfiguration {
/**
* 是否开启异常报警 默认关闭
*/
private boolean alarm = false;
/**
* 排除状态码
*/
private List<Integer> exclusion;
}
}
定义log实体
@Data
public class GatewayLog implements Serializable {
private static final long serialVersionUID = -3205904134722576668L;
/**
* 访问实例
*/
private String targetServer;
/**
* 请求路径
*/
private String requestPath;
/**
* 请求与方法
*/
private String method;
/**
* 请求协议
*/
private String schema;
/**
* 请求ip
*/
private String ip;
/**
* 请求时间
*/
private Date requestTime;
/**
* 请求参数
*/
private Map<String,String> queryParams;
/**
* 请求体
*/
private String requestBody;
/**
* 请求执行时间
*/
private Long executeTime;
/**
* 请求类型
*/
private String requestContentType;
/**
* 相应状态码
*/
private int code;
}
定义相应日志工厂及常量
public interface GatewayLogType {
/**
* 常规输出
*/
String APPLICATION_JSON_REQUEST = "applicationJsonRequest";
String FORM_DATA_REQUEST = "formDataRequest";
String BASIC_REQUEST = "basicRequest";
String NORMAL_REQUEST = "normalRequest";
/**
* 慢查询
*/
String SLOW = "slow";
/**
* 非200响应
*/
String FAIL = "fail";
}
@Slf4j
public class GatewayLogInfoFactory {
public static void log(String type, GatewayLog gatewayLog){
switch (type){
case GatewayLogType.APPLICATION_JSON_REQUEST:
log.info("[{}] {} {},route: {},status: {},excute: {} mills,requestBody: {}"
,gatewayLog.getIp()
,gatewayLog.getMethod()
,gatewayLog.getRequestPath()
,gatewayLog.getTargetServer()
,gatewayLog.getCode()
,gatewayLog.getExecuteTime()
,StrUtil.replace(gatewayLog.getRequestBody(), StrPool.LF,"")
);
break;
case GatewayLogType.FORM_DATA_REQUEST:
log.info("[{}] {} {},route: {},status: {},excute: {} mills,requestBody: {}"
,gatewayLog.getIp()
,gatewayLog.getMethod()
,gatewayLog.getRequestPath()
,gatewayLog.getTargetServer()
,gatewayLog.getCode()
,gatewayLog.getExecuteTime()
,StrUtil.replace(gatewayLog.getRequestBody(), StrPool.LF,"")
);
break;
case GatewayLogType.BASIC_REQUEST:
log.info("[{}] {} {},route: {},status: {},excute: {} mills,requestBody: {}"
,gatewayLog.getIp()
,gatewayLog.getMethod()
,gatewayLog.getRequestPath()
,gatewayLog.getTargetServer()
,gatewayLog.getCode()
,gatewayLog.getExecuteTime()
,StrUtil.replace(gatewayLog.getRequestBody(), StrPool.LF,"")
);
break;
case GatewayLogType.NORMAL_REQUEST:
log.info("[{}] {} {},route: {},status: {},excute: {} mills,queryParams: {}"
,gatewayLog.getIp()
,gatewayLog.getMethod()
,gatewayLog.getRequestPath()
,gatewayLog.getTargetServer()
,gatewayLog.getCode()
,gatewayLog.getExecuteTime()
,gatewayLog.getQueryParams()
);
break;
default:
break;
}
}
}
定义日志全局过滤器
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.text.StrPool;
import cn.hutool.core.util.StrUtil;
import com.zkjc.xlcp.common.notifier.core.Notifier;
import com.zkjc.xlcp.gateway.log.GatewayLogInfoFactory;
import com.zkjc.xlcp.gateway.log.LogProperties;
import com.zkjc.xlcp.gateway.log.constant.GatewayLogType;
import com.zkjc.xlcp.gateway.log.entity.GatewayLog;
import com.zkjc.xlcp.gateway.util.IpUtil;
import io.netty.util.internal.StringUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.reactivestreams.Publisher;
import org.springframework.cloud.gateway.filter.GatewayFilterChain;
import org.springframework.cloud.gateway.filter.GlobalFilter;
import org.springframework.cloud.gateway.filter.factory.rewrite.CachedBodyOutputMessage;
import org.springframework.cloud.gateway.route.Route;
import org.springframework.cloud.gateway.support.BodyInserterContext;
import org.springframework.cloud.gateway.support.ServerWebExchangeUtils;
import org.springframework.core.Ordered;
import org.springframework.core.ResolvableType;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferFactory;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.codec.HttpMessageReader;
import org.springframework.http.codec.multipart.FormFieldPart;
import org.springframework.http.codec.multipart.Part;
import org.springframework.http.server.reactive.ServerHttpRequest;
import org.springframework.http.server.reactive.ServerHttpRequestDecorator;
import org.springframework.http.server.reactive.ServerHttpResponse;
import org.springframework.http.server.reactive.ServerHttpResponseDecorator;
import org.springframework.stereotype.Component;
import org.springframework.util.AntPathMatcher;
import org.springframework.util.CollectionUtils;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.reactive.function.BodyInserter;
import org.springframework.web.reactive.function.BodyInserters;
import org.springframework.web.reactive.function.server.HandlerStrategies;
import org.springframework.web.reactive.function.server.ServerRequest;
import org.springframework.web.server.ServerWebExchange;
import org.springframework.web.util.UriComponentsBuilder;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.net.URI;
import java.util.*;
/**
* @author likun
* @date 2022年10月24日 9:52
*/
@Component
@RequiredArgsConstructor
@Slf4j
public class AccessLogFilter implements GlobalFilter, Ordered {
private final LogProperties logProperties;
private final List<HttpMessageReader<?>> messageReaders = HandlerStrategies.withDefaults().messageReaders();
/**
* default HttpMessageReader.
*/
private static final List<HttpMessageReader<?>> MESSAGE_READERS = HandlerStrategies.withDefaults().messageReaders();
private final AntPathMatcher antPathMatcher = new AntPathMatcher();
private final Notifier notifier;
/*
* 在CncloudRequestGlobalFilter后面执行 先清洗url在进行路径的日志的打印
* */
@Override
public int getOrder() {
return -100;
}
@Override
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
// 判断是否打开相应是日志配置 ingore配置校验
if (!logProperties.getEnabled()||hasIgnoredFlag(exchange,logProperties)){
return chain.filter(exchange);
}
// 获得请求上下文
GatewayLog gatewayLog = parseGateway(exchange);
ServerHttpRequest request = exchange.getRequest();
MediaType mediaType = request.getHeaders().getContentType();
if (Objects.isNull(mediaType)){
return writeNormalLog(exchange,chain,gatewayLog);
}
gatewayLog.setRequestContentType(mediaType.getType() + "/" + mediaType.getSubtype());
// 对不同的请求类型做相应的处理
if (MediaType.APPLICATION_JSON.isCompatibleWith(mediaType)){
return writeBodyLog(exchange,chain,gatewayLog);
}else if (MediaType.MULTIPART_FORM_DATA.isCompatibleWith(mediaType) || MediaType.APPLICATION_FORM_URLENCODED.isCompatibleWith(mediaType)){
return readFormData(exchange,chain,gatewayLog);
}else {
return writeBasicLog(exchange,chain,gatewayLog);
}
}
/**
* 校验白名单
* @param exchange
* @param logProperties
* @return
*/
private Boolean hasIgnoredFlag(ServerWebExchange exchange,LogProperties logProperties){
List<String> ignoredPatterns = logProperties.getIgnoredPatterns();
if (CollectionUtil.isEmpty(ignoredPatterns)){
return Boolean.FALSE;
}
ServerHttpRequest request = exchange.getRequest();
URI uri = request.getURI();
for (String pattern : ignoredPatterns) {
if (antPathMatcher.match(pattern,uri.getPath())){
return Boolean.TRUE;
}
}
return Boolean.FALSE;
}
/**
* 生成相应的报告并推送qq邮箱消息
*/
private void report(GatewayLog gatewayLog){
if (notifier==null){
return;
}
boolean reported = exceptionReport(gatewayLog);
if (!reported){
slowApiReport(gatewayLog);
}
}
/**
* 异常报警
* @param gatewayLog
* @return
*/
private Boolean exceptionReport(GatewayLog gatewayLog){
int code = gatewayLog.getCode();
if (code==HttpStatus.OK.value()){
return Boolean.FALSE;
}
LogProperties.ApiAlarmConfiguration apiAlarmConfiguration = logProperties.getFail();
if (!apiAlarmConfiguration.isAlarm()){
log.debug("api exception alarm disabled.");
return Boolean.FALSE;
}
if (!CollectionUtils.isEmpty(apiAlarmConfiguration.getExclusion()) && apiAlarmConfiguration.getExclusion().contains(code)) {
log.debug("status [{}] excluded.", code);
return Boolean.FALSE;
}
String alarmContent = String.format("【API异常】 请求ip:[{%s}],请求路由:[{%s}],请求地址:[{%s}],返回状态码:[{%d}],执行时间:%d ms",gatewayLog.getIp(),gatewayLog.getTargetServer(),gatewayLog.getRequestPath(),code,gatewayLog.getExecuteTime());
notifier.notify(alarmContent);
return Boolean.TRUE;
}
private Boolean slowApiReport(GatewayLog gatewayLog){
LogProperties.SlowApiAlarmConfiguration slowApiAlarmConfiguration = logProperties.getSlow();
long threshold = slowApiAlarmConfiguration.getThreshold();
if (gatewayLog.getExecuteTime()<threshold){
return Boolean.FALSE;
}
if (!slowApiAlarmConfiguration.isAlarm()) {
log.debug("slow api alarm disabled.");
return Boolean.FALSE;
}
String slowContent = String.format("【API执行时间过长,超过设定阈值】 请求ip:[{%s}],请求路由:[{%s}],请求地址:[{%s}],执行时间:%d ms",gatewayLog.getIp(),gatewayLog.getTargetServer(),gatewayLog.getRequestPath(),gatewayLog.getExecuteTime());
notifier.notify(slowContent);
return Boolean.TRUE;
}
/**
* 获得当前请求分发的路由
* @param exchange
* @return
*/
private Route getGatewayRoute(ServerWebExchange exchange) {
return exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_ROUTE_ATTR);
}
private GatewayLog parseGateway(ServerWebExchange exchange){
ServerHttpRequest request = exchange.getRequest();
String requestPath = request.getPath().pathWithinApplication().value();
Route route = getGatewayRoute(exchange);
String ip = IpUtil.getIpAddress(request);
GatewayLog gatewayLog = new GatewayLog();
gatewayLog.setSchema(request.getURI().getScheme());
gatewayLog.setMethod(request.getMethodValue());
gatewayLog.setRequestPath(requestPath);
gatewayLog.setTargetServer(route.getId());
gatewayLog.setIp(ip);
gatewayLog.setRequestTime(new Date());
return gatewayLog;
}
private Mono writeNormalLog(ServerWebExchange exchange, GatewayFilterChain chain, GatewayLog gatewayLog){
return chain.filter(exchange).then(Mono.fromRunnable(()->{
ServerHttpResponse response = exchange.getResponse();
int value = response.getStatusCode().value();
gatewayLog.setCode(value);
long executeTime = DateUtil.between(gatewayLog.getRequestTime(), new Date(), DateUnit.MS);
gatewayLog.setExecuteTime(executeTime);
ServerHttpRequest request = exchange.getRequest();
MultiValueMap<String, String> queryParams = request.getQueryParams();
Map<String, String> paramsMap = new HashMap<>();
if (CollectionUtil.isNotEmpty(queryParams)) {
for (Map.Entry<String, List<String>> entry : queryParams.entrySet()) {
paramsMap.put(entry.getKey(), StrUtil.join(StrPool.COMMA,entry.getValue()));
}
}
gatewayLog.setQueryParams(paramsMap);
GatewayLogInfoFactory.log(GatewayLogType.NORMAL_REQUEST,gatewayLog);
// 推送相应的报告
report(gatewayLog);
}));
}
/**
* 解决 request body 只能读取一次问题，
* 参考: org.springframework.cloud.gateway.filter.factory.rewrite.ModifyRequestBodyGatewayFilterFactory
* @param exchange
* @param chain
* @param gatewayLog
* @return
*/
@SuppressWarnings("unchecked")
private Mono writeBodyLog(ServerWebExchange exchange, GatewayFilterChain chain, GatewayLog gatewayLog) {
ServerRequest serverRequest = ServerRequest.create(exchange, messageReaders);
Mono<String> modifiedBody = serverRequest.bodyToMono(String.class).flatMap(body -> {
gatewayLog.setRequestBody(body);
return Mono.just(body);
});
// 通过 BodyInserter 插入 body(支持修改body), 避免 request body 只能获取一次
BodyInserter bodyInserter = BodyInserters.fromPublisher(modifiedBody, String.class);
HttpHeaders headers = new HttpHeaders();
headers.putAll(exchange.getRequest().getHeaders());
// the new content type will be computed by bodyInserter
// and then set in the request decorator
headers.remove(HttpHeaders.CONTENT_LENGTH);
CachedBodyOutputMessage outputMessage = new CachedBodyOutputMessage(exchange, headers);
return bodyInserter.insert(outputMessage, new BodyInserterContext()).then(Mono.defer(() -> {
// 重新封装请求
ServerHttpRequest decoratedRequest = requestDecorate(exchange, headers, outputMessage); // 记录响应日志
ServerHttpResponseDecorator decoratedResponse = recordResponseLog(exchange, gatewayLog); // 记录普通的
return chain.filter(exchange.mutate().request(decoratedRequest).response(decoratedResponse).build()).then(Mono.fromRunnable(() -> {
// 打印日志
GatewayLogInfoFactory.log(GatewayLogType.APPLICATION_JSON_REQUEST,gatewayLog);
// 推送相应的报告
report(gatewayLog);
}));
}));
}
/**
* 读取form-data数据
* @param exchange
* @param chain
* @param accessLog
* @return
*/
private Mono<Void> readFormData(ServerWebExchange exchange, GatewayFilterChain chain, GatewayLog accessLog) {
return DataBufferUtils.join(exchange.getRequest().getBody()).flatMap(dataBuffer -> {
DataBufferUtils.retain(dataBuffer);
final Flux<DataBuffer> cachedFlux = Flux.defer(() -> Flux.just(dataBuffer.slice(0, dataBuffer.readableByteCount())));
final ServerHttpRequest mutatedRequest = new ServerHttpRequestDecorator(exchange.getRequest()) {
@Override
public Flux<DataBuffer> getBody() {
return cachedFlux;
}
@Override
public MultiValueMap<String, String> getQueryParams() {
return UriComponentsBuilder.fromUri(exchange.getRequest().getURI()).build().getQueryParams();
}
};
final HttpHeaders headers = exchange.getRequest().getHeaders();
if (headers.getContentLength() == 0) {
return chain.filter(exchange);
}
ResolvableType resolvableType;
if (MediaType.MULTIPART_FORM_DATA.isCompatibleWith(headers.getContentType())) {
resolvableType = ResolvableType.forClassWithGenerics(MultiValueMap.class, String.class, Part.class);
} else {
//解析 application/x-www-form-urlencoded
resolvableType = ResolvableType.forClass(String.class);
}
return MESSAGE_READERS.stream().filter(reader -> reader.canRead(resolvableType, mutatedRequest.getHeaders().getContentType())).findFirst().orElseThrow(() -> new IllegalStateException("no suitable HttpMessageReader.")).readMono(resolvableType, mutatedRequest, Collections.emptyMap()).flatMap(resolvedBody -> {
if (resolvedBody instanceof MultiValueMap) {
LinkedMultiValueMap map = (LinkedMultiValueMap) resolvedBody;
if (CollectionUtil.isNotEmpty(map)) {
StringBuilder builder = new StringBuilder();
final Part bodyPartInfo = (Part) ((MultiValueMap) resolvedBody).getFirst("body");
if (bodyPartInfo instanceof FormFieldPart) {
String body = ((FormFieldPart) bodyPartInfo).value();
builder.append("body=").append(body);
}
accessLog.setRequestBody(builder.toString());
}
} else {
accessLog.setRequestBody((String) resolvedBody);
}
//获取响应体
ServerHttpResponseDecorator decoratedResponse = recordResponseLog(exchange, accessLog);
return chain.filter(exchange.mutate().request(mutatedRequest).response(decoratedResponse).build()).then(Mono.fromRunnable(() -> {
// 打印日志
// 打印响应的日志
GatewayLogInfoFactory.log(GatewayLogType.FORM_DATA_REQUEST,accessLog);
// 推送相应的报告
report(accessLog);
}));
});
});
}
private Mono<Void> writeBasicLog(ServerWebExchange exchange, GatewayFilterChain chain, GatewayLog accessLog) {
return DataBufferUtils.join(exchange.getRequest().getBody()).flatMap(dataBuffer -> {
DataBufferUtils.retain(dataBuffer);
final Flux<DataBuffer> cachedFlux = Flux.defer(() -> Flux.just(dataBuffer.slice(0, dataBuffer.readableByteCount())));
final ServerHttpRequest mutatedRequest = new ServerHttpRequestDecorator(exchange.getRequest()) {
@Override
public Flux<DataBuffer> getBody() {
return cachedFlux;
}
@Override
public MultiValueMap<String, String> getQueryParams() {
return UriComponentsBuilder.fromUri(exchange.getRequest().getURI()).build().getQueryParams();
}
};
StringBuilder builder = new StringBuilder();
MultiValueMap<String, String> queryParams = exchange.getRequest().getQueryParams();
if (CollectionUtil.isNotEmpty(queryParams)) {
for (Map.Entry<String, List<String>> entry : queryParams.entrySet()) {
builder.append(entry.getKey()).append("=").append(entry.getValue()).append(StrPool.COMMA);
}
}
accessLog.setRequestBody(builder.toString()); //获取响应体
ServerHttpResponseDecorator decoratedResponse = recordResponseLog(exchange, accessLog);
return chain.filter(exchange.mutate().request(mutatedRequest).response(decoratedResponse).build()).then(Mono.fromRunnable(() -> {
// 打印日志
GatewayLogInfoFactory.log(GatewayLogType.BASIC_REQUEST,accessLog);
// 推送相应的报告
report(accessLog);
}));
});
}
/**
* 请求装饰器，重新计算 headers
* @param exchange
* @param headers
* @param outputMessage
* @return
*/
private ServerHttpRequestDecorator requestDecorate(ServerWebExchange exchange, HttpHeaders headers, CachedBodyOutputMessage outputMessage) {
return new ServerHttpRequestDecorator(exchange.getRequest()) {
@Override
public HttpHeaders getHeaders() {
long contentLength = headers.getContentLength();
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.putAll(super.getHeaders());
if (contentLength > 0) {
httpHeaders.setContentLength(contentLength);
} else {
httpHeaders.set(HttpHeaders.TRANSFER_ENCODING, "chunked");
}
return httpHeaders;
}
@Override
public Flux<DataBuffer> getBody() {
return outputMessage.getBody();
}
};
}
/**
* 记录响应日志
* 通过 DataBufferFactory 解决响应体分段传输问题。
*/
private ServerHttpResponseDecorator recordResponseLog(ServerWebExchange exchange, GatewayLog gatewayLog) {
ServerHttpResponse response = exchange.getResponse();
DataBufferFactory bufferFactory = response.bufferFactory();
return new ServerHttpResponseDecorator(response) {
@Override
public Mono<Void> writeWith(Publisher<? extends DataBuffer> body) {
if (body instanceof Flux) {
// 计算执行时间
long executeTime = DateUtil.between(gatewayLog.getRequestTime(), new Date(), DateUnit.MS);
gatewayLog.setExecuteTime(executeTime);
// 获取响应类型，如果是 json 就打印
String originalResponseContentType = exchange.getAttribute(ServerWebExchangeUtils.ORIGINAL_RESPONSE_CONTENT_TYPE_ATTR);//
gatewayLog.setCode(this.getStatusCode().value());
//
if (Objects.equals(this.getStatusCode(), HttpStatus.OK)
&& !StringUtil.isNullOrEmpty(originalResponseContentType)
&& originalResponseContentType.contains("application/json")) {
Flux<? extends DataBuffer> fluxBody = Flux.from(body);
return super.writeWith(fluxBody.buffer().map(dataBuffers -> {
// 合并多个流集合，解决返回体分段传输
DataBufferFactory dataBufferFactory = new DefaultDataBufferFactory();
DataBuffer join = dataBufferFactory.join(dataBuffers);
byte[] content = new byte[join.readableByteCount()];
// 释放掉内存
join.read(content);
DataBufferUtils.release(join);
return bufferFactory.wrap(content);
}));
}else {
}
}
return super.writeWith(body);
}
};
}
}