项目地址：https://github.com/lcy19930619/api-center
适用场景
1、有很多后端服务，
2、尚未接入或无法接入分布式框架，
3、需要集中统一管理接口
整体设计思路：
转发网关需要分成以下五部分来考虑
1. 客户端
在网关上，需要将所有的请求转发给每个真实的服务，所以网关是客户端，考虑到客户端的转发性能问题，直接使用的是webflux+netty client pool
2. 服务端
服务端需要放在每个服务中，应尽可能减少侵入性，而且配置尽可能少，便于集成
对spring的支持
支持spring boot和spring mvc
spring boot可以使用spi机制进行支持
package net.jlxxw.apicenter.facade.runner;
import net.jlxxw.apicenter.facade.properties.ApiCenterClientProperties;
import net.jlxxw.apicenter.facade.remote.AbstractRemoteManager;
import net.jlxxw.apicenter.facade.scanner.MethodScanner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
/**
* @author zhanxiumei
*/
@Component
public class ApiCenterRunner implements ApplicationContextAware, ApplicationRunner {
private static final Logger logger = LoggerFactory.getLogger(ApiCenterRunner.class);
/**
* Spring上下文
*/
private ApplicationContext applicationContext;
@Autowired
private MethodScanner methodScanner;
@Autowired
private ApiCenterClientProperties apiCenterClientProperties;
@Autowired
private AbstractRemoteManager remoteManager;
/**
* boot启动完毕后会自动回调这个方法
* @param args
* @throws Exception
*/
@Override
public void run(ApplicationArguments args) {
// 扫描全部bean
logger.info("begin scan method");
String[] beanDefinitionNames = applicationContext.getBeanDefinitionNames();
for (String beanDefinitionName : beanDefinitionNames) {
Object bean = applicationContext.getBean(beanDefinitionName);
methodScanner.scanMethod(bean);
}
logger.info("method registry done");
// 初始化远程执行相关全部内容
remoteManager.init(apiCenterClientProperties);
}
/**
* 获取Spring上下文
* @param applicationContext
* @throws BeansException
*/
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
}
spring mvc机制可以使用事件多播器机制进行支持
package net.jlxxw.apicenter.facade.runner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ApplicationListener;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.event.ContextRefreshedEvent;
/**
* @author zhanxiumei
*/
@ComponentScan("net.jlxxw.apicenter.facade")
public class SpringMvcSupport implements ApplicationContextAware,ApplicationListener<ContextRefreshedEvent> {
private static final Logger logger = LoggerFactory.getLogger(SpringMvcSupport.class);
private ApplicationContext applicationContext;
@Autowired
private ApiCenterRunner apiCenterRunner;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
/**
* Handle an application event.
*
* @param event the event to respond to
*/
@Override
public void onApplicationEvent(ContextRefreshedEvent event) {
try {
Class.forName("org.springframework.boot.autoconfigure.SpringBootApplication");
logger.info("environment is spring boot");
} catch (ClassNotFoundException e) {
// 如果加载失败，则说明非boot环境，需要启动mvc支持
apiCenterRunner.setApplicationContext(applicationContext);
// 执行启动网关内容
logger.info("environment is spring mvc ,enable spring mvc support");
apiCenterRunner.run(null);
}
}
}
方法注册表的实现
利用对spring的支持，在spring ioc完成启动时，扫描全部的bean,将bean的实例对象、具体方法、方法入参、返回值，以及方法标注的serviceCode具体值进行存储，等待远程客户端发出请求，并反射执行此方法
package net.jlxxw.apicenter.facade.scanner;
import net.jlxxw.apicenter.facade.annotation.RemoteRegister;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.aop.support.AopUtils;
import org.springframework.core.LocalVariableTableParameterNameDiscoverer;
import org.springframework.stereotype.Component;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
/**
* 方法扫描
* @author zhanxiumei
*/
@Component
public class MethodScanner {
private static final Logger logger = LoggerFactory.getLogger(MethodScanner.class);
/**
* 方法本地注册表
*/
private static final Map<String,MethodInfo> REGISTRY_TABLE = new ConcurrentHashMap<>(16);
/**
* 将扫描到到方法注册到注册表中
* @param object 实例对象
* @param method 调用的方法
* @param serviceCode 方法唯一识别码
* @param parameterTypes 方法参数类型列表
* @param hasReturn 是否具有返回值
*/
private void registry(Object object, Method method,String serviceCode,Class[] parameterTypes,boolean hasReturn,String[] methodParamNames){
MethodInfo methodInfo = new MethodInfo();
methodInfo.setParameterTypes(parameterTypes);
methodInfo.setMethod(method);
methodInfo.setObject(object);
methodInfo.setHasReturn(hasReturn);
methodInfo.setMethodParamNames(methodParamNames);
REGISTRY_TABLE.put(serviceCode,methodInfo);
logger.info("registry method "+method);
}
/**
* 扫描方法，并检测是否合规
* @param bean spring bean
*/
public void scanMethod(Object bean){
Class clazz;
if(AopUtils.isAopProxy(bean)){
clazz = AopUtils.getTargetClass(bean);
}else{
clazz = bean.getClass();
}
// 获取全部声明的方法
Method[] declaredMethods = clazz.getDeclaredMethods();
if(Objects.nonNull(declaredMethods)){
for (Method declaredMethod : declaredMethods) {
// 如果方法包含指定的注解，则进行相关解析
if(declaredMethod.isAnnotationPresent(RemoteRegister.class)){
RemoteRegister annotation = declaredMethod.getAnnotation(RemoteRegister.class);
String serviceCode = annotation.serviceCode();
if(StringUtils.isBlank(serviceCode)){
// 注解中的 buc code 不能为空
throw new IllegalArgumentException("method:" + declaredMethod +" serviceCode is not null");
}
if(REGISTRY_TABLE.containsKey(serviceCode)){
// 注解中的 buc code 不能重复
MethodInfo methodInfo = REGISTRY_TABLE.get(serviceCode);
throw new IllegalArgumentException("method:" + declaredMethod + " serviceCode exists,please check "+methodInfo.getMethod().getName());
}
// 获取返回值类型
Class<?> returnType = declaredMethod.getReturnType();
// 获取参数列表
Class<?>[] parameterTypes = declaredMethod.getParameterTypes();
if(parameterTypes.length >0){
for (Class<?> parameterType : parameterTypes) {
if(parameterType.isArray() || parameterType.isEnum()){
throw new IllegalArgumentException("method: "+declaredMethod + "param is not support,not support type:array,enum");
}
}
}
// 获取全部方法参数名称
LocalVariableTableParameterNameDiscoverer localVariableTableParameterNameDiscoverer = new LocalVariableTableParameterNameDiscoverer();
String[] parameterNames = localVariableTableParameterNameDiscoverer.getParameterNames(declaredMethod);
registry(bean,declaredMethod,serviceCode,parameterTypes,"void".equals(returnType.getName()),parameterNames);
}
}
}
}
/**
* 根据方法注解编码，获取相关执行的方法
* @param serviceCode
* @return
*/
public MethodInfo getMethod(String serviceCode){
return REGISTRY_TABLE.get(serviceCode);
}
}
3. 注册中心
能保证服务端和客户端的注册与发现即可
在服务端启动完毕后，自动创建zookeeper的临时节点，节点列表如下
/api-center // 转发网关的永久节点
/api-center/applicationName/ip:port // 服务项目的临时节点
示例：
/api-center/demo1/182.168.1.1:1001
/api-center/demo1/182.168.1.2:1001
package net.jlxxw.apicenter.facade.remote;
import net.jlxxw.apicenter.facade.constant.ApiCenterConstant;
import net.jlxxw.apicenter.facade.exception.ApiCenterException;
import net.jlxxw.apicenter.facade.netty.NettyProxy;
import net.jlxxw.apicenter.facade.properties.ApiCenterClientProperties;
import net.jlxxw.apicenter.facade.utils.IPAddressUtils;
import net.jlxxw.apicenter.facade.utils.ZookeeperUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* @author zhanxiumei
*/
@Component
public class RemoteManager extends AbstractRemoteManager {
private static final Logger logger = LoggerFactory.getLogger(RemoteManager.class);
@Autowired
private ZookeeperUtils zookeeperUtils;
@Value("${spring.application.name}")
private String applicationName;
@Autowired
private NettyProxy nettyProxy;
/**
* 向注册中心注册
*
* @param apiCenterClientProperties
*
* @throws ApiCenterException
*/
@Override
protected void registryCenter(ApiCenterClientProperties apiCenterClientProperties) throws ApiCenterException {
if (StringUtils.isBlank( applicationName )) {
throw new IllegalArgumentException( "application name is not null" );
}
if (!zookeeperUtils.existsNode(ApiCenterConstant.PARENT_NODE)) {
// 如果api center主节点不存在，创建节点
zookeeperUtils.createOpenACLPersistentNode(ApiCenterConstant.PARENT_NODE, "".getBytes());
}
String parentPath = ApiCenterConstant.PARENT_NODE + "/" + applicationName ;
if (!zookeeperUtils.existsNode( parentPath )) {
// 如果节点不存在，创建节点
zookeeperUtils.createOpenACLPersistentNode( parentPath, "".getBytes() );
}
String serverIp = apiCenterClientProperties.getServerIp();
if(StringUtils.isBlank(serverIp)){
String ipAddress = IPAddressUtils.getIpAddress();
logger.info("server ip not found,enable automatic acquisition,ip address :"+ipAddress);
}
parentPath = parentPath + "/" + serverIp + ":" + apiCenterClientProperties.getPort();
if (!zookeeperUtils.existsNode( parentPath )) {
// 如果节点不存在，创建节点
zookeeperUtils.createOpenACLEphemeralNode( parentPath, "".getBytes() );
}
}
/**
* 初始化通信框架
*
* @param apiCenterClientProperties
*/
@Override
protected void initNetty(ApiCenterClientProperties apiCenterClientProperties) throws ApiCenterException {
nettyProxy.initProxy( apiCenterClientProperties );
}
/**
* 关闭代理对象
*/
@Override
public void closeProxy() {
}
}
4. 安全认证
调用接口需要进行安全认证，比如用户身份识别一类的
package net.jlxxw.apicenter.service.impl;
import com.alibaba.fastjson.JSON;
import net.jlxxw.apicenter.constant.ResultCodeEnum;
import net.jlxxw.apicenter.dao.ServiceInfoDAO;
import net.jlxxw.apicenter.domain.ServiceInfoDO;
import net.jlxxw.apicenter.dto.ForwardingDTO;
import net.jlxxw.apicenter.facade.constant.ApiCenterConstant;
import net.jlxxw.apicenter.facade.dto.RemoteExecuteReturnDTO;
import net.jlxxw.apicenter.facade.enums.MethodFlagEnum;
import net.jlxxw.apicenter.facade.impl.netty.NettyClient;
import net.jlxxw.apicenter.facade.param.RemoteExecuteParam;
import net.jlxxw.apicenter.facade.utils.ZookeeperUtils;
import net.jlxxw.apicenter.intergration.buc.BucClient;
import net.jlxxw.apicenter.service.ForwardingService;
import net.jlxxw.apicenter.vo.ApiCenterResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Mono;
import javax.annotation.Resource;
import java.util.List;
import java.util.Objects;
import java.util.Random;
/**
* 2020-10-18 12:08
*
* @author LCY
*/
@Service
public class ForwardingServiceImpl implements ForwardingService {
private static final Logger logger =LoggerFactory.getLogger(ForwardingServiceImpl.class);
@Resource
private ServiceInfoDAO serviceInfoDAO;
@Autowired
private ZookeeperUtils zookeeperUtils;
@Autowired
private NettyClient nettyClient;
@Autowired
private BucClient bucClient;
/**
* 处理网关转发服务
*
* @param dto 前端页面入参对象
*
* @return 网关处理结果
*/
@Override
public Mono<ApiCenterResult> forward(ForwardingDTO dto) {
/*
判断service code 是否正确
*/
ServiceInfoDO serviceInfoDO = serviceInfoDAO.findByServiceCode( dto.getServiceCode() );
if (Objects.isNull( serviceInfoDO )) {
return Mono.just( ApiCenterResult.failed( ResultCodeEnum.SERVICE_CODE_IS_NOT_EXISTS ) );
}
/*
检测服务是否在线
*/
String appName = serviceInfoDO.getAppName();
List<String> nodes = zookeeperUtils.listChildrenNodes( ApiCenterConstant.PARENT_NODE + "/" + appName );
if (CollectionUtils.isEmpty( nodes )) {
return Mono.just( ApiCenterResult.failed( ResultCodeEnum.SERVER_IS_OFFLINE ) );
}
/*
todo 网关接口鉴权
*/
if (!bucClient.auth( "", dto.getServiceCode() )) {
return Mono.just( ApiCenterResult.failed( ResultCodeEnum.SERVER_IS_OFFLINE ) );
}
/*
随机获取一个服务节点
*/
Random random = new Random();
int index = random.nextInt( nodes.size() );
String address = nodes.get( index );
String[] split = address.split( ":" );
/*
执行远程方法
*/
RemoteExecuteParam remoteExecuteParam = new RemoteExecuteParam();
remoteExecuteParam.setServiceCode( dto.getServiceCode() );
remoteExecuteParam.setMethodParamJson( JSON.toJSONString( dto.getRequestParam() ) );
remoteExecuteParam.setMethodFlag( MethodFlagEnum.NORMAL.name() );
try {
remoteExecuteParam.setIp( split[0] );
remoteExecuteParam.setPort( Integer.valueOf( split[1] ) );
RemoteExecuteReturnDTO result = nettyClient.send( remoteExecuteParam );
return Mono.just( ApiCenterResult.success( result ) );
} catch (Exception e) {
logger.error("remote method execute failed!!!",e);
return Mono.just( ApiCenterResult.failed( ResultCodeEnum.REMOTE_EXECUTE_FAILED ) );
}
}
}
5. 网络通信
性能问题，还没想出来怎么解决netty异步获取返回值这个问题，直接使用的wait 和notify
package net.jlxxw.apicenter.facade.impl.netty.impl;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelId;
import io.netty.channel.ChannelOption;
import io.netty.channel.DefaultChannelPromise;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.pool.AbstractChannelPoolMap;
import io.netty.channel.pool.ChannelPoolHandler;
import io.netty.channel.pool.FixedChannelPool;
import io.netty.channel.pool.SimpleChannelPool;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.serialization.ClassResolvers;
import io.netty.handler.codec.serialization.ObjectDecoder;
import io.netty.handler.codec.serialization.ObjectEncoder;
import io.netty.util.concurrent.Future;
import io.netty.util.concurrent.FutureListener;
import io.netty.util.concurrent.Promise;
import net.jlxxw.apicenter.facade.dto.RemoteExecuteReturnDTO;
import net.jlxxw.apicenter.facade.impl.netty.ClientHandler;
import net.jlxxw.apicenter.facade.impl.netty.NettyClient;
import net.jlxxw.apicenter.facade.param.RemoteExecuteParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
/**
* @author zhanxiumei
*/
@Service
public class NettyClientImpl implements NettyClient {
private static final Logger logger = LoggerFactory.getLogger(NettyClientImpl.class);
//管理以ip:端口号为key的连接池 FixedChannelPool继承SimpleChannelPool，有大小限制的连接池实现
private static AbstractChannelPoolMap<InetSocketAddress, FixedChannelPool> poolMap;
/**
* key channel ID，value 请求参数
*/
private static Map<String,RemoteExecuteParam> map = new ConcurrentHashMap<>();
//启动辅助类 用于配置各种参数
private Bootstrap bootstrap =new Bootstrap();
public NettyClientImpl(){
ClientHandler clientHandler = new ClientHandler( this );
bootstrap.group(new NioEventLoopGroup())
.channel( NioSocketChannel.class)
.option( ChannelOption.TCP_NODELAY,true);
poolMap = new AbstractChannelPoolMap<InetSocketAddress, FixedChannelPool>() {
@Override
protected FixedChannelPool newPool(InetSocketAddress inetSocketAddress) {
ChannelPoolHandler handler = new ChannelPoolHandler() {
//使用完channel需要释放才能放入连接池
@Override
public void channelReleased(Channel ch) throws Exception {
}
//当链接创建的时候添加channel handler，只有当channel不足时会创建，但不会超过限制的最大channel数
@Override
public void channelCreated(Channel ch) throws Exception {
logger.info("channelCreated. Channel ID: " + ch.id());
ch.pipeline().addLast(new ObjectEncoder());
ch.pipeline().addLast(new ObjectDecoder(Integer.MAX_VALUE, ClassResolvers.weakCachingConcurrentResolver(null)));
ch.pipeline().addLast(clientHandler);//添加相应回调处理
}
//获取连接池中的channel
@Override
public void channelAcquired(Channel ch) throws Exception {
}
};
return new FixedChannelPool(bootstrap.remoteAddress(inetSocketAddress), handler, 5); //单个服务端连接池大小
}
};
}
/**
* 向远程服务发送相关数据
*
* @param param
* @return
*/
@Override
public RemoteExecuteReturnDTO send(RemoteExecuteParam param) throws InterruptedException {
String ip = param.getIp();
Integer port = param.getPort();
InetSocketAddress address = new InetSocketAddress(ip, port);
final SimpleChannelPool pool = poolMap.get(address);
final Future<Channel> future = pool.acquire();
future.addListener( (FutureListener<Channel>) arg0 -> {
if (future.isSuccess()) {
Channel ch = future.getNow();
ChannelId id = ch.id();
String tempId = id.toString();
param.setChannelId( tempId );
map.put( tempId,param );
ch.writeAndFlush(param);
synchronized (param) {
//因为异步 所以不阻塞的话 该线程获取不到返回值
//放弃对象锁 并阻塞等待notify
try {
// 超时时间十秒，到时自动释放
param.wait(10000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
//放回去
pool.release(ch);
}
} );
return param.getResult();
}
/**
* 根据channelId 获取执行参数
*
* @param channelId
* @return
*/
@Override
public RemoteExecuteParam getRemoteExecuteParam(String channelId) {
return map.get(channelId);
}
/**
* 当指定的服务下线后，移除此通道
*
* @param ip
* @param port
*/
@Override
public void removeChannel(String ip, Integer port) {
InetSocketAddress address = new InetSocketAddress(ip, port);
poolMap.remove( address );
}
}
package net.jlxxw.apicenter.facade.impl.netty;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import net.jlxxw.apicenter.facade.dto.RemoteExecuteReturnDTO;
import net.jlxxw.apicenter.facade.impl.netty.impl.NettyClientImpl;
import net.jlxxw.apicenter.facade.param.RemoteExecuteParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 读取服务器返回的响应信息
* @author zhanxiumei
*
*/
public class ClientHandler extends ChannelInboundHandlerAdapter {
private static final Logger logger = LoggerFactory.getLogger(ClientHandler.class);
private NettyClient nettyClient;
public ClientHandler(NettyClient nettyClient) {
this.nettyClient = nettyClient;
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
RemoteExecuteReturnDTO result = null;
String channelId = null;
RemoteExecuteParam remoteExecuteParam = null;
try {
result = (RemoteExecuteReturnDTO)msg;
channelId = result.getChannelId();
remoteExecuteParam =nettyClient.getRemoteExecuteParam(channelId);
} catch (Exception e){
RemoteExecuteReturnDTO obj = new RemoteExecuteReturnDTO();
obj.setSuccess( false );
obj.setMessage( "远程执行产生位置异常！" );
logger.error( "远程执行产生位置异常！",e );
}finally {
synchronized (remoteExecuteParam){
remoteExecuteParam.setResult( result);
remoteExecuteParam.notify();
}
}
}
// 数据读取完毕的处理
@Override
public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
logger.info("客户端读取数据完毕");
ctx.flush();
}
// 出现异常的处理
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
logger.error("client 读取数据出现异常",cause);
ctx.close();
}
}
使用说明
任意client为spring 项目，需要支持远程调用的方法，需要加入@RemoteRegister 注解，并将此bean交由IOC进行管理
使用方法：
1、api-center项目 facade 模块执行 mvn clean instaill 安装到本地仓库，有条件的可以上传到私服
2、在新项目中，加入 facade 依赖，坐标参考api-center-facade工程pom文件
3、在指定的方法加上@RemoteRegister，并配置serviceCode具体值
4、apicenter需要配置注册中心，数据库等基础数据信息，同时添加项目的基本信息到数据表中
5、client需要和api-center处于同一个zookeeper环境中
注意事项
client：
1、方法入参目前仅支持对象，不支持基本数据类型和String等，需要自定义结构体
2、方法返回值不支持数据结构，如Map,List等，需要使用对象
3、client的基本配置可以在yml中输入 api-center进行联想
