导读
本篇将会讲解注册中心的原理，自研RPC，netty的应用以及相关源码
注册中心
xxl-job也是有自己的分布式注册中心的，其注册表使用mysql来进行存储，心跳机制通过http请求实现
xxl-job中服务注册需要绑定一个执行器作为载体，当我们服务注册后首先会将注册机器信息存入注册表中，扫描线程会不断扫描注册表，将注册表中的机器根据appName配置绑定到执行器上。
先来看注册中心的核心helpercom.xxl.job.admin.core.thread.JobRegistryHelper#start
public void start(){
// 服务注册/删除线程池
registryOrRemoveThreadPool = new ThreadPoolExecutor(
2,
10,
30L,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(2000),
new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "xxl-job, admin JobRegistryMonitorHelper-registryOrRemoveThreadPool-" + r.hashCode());
}
},
new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
r.run();
logger.warn(">>>>>>>>>>> xxl-job, registry or remove too fast, match threadpool rejected handler(run now).");
}
});
// 注册服务监听器
registryMonitorThread = new Thread(new Runnable() {
@Override
public void run() {
while (!toStop) {
try {
// 获得配置为自动注册的执行器
List<XxlJobGroup> groupList = XxlJobAdminConfig.getAdminConfig().getXxlJobGroupDao().findByAddressType(0);
if (groupList!=null && !groupList.isEmpty()) {
// 将90秒没有心跳的机器移除
List<Integer> ids = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findDead(RegistryConfig.DEAD_TIMEOUT, new Date());
if (ids!=null && ids.size()>0) {
// 移除注册表中的机器数据
XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().removeDead(ids);
}
// 归集执行器下的注册机ip地址
HashMap<String, List<String>> appAddressMap = new HashMap<String, List<String>>();
// 获得90秒内有心跳的机器
List<XxlJobRegistry> list = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findAll(RegistryConfig.DEAD_TIMEOUT, new Date());
if (list != null) {
for (XxlJobRegistry item: list) {
if (RegistryConfig.RegistType.EXECUTOR.name().equals(item.getRegistryGroup())) {
String appname = item.getRegistryKey();
List<String> registryList = appAddressMap.get(appname);
if (registryList == null) {
registryList = new ArrayList<String>();
}
if (!registryList.contains(item.getRegistryValue())) {
registryList.add(item.getRegistryValue());
}
appAddressMap.put(appname, registryList);
}
}
}
// 将注册表中的数据归档存入到执行器中并入库
for (XxlJobGroup group: groupList) {
List<String> registryList = appAddressMap.get(group.getAppname());
String addressListStr = null;
if (registryList!=null && !registryList.isEmpty()) {
Collections.sort(registryList);
StringBuilder addressListSB = new StringBuilder();
for (String item:registryList) {
addressListSB.append(item).append(",");
}
addressListStr = addressListSB.toString();
addressListStr = addressListStr.substring(0, addressListStr.length()-1);
}
group.setAddressList(addressListStr);
group.setUpdateTime(new Date());
XxlJobAdminConfig.getAdminConfig().getXxlJobGroupDao().update(group);
}
}
} catch (Exception e) {
if (!toStop) {
logger.error(">>>>>>>>>>> xxl-job, job registry monitor thread error:{}", e);
}
}
try {
// 检查周期30秒
TimeUnit.SECONDS.sleep(RegistryConfig.BEAT_TIMEOUT);
} catch (InterruptedException e) {
if (!toStop) {
logger.error(">>>>>>>>>>> xxl-job, job registry monitor thread error:{}", e);
}
}
}
logger.info(">>>>>>>>>>> xxl-job, job registry monitor thread stop");
}
});
registryMonitorThread.setDaemon(true);
registryMonitorThread.setName("xxl-job, admin JobRegistryMonitorHelper-registryMonitorThread");
registryMonitorThread.start();
}
该helper启动了一个用于注册和移除的线程池，该线程会与数据库交互，使用线程池主要是为了不阻塞调度线程。还启动了一个守护的监听线程，用于监听服务心跳和绑定执行器。
接下来看看服务注册逻辑com.xxl.job.admin.core.thread.JobRegistryHelper#registry
public ReturnT<String> registry(RegistryParam registryParam) {
// 参数校验
if (!StringUtils.hasText(registryParam.getRegistryGroup())
|| !StringUtils.hasText(registryParam.getRegistryKey())
|| !StringUtils.hasText(registryParam.getRegistryValue())) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "Illegal Argument.");
}
// 使用线程池执行
registryOrRemoveThreadPool.execute(new Runnable() {
@Override
public void run() {
// 更新注册表的心跳时间
int ret = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().registryUpdate(registryParam.getRegistryGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue(), new Date());
if (ret < 1) {
// 更新失败说明不存在该数据，存入注册表
XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().registrySave(registryParam.getRegistryGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue(), new Date());
// 刷新注册信息，暂时没用到，是个空方法
freshGroupRegistryInfo(registryParam);
}
}
});
return ReturnT.SUCCESS;
}
// 移除注册机
public ReturnT<String> registryRemove(RegistryParam registryParam) {
if (!StringUtils.hasText(registryParam.getRegistryGroup())
|| !StringUtils.hasText(registryParam.getRegistryKey())
|| !StringUtils.hasText(registryParam.getRegistryValue())) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "Illegal Argument.");
}
registryOrRemoveThreadPool.execute(new Runnable() {
@Override
public void run() {
// 移除注册表数据
int ret = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().registryDelete(registryParam.getRegistryGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue());
if (ret > 0) {
// 刷新注册信息，暂时没用到，是个空方法
freshGroupRegistryInfo(registryParam);
}
}
});
return ReturnT.SUCCESS;
}
可以看到注册逻辑其实并不复杂，只需要存入心跳数据，并不停更新心跳时间即可确定机器存活，而移除注册机则是直接将注册表数据移除。
接下来看看服务注册的入口
com.xxl.job.admin.controller.JobApiController#api
@RequestMapping("/{uri}")
@ResponseBody
@PermissionLimit(limit=false)
public ReturnT<String> api(HttpServletRequest request, @PathVariable("uri") String uri, @RequestBody(required = false) String data) {
// 请求方法校验
if (!"POST".equalsIgnoreCase(request.getMethod())) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "invalid request, HttpMethod not support.");
}
// 请求uri校验
if (uri==null || uri.trim().length()==0) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "invalid request, uri-mapping empty.");
}
// 请求token校验
if (XxlJobAdminConfig.getAdminConfig().getAccessToken()!=null
&& XxlJobAdminConfig.getAdminConfig().getAccessToken().trim().length()>0
&& !XxlJobAdminConfig.getAdminConfig().getAccessToken().equals(request.getHeader(XxlJobRemotingUtil.XXL_JOB_ACCESS_TOKEN))) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "The access token is wrong.");
}
// 根据请求uri解析参数并调度
if ("callback".equals(uri)) {
List<HandleCallbackParam> callbackParamList = GsonTool.fromJson(data, List.class, HandleCallbackParam.class);
return adminBiz.callback(callbackParamList);
} else if ("registry".equals(uri)) {
RegistryParam registryParam = GsonTool.fromJson(data, RegistryParam.class);
return adminBiz.registry(registryParam);
} else if ("registryRemove".equals(uri)) {
RegistryParam registryParam = GsonTool.fromJson(data, RegistryParam.class);
return adminBiz.registryRemove(registryParam);
} else {
return new ReturnT<String>(ReturnT.FAIL_CODE, "invalid request, uri-mapping("+ uri +") not found.");
}
}
此处就是客户端调度服务端的总入口，可以看到入口是http请求，根据不同的uri转发各个业务，分别有callback（回调），registry（注册/心跳），registryRemove（移除注册）
netty的应用
netty主要应用于客户端，为服务端提供调度接口，xxl-job并没有直接使用springMVC作为请求入口，其好处是低耦合，非spring应用也可以使用，其次使用netty可控性，稳定性，性能都会有更好的表现。
我们需要先到XxlJobExecutor类，该类为客户端实例，在启动后会调用start方法
启动入口com.xxl.job.core.executor.XxlJobExecutor#start
public void start() throws Exception {
// 初始化日志路径
XxlJobFileAppender.initLogPath(logPath);
// 初始化xxl-job连接信息
initAdminBizList(adminAddresses, accessToken);
// 初始化日志文件切割线程
JobLogFileCleanThread.getInstance().start(logRetentionDays);
// 初始化回调线程
TriggerCallbackThread.getInstance().start();
// 初始化netty服务
initEmbedServer(address, ip, port, appname, accessToken);
}
进入初始化netty服务的方法com.xxl.job.core.executor.XxlJobExecutor#initEmbedServer
private void initEmbedServer(String address, String ip, int port, String appname, String accessToken) throws Exception {
// 选择可用端口号，从9999向下找到未使用过的端口
port = port>0?port: NetUtil.findAvailablePort(9999);
// 获得本机内网ip地址
ip = (ip!=null&&ip.trim().length()>0)?ip: IpUtil.getIp();
// 拼接应用请求地址
if (address==null || address.trim().length()==0) {
String ip_port_address = IpUtil.getIpPort(ip, port); // registry-address：default use address to registry , otherwise use ip:port if address is null
address = "http://{ip_port}/".replace("{ip_port}", ip_port_address);
}
if (accessToken==null || accessToken.trim().length()==0) {
logger.warn(">>>>>>>>>>> xxl-job accessToken is empty. To ensure system security, please set the accessToken.");
}
// 创建server实例并启动
embedServer = new EmbedServer();
embedServer.start(address, port, appname, accessToken);
}
进入到真正创建netty服务的方法com.xxl.job.core.server.EmbedServer#start
public void start(final String address, final int port, final String appname, final String accessToken) {
// 创建业务实例
executorBiz = new ExecutorBizImpl();
thread = new Thread(new Runnable() {
@Override
public void run() {
// 用于接受ServerSocketChannel的io请求，再把请求具体执行的回调函数转交给worker group执行
EventLoopGroup bossGroup = new NioEventLoopGroup();
EventLoopGroup workerGroup = new NioEventLoopGroup();
// 创建业务消费线程
ThreadPoolExecutor bizThreadPool = new ThreadPoolExecutor(
0,
200,
60L,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(2000),
new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "xxl-job, EmbedServer bizThreadPool-" + r.hashCode());
}
},
new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
throw new RuntimeException("xxl-job, EmbedServer bizThreadPool is EXHAUSTED!");
}
});
try {
// 启动netty服务
ServerBootstrap bootstrap = new ServerBootstrap();
bootstrap.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel channel) throws Exception {
channel.pipeline()
// 空闲检测
.addLast(new IdleStateHandler(0, 0, 30 * 3, TimeUnit.SECONDS)) // beat 3N, close if idle
// http编码处理类
.addLast(new HttpServerCodec())
// post请求参数解析器
.addLast(new HttpObjectAggregator(5 * 1024 * 1024)) // merge request & reponse to FULL
// 自定义业务handler
.addLast(new EmbedHttpServerHandler(executorBiz, accessToken, bizThreadPool));
}
})
.childOption(ChannelOption.SO_KEEPALIVE, true);
// 将端口号绑定到netty服务上
ChannelFuture future = bootstrap.bind(port).sync();
logger.info(">>>>>>>>>>> xxl-job remoting server start success, nettype = {}, port = {}", EmbedServer.class, port);
// 开始服务注册，将服务注册到xxl-job
startRegistry(appname, address);
// 同步阻塞线程
future.channel().closeFuture().sync();
} catch (InterruptedException e) {
if (e instanceof InterruptedException) {
logger.info(">>>>>>>>>>> xxl-job remoting server stop.");
} else {
logger.error(">>>>>>>>>>> xxl-job remoting server error.", e);
}
} finally {
// 执行到此处表示netty服务停止，释放资源
try {
workerGroup.shutdownGracefully();
bossGroup.shutdownGracefully();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
});
// 设置守护线程，防止netty线程被回收
thread.setDaemon(true);
// 启动线程
thread.start();
}
可以看到当netty服务启动完成后，会将服务注册到xxl-job
com.xxl.job.core.server.EmbedServer#startRegistry
com.xxl.job.core.thread.ExecutorRegistryThread#start
public void start(final String appname, final String address){
// valid
if (appname==null || appname.trim().length()==0) {
logger.warn(">>>>>>>>>>> xxl-job, executor registry config fail, appname is null.");
return;
}
if (XxlJobExecutor.getAdminBizList() == null) {
logger.warn(">>>>>>>>>>> xxl-job, executor registry config fail, adminAddresses is null.");
return;
}
registryThread = new Thread(new Runnable() {
@Override
public void run() {
// 循环心跳机制
while (!toStop) {
try {
// 构建请求参数
RegistryParam registryParam = new RegistryParam(RegistryConfig.RegistType.EXECUTOR.name(), appname, address);
for (AdminBiz adminBiz: XxlJobExecutor.getAdminBizList()) {
try {
// 服务注册/心跳
ReturnT<String> registryResult = adminBiz.registry(registryParam);
if (registryResult!=null && ReturnT.SUCCESS_CODE == registryResult.getCode()) {
registryResult = ReturnT.SUCCESS;
logger.debug(">>>>>>>>>>> xxl-job registry success, registryParam:{}, registryResult:{}", new Object[]{
registryParam, registryResult});
break;
} else {
logger.info(">>>>>>>>>>> xxl-job registry fail, registryParam:{}, registryResult:{}", new Object[]{
registryParam, registryResult});
}
} catch (Exception e) {
logger.info(">>>>>>>>>>> xxl-job registry error, registryParam:{}", registryParam, e);
}
}
} catch (Exception e) {
if (!toStop) {
logger.error(e.getMessage(), e);
}
}
try {
if (!toStop) {
// 心跳时长 30秒
TimeUnit.SECONDS.sleep(RegistryConfig.BEAT_TIMEOUT);
}
} catch (InterruptedException e) {
if (!toStop) {
logger.warn(">>>>>>>>>>> xxl-job, executor registry thread interrupted, error msg:{}", e.getMessage());
}
}
}
// 服务停止后以下代码服务下线通知
try {
RegistryParam registryParam = new RegistryParam(RegistryConfig.RegistType.EXECUTOR.name(), appname, address);
for (AdminBiz adminBiz: XxlJobExecutor.getAdminBizList()) {
try {
ReturnT<String> registryResult = adminBiz.registryRemove(registryParam);
if (registryResult!=null && ReturnT.SUCCESS_CODE == registryResult.getCode()) {
registryResult = ReturnT.SUCCESS;
logger.info(">>>>>>>>>>> xxl-job registry-remove success, registryParam:{}, registryResult:{}", new Object[]{
registryParam, registryResult});
break;
} else {
logger.info(">>>>>>>>>>> xxl-job registry-remove fail, registryParam:{}, registryResult:{}", new Object[]{
registryParam, registryResult});
}
} catch (Exception e) {
if (!toStop) {
logger.info(">>>>>>>>>>> xxl-job registry-remove error, registryParam:{}", registryParam, e);
}
}
}
} catch (Exception e) {
if (!toStop) {
logger.error(e.getMessage(), e);
}
}
logger.info(">>>>>>>>>>> xxl-job, executor registry thread destroy.");
}
});
registryThread.setDaemon(true);
registryThread.setName("xxl-job, executor ExecutorRegistryThread");
registryThread.start();
}
以上代码为服务注册，也是心跳机制，当服务停止后stop变量会变为true，这时会通知服务端下线，如果服务突然宕机也没有关系，服务端有循环检查机制，当长时间没有收到客户端的心跳，会自动下线。
接下来看自定义的netty handler是如何处理服务端消息的
com.xxl.job.core.server.EmbedServer.EmbedHttpServerHandler#channelRead0
protected void channelRead0(final ChannelHandlerContext ctx, FullHttpRequest msg) throws Exception {
// 消息解码
String requestData = msg.content().toString(CharsetUtil.UTF_8);
// 请求uri
String uri = msg.uri();
HttpMethod httpMethod = msg.method();
boolean keepAlive = HttpUtil.isKeepAlive(msg);
String accessTokenReq = msg.headers().get(XxlJobRemotingUtil.XXL_JOB_ACCESS_TOKEN);
// 业务线程执行处理
bizThreadPool.execute(new Runnable() {
@Override
public void run() {
Object responseObj = process(httpMethod, uri, requestData, accessTokenReq);
// 返回对象转换json
String responseJson = GsonTool.toJson(responseObj);
// 返回
writeResponse(ctx, keepAlive, responseJson);
}
});
}
com.xxl.job.core.server.EmbedServer.EmbedHttpServerHandler#process
private Object process(HttpMethod httpMethod, String uri, String requestData, String accessTokenReq) {
// 校验请求方式，token，uri
if (HttpMethod.POST != httpMethod) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "invalid request, HttpMethod not support.");
}
if (uri == null || uri.trim().length() == 0) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "invalid request, uri-mapping empty.");
}
if (accessToken != null
&& accessToken.trim().length() > 0
&& !accessToken.equals(accessTokenReq)) {
return new ReturnT<String>(ReturnT.FAIL_CODE, "The access token is wrong.");
}
try {
// 处理业务
if ("/beat".equals(uri)) {
// 心跳检查
return executorBiz.beat();
} else if ("/idleBeat".equals(uri)) {
// 任务空闲检查
IdleBeatParam idleBeatParam = GsonTool.fromJson(requestData, IdleBeatParam.class);
return executorBiz.idleBeat(idleBeatParam);
} else if ("/run".equals(uri)) {
// 运行一个任务
TriggerParam triggerParam = GsonTool.fromJson(requestData, TriggerParam.class);
return executorBiz.run(triggerParam);
} else if ("/kill".equals(uri)) {
// 杀死一个任务
KillParam killParam = GsonTool.fromJson(requestData, KillParam.class);
return executorBiz.kill(killParam);
} else if ("/log".equals(uri)) {
// 获得客户端记录的日志
LogParam logParam = GsonTool.fromJson(requestData, LogParam.class);
return executorBiz.log(logParam);
} else {
// 无法解析的业务uri，异常返回
return new ReturnT<String>(ReturnT.FAIL_CODE, "invalid request, uri-mapping(" + uri + ") not found.");
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new ReturnT<String>(ReturnT.FAIL_CODE, "request error:" + ThrowableUtil.toString(e));
}
}
可以看到逻辑解析和服务端解析没有区别，都是根据uri来判断并执行业务动作，整个netty的调度链路也就是xxl-job的自研RPC