线程模型:
参数：
public static class Args extends AbstractNonblockingServerArgs<Args> {
//selector线程数 selector线程负责1.事件注册 2.读写IO轮询 3.将数据读取或者写入到transport
public int selectorThreads = 2;
//工作线程数 负责协议的解析 对象序列化 业务逻辑的执行 响应写入到ByteBuffer 工作线程为0 就直接在selector线程上执行
private int workerThreads = 5;
//服务关闭后等待时间 主要等到已经到达的请求处理完成 后面来的请求拒绝掉
private int stopTimeoutVal = 60;
private TimeUnit stopTimeoutUnit = TimeUnit.SECONDS;
//工作线程池
private ExecutorService executorService = null;
private int acceptQueueSizePerThread = 4;//每个selector线程中请求的最大个数，这些请求还没有注册读取事件
//策略 决定添加到selector线程队列中这个动作在当前线程上执行还是在线程池上执行。
public static enum AcceptPolicy {
/**
* Require accepted connection registration to be handled by the executor.
* If the worker pool is saturated, further accepts will be closed
* immediately. Slightly increases latency due to an extra scheduling.
*/
FAIR_ACCEPT,
/**
* Handle the accepts as fast as possible, disregarding the status of the
* executor service.
*/
FAST_ACCEPT
}
private AcceptPolicy acceptPolicy = AcceptPolicy.FAST_ACCEPT;
public Args(TNonblockingServerTransport transport) {
super(transport);
}
public Args selectorThreads(int i) {
selectorThreads = i;
return this;
}
public int getSelectorThreads() {
return selectorThreads;
}
public Args workerThreads(int i) {
workerThreads = i;
return this;
}
public int getWorkerThreads() {
return workerThreads;
}
public int getStopTimeoutVal() {
return stopTimeoutVal;
}
public Args stopTimeoutVal(int stopTimeoutVal) {
this.stopTimeoutVal = stopTimeoutVal;
return this;
}
public TimeUnit getStopTimeoutUnit() {
return stopTimeoutUnit;
}
public Args stopTimeoutUnit(TimeUnit stopTimeoutUnit) {
this.stopTimeoutUnit = stopTimeoutUnit;
return this;
}
public ExecutorService getExecutorService() {
return executorService;
}
public Args executorService(ExecutorService executorService) {
this.executorService = executorService;
return this;
}
public int getAcceptQueueSizePerThread() {
return acceptQueueSizePerThread;
}
public Args acceptQueueSizePerThread(int acceptQueueSizePerThread) {
this.acceptQueueSizePerThread = acceptQueueSizePerThread;
return this;
}
public AcceptPolicy getAcceptPolicy() {
return acceptPolicy;
}
public Args acceptPolicy(AcceptPolicy acceptPolicy) {
this.acceptPolicy = acceptPolicy;
return this;
}
构造方法：
//请求接收线程 只负责请求的接收 后将请求转给SelectThread
private AcceptThread acceptThread;
//SelectThread负责连接上的IO轮询（读写事件） 新来的请求读事件的注册。
private final Set<SelectorThread> selectorThreads = new HashSet<SelectorThread>();
// This wraps all the functionality of queueing and thread pool management
// for the passing of Invocations from the selector thread(s) to the workers
// (if any).
private final ExecutorService invoker;
private final Args args;
public TThreadedSelectorServer(Args args) {
super(args);
args.validate();
invoker = args.executorService == null ? createDefaultExecutor(args) : args.executorService;
this.args = args;
} 开启线程
protected boolean startThreads() {
try {
for (int i = 0; i < args.selectorThreads; ++i) {
//创建SelectorThread acceptQueueSizePerThread队列容纳的是AcceptThread已经转给SelectorThread 尚未注册读事件的请求
selectorThreads.add(new SelectorThread(args.acceptQueueSizePerThread));
}
acceptThread = new AcceptThread((TNonblockingServerTransport) serverTransport_,
createSelectorThreadLoadBalancer(selectorThreads));
stopped_ = false;
for (SelectorThread thread : selectorThreads) {
thread.start();
}
acceptThread.start();
return true;
} catch (IOException e) {
LOGGER.error("Failed to start threads!", e);
return false;
}
} 请求接收线程
public void run() {
try {
while (!stopped_) {
select();//轮询
}
} catch (Throwable t) {
LOGGER.error("run() exiting due to uncaught error", t);
} finally {
// This will wake up the selector threads
TThreadedSelectorServer.this.stop();
}
}
private void select() {
try {
// 等连接事件发生
acceptSelector.select();
// process the io events we received
Iterator<SelectionKey> selectedKeys = acceptSelector.selectedKeys().iterator();
while (!stopped_ && selectedKeys.hasNext()) {
SelectionKey key = selectedKeys.next();
selectedKeys.remove();
// skip if not valid
if (!key.isValid()) {
continue;
}
if (key.isAcceptable()) {
handleAccept();//分发
} else {
LOGGER.warn("Unexpected state in select! " + key.interestOps());
}
}
} catch (IOException e) {
LOGGER.warn("Got an IOException while selecting!", e);
}
}
private void handleAccept() {
final TNonblockingTransport client = doAccept();
if (client != null) {
// 选择一个线程
final SelectorThread targetThread = threadChooser.nextThread();
if (args.acceptPolicy == Args.AcceptPolicy.FAST_ACCEPT || invoker == null) {
doAddAccept(targetThread, client);//没有定义线程池或者定义FAST_ACCEPT 在当前线程中执行
} else {
// FAIR_ACCEPT
try {
invoker.submit(new Runnable() {
public void run() {
doAddAccept(targetThread, client);
}
});
} catch (RejectedExecutionException rx) {
LOGGER.warn("ExecutorService rejected accept registration!", rx);
// close immediately
client.close();
}
}
}
}
private void doAddAccept(SelectorThread thread, TNonblockingTransport client) {
if (!thread.addAcceptedConnection(client)) {
client.close();
}
}
public boolean addAcceptedConnection(TNonblockingTransport accepted) {
try {
acceptedQueue.put(accepted);//将请求放到选中线程的队列里面
} catch (InterruptedException e) {
LOGGER.warn("Interrupted while adding accepted connection!", e);
return false;
}
selector.wakeup();//将对应的SelectorThread唤醒
return true;
} 下面看
SelectorThread做些什么事情：
try {
while (!stopped_) {
select();
processAcceptedConnections();
processInterestChanges();
}
for (SelectionKey selectionKey : selector.keys()) {
cleanupSelectionKey(selectionKey);
}
} catch (Throwable t) {
LOGGER.error("run() exiting due to uncaught error", t);
} finally {
// This will wake up the accept thread and the other selector threads
TThreadedSelectorServer.this.stop();
}
select();
private void select() {
try {
// 刚才AcceptorThread wakeup方法的调用会唤醒select（）还有比如读写事件的发生也会触发
selector.select();
// process the io events we received
Iterator<SelectionKey> selectedKeys = selector.selectedKeys().iterator();
while (!stopped_ && selectedKeys.hasNext()) {
SelectionKey key = selectedKeys.next();
selectedKeys.remove();
// skip if not valid
if (!key.isValid()) {//如果是AcceptorThread wakeup唤醒 那么就做这里。
cleanupSelectionKey(key);
continue;
}
if (key.isReadable()) {
// deal with reads
handleRead(key);
} else if (key.isWritable()) {
// deal with writes
handleWrite(key);
} else {
LOGGER.warn("Unexpected state in select! " + key.interestOps());
}
}
} catch (IOException e) {
LOGGER.warn("Got an IOException while selecting!", e);
}
}
processAcceptedConnections();
private void processAcceptedConnections() {
while (!stopped_) {
TNonblockingTransport accepted = acceptedQueue.poll();//循环从队列中获取并移除直到没有为止 对获取的Transport进行注册读取事件
if (accepted == null) {
break;
}
registerAccepted(accepted);
}
}
private void registerAccepted(TNonblockingTransport accepted) {
SelectionKey clientKey = null;
try {
clientKey = accepted.registerSelector(selector, SelectionKey.OP_READ);//给刚才的请求注册读取事件
FrameBuffer frameBuffer = new FrameBuffer(accepted, clientKey, SelectorThread.this);//将FrameBuffer对象附着到Key上面 FrameBuffer很重要后面补上
clientKey.attach(frameBuffer);
} catch (IOException e) {
LOGGER.warn("Failed to register accepted connection to selector!", e);
if (clientKey != null) {
cleanupSelectionKey(clientKey);
}
accepted.close();
}
}
protected void processInterestChanges() {//状态切换事件注册 比如读取完了业务逻辑也处理了 返回也得到了 那么就要注册写事件向客户端写数据
synchronized (selectInterestChanges) {
for (FrameBuffer fb : selectInterestChanges) {
fb.changeSelectInterests();
}
selectInterestChanges.clear();
}
}
public void changeSelectInterests() {
if (state_ == FrameBufferState.AWAITING_REGISTER_WRITE) {
// set the OP_WRITE interest
selectionKey_.interestOps(SelectionKey.OP_WRITE);
state_ = FrameBufferState.WRITING;
} else if (state_ == FrameBufferState.AWAITING_REGISTER_READ) {
prepareRead();
} else if (state_ == FrameBufferState.AWAITING_CLOSE) {
close();
selectionKey_.cancel();
} else {
LOGGER.error("changeSelectInterest was called, but state is invalid (" + state_ + ")");
}
}
下面细看select里面的读写做了什么事情：
if (key.isReadable()) {
// deal with reads
handleRead(key);
} else if (key.isWritable()) {
// deal with writes
handleWrite(key);
}
protected void handleRead(SelectionKey key) {
FrameBuffer buffer = (FrameBuffer) key.attachment();//获取到附着在上面的FrameBuffer对象
if (!buffer.read()) {//读取数据
cleanupSelectionKey(key);
return;
}
if (buffer.isFrameFullyRead()) {//读取完成执行业务方法
if (!requestInvoke(buffer)) {
cleanupSelectionKey(key);
}
}
}
public boolean read() {
if (state_ == FrameBufferState.READING_FRAME_SIZE) {
// try to read the frame size completely
if (!internalRead()) {//这里读取的是数据大小
return false;
}
// if the frame size has been read completely, then prepare to read the
// actual frame.
if (buffer_.remaining() == 0) {
// pull out the frame size as an integer.
int frameSize = buffer_.getInt(0);
if (frameSize <= 0) {
LOGGER.error("Read an invalid frame size of " + frameSize
+ ". Are you using TFramedTransport on the client side?");
return false;
}
// if this frame will always be too large for this server, log the
// error and close the connection.
if (frameSize > MAX_READ_BUFFER_BYTES) {
LOGGER.error("Read a frame size of " + frameSize
+ ", which is bigger than the maximum allowable buffer size for ALL connections.");
return false;
}
// if this frame will push us over the memory limit, then return.
// with luck, more memory will free up the next time around.
if (readBufferBytesAllocated.get() + frameSize > MAX_READ_BUFFER_BYTES) {
return true;
}
// increment the amount of memory allocated to read buffers
readBufferBytesAllocated.addAndGet(frameSize + 4);
// reallocate the readbuffer as a frame-sized buffer
buffer_ = ByteBuffer.allocate(frameSize + 4);//将容量扩展
buffer_.putInt(frameSize);
state_ = FrameBufferState.READING_FRAME;//修改状态
} else {
// this skips the check of READING_FRAME state below, since we can't
// possibly go on to that state if there's data left to be read at
// this one.
return true;
}
}
// it is possible to fall through from the READING_FRAME_SIZE section
// to READING_FRAME if there's already some frame data available once
// READING_FRAME_SIZE is complete.
if (state_ == FrameBufferState.READING_FRAME) {
if (!internalRead()) {//再次读取真实数据
return false;
}
// since we're already in the select loop here for sure, we can just
// modify our selection key directly.
if (buffer_.remaining() == 0) {
// get rid of the read select interests
selectionKey_.interestOps(0);
state_ = FrameBufferState.READ_FRAME_COMPLETE;//读取完成
}
return true;
}
// if we fall through to this point, then the state must be invalid.
LOGGER.error("Read was called but state is invalid (" + state_ + ")");
return false;
}
private boolean internalRead() {//底层的读取操作
try {
if (trans_.read(buffer_) < 0) {
return false;
}
return true;
} catch (IOException e) {
LOGGER.warn("Got an IOException in internalRead!", e);
return false;
}
}
//数据读取完了开始处理（协议层要进行一些处理 反序列化 最好业务逻辑处理 结果的返回）
protected boolean requestInvoke(FrameBuffer frameBuffer) {
Runnable invocation = getRunnable(frameBuffer);
if (invoker != null) {
try {
invoker.execute(invocation);
return true;
} catch (RejectedExecutionException rx) {
LOGGER.warn("ExecutorService rejected execution!", rx);
return false;
}
} else {
// Invoke on the caller's thread
invocation.run();
return true;
}
}
public void run() {
frameBuffer.invoke();
}
//这个时候数据已经到内存里面了 实际就是开通流通道进行交互
private ByteBuffer buffer_;//读取地方 private TByteArrayOutputStream response_;//相应的地方
public void invoke() {
TTransport inTrans = getInputTransport();
TProtocol inProt = inputProtocolFactory_.getProtocol(inTrans);
TProtocol outProt = outputProtocolFactory_.getProtocol(getOutputTransport());
try {
processorFactory_.getProcessor(inTrans).process(inProt, outProt);
responseReady();
return;
} catch (TException te) {
LOGGER.warn("Exception while invoking!", te);
} catch (Throwable t) {
LOGGER.error("Unexpected throwable while invoking!", t);
}
// This will only be reached when there is a throwable.
state_ = FrameBufferState.AWAITING_CLOSE;
requestSelectInterestChange();
}
public boolean process(TProtocol in, TProtocol out) throws TException {//这个到了后面协议层再写
TMessage msg = in.readMessageBegin();
ProcessFunction fn = processMap.get(msg.name);
if (fn == null) {
TProtocolUtil.skip(in, TType.STRUCT);
in.readMessageEnd();
TApplicationException x = new TApplicationException(TApplicationException.UNKNOWN_METHOD, "Invalid method name: '"+msg.name+"'");
out.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid));
x.write(out);
out.writeMessageEnd();
out.getTransport().flush();
return true;
}
fn.process(msg.seqid, in, out, iface);
return true;
}
public void responseReady() {
// the read buffer is definitely no longer in use, so we will decrement
// our read buffer count. we do this here as well as in close because
// we'd like to free this read memory up as quickly as possible for other
// clients.
readBufferBytesAllocated.addAndGet(-buffer_.array().length);
if (response_.len() == 0) {
// go straight to reading again. this was probably an oneway method
state_ = FrameBufferState.AWAITING_REGISTER_READ;
buffer_ = null;
} else {
buffer_ = ByteBuffer.wrap(response_.get(), 0, response_.len());//将要写的数据转换到buffer中去
// set state that we're waiting to be switched to write. we do this
// asynchronously through requestSelectInterestChange() because there is
// a possibility that we're not in the main thread, and thus currently
// blocked in select(). (this functionality is in place for the sake of
// the HsHa server.)
state_ = FrameBufferState.AWAITING_REGISTER_WRITE;
}
requestSelectInterestChange();
}
private void requestSelectInterestChange() {//requestInvoke=>invoker.execute(invocation);|invocation.run(); 如果有线程池那么在线程池中执行 否则就在当前线程中执行
if (Thread.currentThread() == this.selectThread_) {
changeSelectInterests();
} else {
this.selectThread_.requestSelectInterestChange(this);//将FrameBuffer加入到对应SelectorThread的SelectInterestChanages里面去然后将其唤醒 唤醒的线程会修改状态。
}
}