HBase的RPC沿用了hadoop的RPC部分代码。HMaster，RegionServer和client都是通过RPC交换数据的。其实大抵相似。RegionServer端RPC部分类图如下：
HBaseServer核心类，实现了Reactor模型，主线程Listener负责accept外部连接，子线程Reader负责连接的具体读写操作，将数据反序列化成Call对象，通过Queue交给后面的Handler线程处理，Handler线程发起反射调用，并将response数据交给Responder线程处理，Responder线程将数据最终写回给client。
0.94代码实现如下：
HRegionServer启动时，构造函数中
//端口默认60020
this.rpcServer = HBaseRPC.getServer(this,
new Class<?>[]{HRegionInterface.class, HBaseRPCErrorHandler.class,
OnlineRegions.class},
initialIsa.getHostName(), // BindAddress is IP we got for this server.
initialIsa.getPort(),
conf.getInt("hbase.regionserver.handler.count", 10),
conf.getInt("hbase.regionserver.metahandler.count", 10),
conf.getBoolean("hbase.rpc.verbose", false),
conf, QOS_THRESHOLD);
之后初始化HBaseServer，主要就是初始化Listener和Responder
protected HBaseServer(String bindAddress, int port,
Class<? extends Writable> paramClass, int handlerCount,
int priorityHandlerCount, Configuration conf, String serverName,
int highPriorityLevel)
throws IOException {
this.bindAddress = bindAddress;
......
// temporary backward compatibility
String oldMaxQueueSize = this.conf.get("ipc.server.max.queue.size");
if (oldMaxQueueSize == null) {
this.maxQueueLength =
this.conf.getInt("ipc.server.max.callqueue.length",
handlerCount * DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
} else {
LOG.warn("ipc.server.max.queue.size was renamed " +
"ipc.server.max.callqueue.length, " +
"please update your configuration");
this.maxQueueLength = Integer.getInteger(oldMaxQueueSize);
}
//默认queue length是100，queue size是3K
this.maxQueueSize =
this.conf.getInt("ipc.server.max.callqueue.size",
DEFAULT_MAX_CALLQUEUE_SIZE);
this.readThreads = conf.getInt(
"ipc.server.read.threadpool.size",
10);
this.callQueue = new LinkedBlockingQueue<Call>(maxQueueLength);
if (priorityHandlerCount > 0) {
this.priorityCallQueue = new LinkedBlockingQueue<Call>(maxQueueLength); // TODO hack on size
} else {
this.priorityCallQueue = null;
}
this.highPriorityLevel = highPriorityLevel;
this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000);
this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);
this.purgeTimeout = conf.getLong("ipc.client.call.purge.timeout",
2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
// Start the listener here and let it bind to the port
//Reactor主线程，接受connect请求
listener = new Listener();
......
// Create the responder here
//写回Response的线程，异步写
responder = new Responder();
}
Listener初始化
public Listener() throws IOException {
address = new InetSocketAddress(bindAddress, port);
// Create a new server socket and set to non blocking mode
//打开一个Server端Channel，非阻塞模式
acceptChannel = ServerSocketChannel.open();
acceptChannel.configureBlocking(false);
// Bind the server socket to the local host and port
//Bind端口
bind(acceptChannel.socket(), address, backlogLength);
port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port
// create a selector;
//Listener主线程的selector
selector= Selector.open();
//Reactor的子线程，负责对通道进行读取和处理，一个Reader可同时处理多个链接
readers = new Reader[readThreads];
//Reader线程池
readPool = Executors.newFixedThreadPool(readThreads,
new ThreadFactoryBuilder().setNameFormat(
"IPC Reader %d on port " + port).setDaemon(true).build());
for (int i = 0; i < readThreads; ++i) {
Reader reader = new Reader();
readers[i] = reader;
readPool.execute(reader);
}
// Register accepts on the server socket with the selector.
//注册ACCEPT事件，selector可以处理链接请求
acceptChannel.register(selector, SelectionKey.OP_ACCEPT);
this.setName("IPC Server listener on " + port);
this.setDaemon(true);
}
Responder初始化
Responder() throws IOException {
this.setName("IPC Server Responder");
this.setDaemon(true);
//先打开一个Selector，后续需要写回数据的通道会注册WRITE事件上来
writeSelector = Selector.open(); // create a selector
pending = 0;
}
初始化完成之后，RegionServer进行启动前的其他操作，比如新建zk链接，处理master，启动flush等线程。最终会启动RPC服务。
HBaseServer启动
public void start() {
startThreads();
openServer();
}
public synchronized void startThreads() {
//Responder线程启动
responder.start();
//Listener线程启动
listener.start();
//Handler线程启动，默认10
handlers = new Handler[handlerCount];
for (int i = 0; i < handlerCount; i++) {
handlers[i] = new Handler(callQueue, i);
handlers[i].start();
}
if (priorityHandlerCount > 0) {
priorityHandlers = new Handler[priorityHandlerCount];
for (int i = 0 ; i < priorityHandlerCount; i++) {
priorityHandlers[i] = new Handler(priorityCallQueue, i);
priorityHandlers[i].start();
}
}
}
Responder启动,Responder负责RPC响应写回操作，异步写
while (running) {
try {
waitPending(); // If a channel is being registered, wait.
//writeSelector监听OS WRITE READY的事件
writeSelector.select(purgeTimeout);
Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
iter.remove();
try {
if (key.isValid() && key.isWritable()) {
//异步写回，如果一次没写完，继续监听对应channel的WRITE事件，继续写
doAsyncWrite(key);
}
} catch (IOException e) {
LOG.info(getName() + ": doAsyncWrite threw exception " + e);
}
}
long now = System.currentTimeMillis();
if (now < lastPurgeTime + purgeTimeout) {
continue;
}
lastPurgeTime = now;
//
// If there were some calls that have not been sent out for a
// long time, discard them.
//
LOG.debug("Checking for old call responses.");
//长时间还没写完的请求，清理之，并关闭链接
ArrayList<Call> calls;
// get the list of channels from list of keys.
synchronized (writeSelector.keys()) {
calls = new ArrayList<Call>(writeSelector.keys().size());
iter = writeSelector.keys().iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
Call call = (Call)key.attachment();
if (call != null && key.channel() == call.connection.channel) {
calls.add(call);
}
}
}
for(Call call : calls) {
try {
doPurge(call, now);
} catch (IOException e) {
LOG.warn("Error in purging old calls " + e);
}
}
.......
}
写数据过程
private void doAsyncWrite(SelectionKey key) throws IOException {
Call call = (Call)key.attachment();
if (call == null) {
return;
}
if (key.channel() != call.connection.channel) {
throw new IOException("doAsyncWrite: bad channel");
}
synchronized(call.connection.responseQueue) {
//如果这个Connection的数据写完了，则清理key
if (processResponse(call.connection.responseQueue, false)) {
try {
key.interestOps(0);
} catch (CancelledKeyException e) {
/* The Listener/reader might have closed the socket.
* We don't explicitly cancel the key, so not sure if this will
* ever fire.
* This warning could be removed.
*/
LOG.warn("Exception while changing ops : " + e);
}
}
}
}
private boolean processResponse(final LinkedList<Call> responseQueue,
boolean inHandler) throws IOException {
boolean error = true;
boolean done = false; // there is more data for this channel.
int numElements;
Call call = null;
try {
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (responseQueue) {
//
// If there are no items for this channel, then we are done
//
numElements = responseQueue.size();
if (numElements == 0) {
error = false;
return true; // no more data for this channel.
}
//
// Extract the first call
//写请求
call = responseQueue.removeFirst();
SocketChannel channel = call.connection.channel;
if (LOG.isDebugEnabled()) {
LOG.debug(getName() + ": responding to #" + call.id + " from " +
call.connection);
}
//
// Send as much data as we can in the non-blocking fashion
//异步写回
int numBytes = channelWrite(channel, call.response);
if (numBytes < 0) {
return true;
}
//如果写完了，递减RPC统计数目
if (!call.response.hasRemaining()) {
call.connection.decRpcCount();
//noinspection RedundantIfStatement
if (numElements == 1) { // last call fully processes.
done = true; // no more data for this channel.
} else {
done = false; // more calls pending to be sent.
}
if (LOG.isDebugEnabled()) {
LOG.debug(getName() + ": responding to #" + call.id + " from " +
call.connection + " Wrote " + numBytes + " bytes.");
}
}
//如果没写完，放回去，下次继续写
else {
//
// If we were unable to write the entire response out, then
// insert in Selector queue.
//
call.connection.responseQueue.addFirst(call);
......
}
error = false; // everything went off well
}
} finally {
if (error && call != null) {
LOG.warn(getName()+", call " + call + ": output error");
done = true; // error. no more data for this channel.
closeConnection(call.connection);
}
}
return done;
}
具体channelWrite
protected int channelWrite(WritableByteChannel channel,
ByteBuffer buffer) throws IOException {
//如果小于BUFFER，直接写，否则分批写
int count = (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
channel.write(buffer) : channelIO(null, channel, buffer);
if (count > 0) {
rpcMetrics.sentBytes.inc(count);
}
return count;
}
分批写
private static int channelIO(ReadableByteChannel readCh,
WritableByteChannel writeCh,
ByteBuffer buf) throws IOException {
int originalLimit = buf.limit();
int initialRemaining = buf.remaining();
int ret = 0;
//没写完，一直写
while (buf.remaining() > 0) {
try {
//每次要写的数量，分批
int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
//写之前，写调整limit到目标index，写回的时候是取position和limit之间的数据
buf.limit(buf.position() + ioSize);
//异步写，有可能只写了部分数据
ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
//只写了部分数据，说明OS IO很繁忙，直接返回，下次继续写
if (ret < ioSize) {
break;
}
}
//每批数据写完，将limit重置
finally {
buf.limit(originalLimit);
}
}
//返回写了多少数据
int nBytes = initialRemaining - buf.remaining();
return (nBytes > 0) ? nBytes : ret;
}
Listener启动
while (running) {
SelectionKey key = null;
try {
//阻塞select，ACCEPT并发不高
selector.select(); // FindBugs IS2_INCONSISTENT_SYNC
Iterator<SelectionKey> iter = selector.selectedKeys().iterator();
while (iter.hasNext()) {
key = iter.next();
iter.remove();
try {
if (key.isValid()) {
if (key.isAcceptable())
doAccept(key);
}
} catch (IOException ignored) {
}
key = null;
}
......
}
accept过程
void doAccept(SelectionKey key) throws IOException, OutOfMemoryError {
Connection c;
ServerSocketChannel server = (ServerSocketChannel) key.channel();
SocketChannel channel;
//ACCEPT成功，则分配给Reader处理具体IO
while ((channel = server.accept()) != null) {
channel.configureBlocking(false);
channel.socket().setTcpNoDelay(tcpNoDelay);
channel.socket().setKeepAlive(tcpKeepAlive);
//顺序拿Reader
Reader reader = getReader();
try {
reader.startAdd();
//注册，READ事件到selector
SelectionKey readKey = reader.registerChannel(channel);
//业务层面的Connection，放入attach共享
c = getConnection(channel, System.currentTimeMillis());
readKey.attach(c);
//当前链接
synchronized (connectionList) {
connectionList.add(numConnections, c);
numConnections++;
}
......
} finally {
reader.finishAdd();
}
}
rpcMetrics.numOpenConnections.set(numConnections);
}
accept之后，可以进行IO操作，Reader处理
private synchronized void doRunLoop() {
while (running) {
SelectionKey key = null;
try {
//同步读
readSelector.select();
//有新链接进来，等之
while (adding) {
this.wait(1000);
}
alive = true;
Iterator<SelectionKey> iter = readSelector.selectedKeys().iterator();
while (iter.hasNext()) {
key = iter.next();
iter.remove();
if (key.isValid()) {
if (key.isReadable()) {
//读就位
doRead(key);
}
}
key = null;
}
} catch (InterruptedException e) {
if (running) { // unexpected -- log it
LOG.info(getName() + " unexpectedly interrupted: " +
StringUtils.stringifyException(e));
}
} catch (IOException ex) {
LOG.error("Error in Reader", ex);
} finally {
alive = false;
}
}
}
具体读
void doRead(SelectionKey key) throws InterruptedException {
int count = 0;
Connection c = (Connection)key.attachment();
if (c == null) {
return;
}
c.setLastContact(System.currentTimeMillis());
try {
//业务逻辑都封装在Connection里
count = c.readAndProcess();
} catch (InterruptedException ieo) {
throw ieo;
} catch (Exception e) {
LOG.warn(getName() + ": readAndProcess threw exception " + e + ". Count of bytes read: " + count, e);
count = -1; //so that the (count < 0) block is executed
}
if (count < 0) {
if (LOG.isDebugEnabled())
LOG.debug(getName() + ": disconnecting client " +
c.getHostAddress() + ". Number of active connections: "+
numConnections);
closeConnection(c);
// c = null;
}
else {
c.setLastContact(System.currentTimeMillis());
}
}
Connection的read
public int readAndProcess() throws IOException, InterruptedException {
while (true) {
/* Read at most one RPC. If the header is not read completely yet
* then iterate until we read first RPC or until there is no data left.
*/
int count;
//读int，4个字节，代表data长度
if (dataLengthBuffer.remaining() > 0) {
count = channelRead(channel, dataLengthBuffer);
if (count < 0 || dataLengthBuffer.remaining() > 0)
return count;
}
//version还没读，意味着这个client是刚连上来，则读version，一个字节
if (!versionRead) {
//Every connection is expected to send the header.
ByteBuffer versionBuffer = ByteBuffer.allocate(1);
count = channelRead(channel, versionBuffer);
if (count <= 0) {
return count;
}
int version = versionBuffer.get(0);
dataLengthBuffer.flip();
//前四个字节固定，或者版本不对，返回异常
if (!HEADER.equals(dataLengthBuffer) || version != CURRENT_VERSION) {
//Warning is ok since this is not supposed to happen.
LOG.warn("Incorrect header or version mismatch from " +
hostAddress + ":" + remotePort +
" got version " + version +
" expected version " + CURRENT_VERSION);
setupBadVersionResponse(version);
return -1;
}
//HEADER读完，继续下一个请求
dataLengthBuffer.clear();
versionRead = true;
continue;
}
//根据data length，读data
if (data == null) {
dataLengthBuffer.flip();
dataLength = dataLengthBuffer.getInt();
//-1代表ping请求
if (dataLength == HBaseClient.PING_CALL_ID) {
dataLengthBuffer.clear();
return 0; //ping message
}
//分配内存
data = ByteBuffer.allocate(dataLength);
incRpcCount(); // Increment the rpc count
}
//读入数据
count = channelRead(channel, data);
//读满了，继续业务处理，否则直接返回，下次继续读，一直读满为止
if (data.remaining() == 0) {
dataLengthBuffer.clear();
data.flip();
//header之后是业务请求
if (headerRead) {
processData(data.array());
data = null;
return count;
}
//读header信息，主要是初始化Connection的protocol属性
processHeader();
headerRead = true;
data = null;
continue;
}
return count;
}
}
请求处理
protected void processData(byte[] buf) throws IOException, InterruptedException {
DataInputStream dis =
new DataInputStream(new ByteArrayInputStream(buf));
//请求的客户端id
int id = dis.readInt(); // try to read an id
//请求大小
long callSize = buf.length;
......
// Enforcing the call queue size, this triggers a retry in the client
//大小超过限制，则返回异常
if ((callSize + callQueueSize.get()) > maxQueueSize) {
final Call callTooBig =
new Call(id, null, this, responder, callSize);
ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
setupResponse(responseBuffer, callTooBig, Status.FATAL, null,
IOException.class.getName(),
"Call queue is full, is ipc.server.max.callqueue.size too small?");
responder.doRespond(callTooBig);
return;
}
//param对象，默认Invocation
Writable param;
try {
param = ReflectionUtils.newInstance(paramClass, conf);//read param
param.readFields(dis);
} catch (Throwable t) {
LOG.warn("Unable to read call parameters for client " +
getHostAddress(), t);
final Call readParamsFailedCall =
new Call(id, null, this, responder, callSize);
ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
setupResponse(responseBuffer, readParamsFailedCall, Status.FATAL, null,
t.getClass().getName(),
"IPC server unable to read call parameters: " + t.getMessage());
responder.doRespond(readParamsFailedCall);
return;
}
//将请求添加到处理队列，后续Handler线程处理
Call call = new Call(id, param, this, responder, callSize);
//递增请求队列大小
callQueueSize.add(callSize);
if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
priorityCallQueue.put(call);
updateCallQueueLenMetrics(priorityCallQueue);
} else {
callQueue.put(call); // queue the call; maybe blocked here
updateCallQueueLenMetrics(callQueue);
}
}
Invocation反序列化
public void readFields(DataInput in) throws IOException {
//请求方法名
methodName = UTF8.readString(in);
//参数个数
parameters = new Object[in.readInt()];
//对应类型
parameterClasses = new Class[parameters.length];
ObjectWritable objectWritable = new ObjectWritable();
//每个参数根据不同类型，反序列化
for (int i = 0; i < parameters.length; i++) {
parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
parameterClasses[i] = objectWritable.getDeclaredClass();
}
}
序列化过程，支持原生类型，字符串，Writable实现，和数组
public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
throws IOException {
//类型->class对象
String className = UTF8.readString(in);
Class<?> declaredClass = PRIMITIVE_NAMES.get(className);
if (declaredClass == null) {
try {
declaredClass = conf.getClassByName(className);
} catch (ClassNotFoundException e) {
throw new RuntimeException("readObject can't find class " + className, e);
}
}
Object instance;
//原生类型
if (declaredClass.isPrimitive()) { // primitive types
if (declaredClass == Boolean.TYPE) { // boolean
instance = Boolean.valueOf(in.readBoolean());
} else if (declaredClass == Character.TYPE) { // char
instance = Character.valueOf(in.readChar());
} else if (declaredClass == Byte.TYPE) { // byte
instance = Byte.valueOf(in.readByte());
} else if (declaredClass == Short.TYPE) { // short
instance = Short.valueOf(in.readShort());
} else if (declaredClass == Integer.TYPE) { // int
instance = Integer.valueOf(in.readInt());
} else if (declaredClass == Long.TYPE) { // long
instance = Long.valueOf(in.readLong());
} else if (declaredClass == Float.TYPE) { // float
instance = Float.valueOf(in.readFloat());
} else if (declaredClass == Double.TYPE) { // double
instance = Double.valueOf(in.readDouble());
} else if (declaredClass == Void.TYPE) { // void
instance = null;
} else {
throw new IllegalArgumentException("Not a primitive: "+declaredClass);
}
}
//递归读数组
else if (declaredClass.isArray()) { // array
int length = in.readInt();
instance = Array.newInstance(declaredClass.getComponentType(), length);
for (int i = 0; i < length; i++) {
Array.set(instance, i, readObject(in, conf));
}
} else if (declaredClass == String.class) { // String
instance = UTF8.readString(in);
} else if (declaredClass.isEnum()) { // enum
instance = Enum.valueOf((Class<? extends Enum>) declaredClass, UTF8.readString(in));
}
//Writable
else { // Writable
Class instanceClass = null;
String str = "";
//实现类名
try {
str = UTF8.readString(in);
instanceClass = conf.getClassByName(str);
} catch (ClassNotFoundException e) {
throw new RuntimeException("readObject can't find class " + str, e);
}
//Writable实现类需要自己实现反序列化
Writable writable = WritableFactories.newInstance(instanceClass, conf);
writable.readFields(in);
instance = writable;
if (instanceClass == NullInstance.class) { // null
declaredClass = ((NullInstance)instance).declaredClass;
instance = null;
}
}
if (objectWritable != null) { // store values
objectWritable.declaredClass = declaredClass;
objectWritable.instance = instance;
}
return instance;
}
之后就是Handler线程处理了
public void run() {
LOG.info(getName() + ": starting");
status.setStatus("starting");
SERVER.set(HBaseServer.this);
while (running) {
try {
status.pause("Waiting for a call");
//从queue中拿请求
Call call = myCallQueue.take(); // pop the queue; maybe blocked here
updateCallQueueLenMetrics(myCallQueue);
status.setStatus("Setting up call");
status.setConnection(call.connection.getHostAddress(),
call.connection.getRemotePort());
if (LOG.isDebugEnabled())
LOG.debug(getName() + ": has #" + call.id + " from " +
call.connection);
String errorClass = null;
String error = null;
Writable value = null;
//ThreadLocal请求
CurCall.set(call);
alive = true;
try {
if (!started)
throw new ServerNotRunningYetException("Server is not running yet");
......
RequestContext.set(call.connection.ticket, getRemoteIp(),
call.connection.protocol);
// make the call
//反射调用对应服务，返回结果
value = call(call.connection.protocol, call.param, call.timestamp,
status);
} catch (Throwable e) {
LOG.debug(getName()+", call "+call+": error: " + e, e);
errorClass = e.getClass().getName();
error = StringUtils.stringifyException(e);
} finally {
alive = false;
// Must always clear the request context to avoid leaking
// credentials between requests.
RequestContext.clear();
}
CurCall.set(null);
//减小queue大小
callQueueSize.add(call.getSize() * -1);
// Set the response for undelayed calls and delayed calls with
// undelayed responses.
//序列化写回response
if (!call.isDelayed() || !call.isReturnValueDelayed()) {
call.setResponse(value,
errorClass == null? Status.SUCCESS: Status.ERROR,
errorClass, error);
}
//通过responder写回响应
call.sendResponseIfReady();
status.markComplete("Sent response");
}
.......
}
}
反射调用过程
public Writable call(Class<? extends VersionedProtocol> protocol,
Writable param, long receivedTime, MonitoredRPCHandler status)
throws IOException {
try {
Invocation call = (Invocation)param;
if(call.getMethodName() == null) {
throw new IOException("Could not find requested method, the usual " +
"cause is a version mismatch between client and server.");
}
if (verbose) log("Call: " + call);
status.setRPC(call.getMethodName(), call.getParameters(), receivedTime);
status.setRPCPacket(param);
status.resume("Servicing call");
//method对象
Method method =
protocol.getMethod(call.getMethodName(),
call.getParameterClasses());
//make invokable
method.setAccessible(true);
//Verify protocol version.
//Bypass the version check for VersionedProtocol
if (!method.getDeclaringClass().equals(VersionedProtocol.class)) {
long clientVersion = call.getProtocolVersion();
ProtocolSignature serverInfo = ((VersionedProtocol) instance)
.getProtocolSignature(protocol.getCanonicalName(), call
.getProtocolVersion(), call.getClientMethodsHash());
long serverVersion = serverInfo.getVersion();
if (serverVersion != clientVersion) {
LOG.warn("Version mismatch: client version=" + clientVersion
+ ", server version=" + serverVersion);
throw new RPC.VersionMismatch(protocol.getName(), clientVersion,
serverVersion);
}
}
Object impl = null;
//实现类是HRegionServer
if (protocol.isAssignableFrom(this.implementation)) {
impl = this.instance;
}
else {
throw new HBaseRPC.UnknownProtocolException(protocol);
}
long startTime = System.currentTimeMillis();
Object[] params = call.getParameters();
//反射调用
Object value = method.invoke(impl, params);
int processingTime = (int) (System.currentTimeMillis() - startTime);
int qTime = (int) (startTime-receivedTime);
......
rpcMetrics.rpcQueueTime.inc(qTime);
rpcMetrics.rpcProcessingTime.inc(processingTime);
rpcMetrics.inc(call.getMethodName(), processingTime);
if (verbose) log("Return: "+value);
//包装返回对象
HbaseObjectWritable retVal =
new HbaseObjectWritable(method.getReturnType(), value);
long responseSize = retVal.getWritableSize();
// log any RPC responses that are slower than the configured warn
// response time or larger than configured warning size
boolean tooSlow = (processingTime > warnResponseTime
&& warnResponseTime > -1);
boolean tooLarge = (responseSize > warnResponseSize
&& warnResponseSize > -1);
if (tooSlow || tooLarge) {
// when tagging, we let TooLarge trump TooSmall to keep output simple
// note that large responses will often also be slow.
logResponse(call, (tooLarge ? "TooLarge" : "TooSlow"),
status.getClient(), startTime, processingTime, qTime,
responseSize);
// provides a count of log-reported slow responses
if (tooSlow) {
rpcMetrics.rpcSlowResponseTime.inc(processingTime);
}
}
if (processingTime > 1000) {
// we use a hard-coded one second period so that we can clearly
// indicate the time period we're warning about in the name of the
// metric itself
rpcMetrics.inc(call.getMethodName() + ABOVE_ONE_SEC_METRIC,
processingTime);
}
return retVal;
} catch (InvocationTargetException e) {
Throwable target = e.getTargetException();
if (target instanceof IOException) {
throw (IOException)target;
}
IOException ioe = new IOException(target.toString());
ioe.setStackTrace(target.getStackTrace());
throw ioe;
} catch (Throwable e) {
if (!(e instanceof IOException)) {
LOG.error("Unexpected throwable object ", e);
}
IOException ioe = new IOException(e.toString());
ioe.setStackTrace(e.getStackTrace());
throw ioe;
}
}
响应对象序列化
protected synchronized void setResponse(Object value, Status status,
String errorClass, String error) {
// Avoid overwriting an error value in the response. This can happen if
// endDelayThrowing is called by another thread before the actual call
// returning.
if (this.isError)
return;
if (errorClass != null) {
this.isError = true;
}
Writable result = null;
if (value instanceof Writable) {
result = (Writable) value;
} else {
/* We might have a null value and errors. Avoid creating a
* HbaseObjectWritable, because the constructor fails on null. */
if (value != null) {
result = new HbaseObjectWritable(value);
}
}
//序列化大小
int size = BUFFER_INITIAL_SIZE;
if (result instanceof WritableWithSize) {
// get the size hint.
WritableWithSize ohint = (WritableWithSize) result;
long hint = ohint.getWritableSize() + Bytes.SIZEOF_BYTE +
(2 * Bytes.SIZEOF_INT);
if (hint > Integer.MAX_VALUE) {
// oops, new problem.
IOException ioe =
new IOException("Result buffer size too large: " + hint);
errorClass = ioe.getClass().getName();
error = StringUtils.stringifyException(ioe);
} else {
size = (int)hint;
}
}
ByteBufferOutputStream buf = new ByteBufferOutputStream(size);
DataOutputStream out = new DataOutputStream(buf);
try {
// Call id.
//客户端请求id
out.writeInt(this.id);
// Write flag.
//异常标示
byte flag = (error != null)?
ResponseFlag.getErrorAndLengthSet(): ResponseFlag.getLengthSetOnly();
out.writeByte(flag);
// Place holder for length set later below after we
// fill the buffer with data.
//长度占位
out.writeInt(0xdeadbeef);
//处理结果
out.writeInt(status.state);
} catch (IOException e) {
errorClass = e.getClass().getName();
error = StringUtils.stringifyException(e);
}
try {
//序列化响应对象
if (error == null) {
result.write(out);
}
//异常信息
else {
WritableUtils.writeString(out, errorClass);
WritableUtils.writeString(out, error);
}
} catch (IOException e) {
LOG.warn("Error sending response to call: ", e);
}
// Set the length into the ByteBuffer after call id and after
// byte flag.
ByteBuffer bb = buf.getByteBuffer();
//数据总大小
int bufSiz = bb.remaining();
// Move to the size location in our ByteBuffer past call.id
// and past the byte flag.
//长度占位字节填充
bb.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE);
bb.putInt(bufSiz);
bb.position(0);
this.response = bb;
}
最终数据被写回client，client部分代码见下一篇