HANBO autopsy
hanbo is a high-performance, high-availability, low-latency in-memory database.
Protocol overview
Same as redis protocol, parsed by \r\n.
decoding
public class RedisCommandDecoder extends ReplayingDecoder<Void> {
private byte[][] bytes;
@Override
public void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
if (bytes != null) {
int numArgs = bytes.length;
for (int i = 0; i < numArgs; i++) {
if (in.readByte() == '$') {
long l = readLong(in);
if (l > Integer.MAX_VALUE) {
throw new IllegalArgumentException("Java only supports arrays up to " + Integer.MAX_VALUE + " in size");
}
int size = (int) l;
bytes[i] = new byte[size];
in.readBytes(bytes[i]);
if (in.bytesBefore((byte) '\r') != 0) {
throw new Exception("Argument doesn't end in CRLF");
}
in.skipBytes(2);
checkpoint();
} else {
throw new IOException("Unexpected character");
}
}
try {
out.add(new Command(bytes));
} finally {
bytes = null;
}
} else if (in.readByte() == '*') {
long l = readLong(in);
if (l > Integer.MAX_VALUE) {
throw new IllegalArgumentException("Java only supports arrays up to " + Integer.MAX_VALUE + " in size");
}
int numArgs = (int) l;
if (numArgs < 0) {
throw new Exception("Invalid size: " + numArgs);
}
bytes = new byte[numArgs][];
checkpoint();
decode(ctx, in, out);
} else {
// Go backwards one
in.readerIndex(in.readerIndex() - 1);
// Read command -- can't be interrupted
byte[][] b = new byte[1][];
b[0] = in.readBytes(in.bytesBefore((byte) '\r')).array();
in.skipBytes(2);
out.add(new Command(b, true));
}
}
}
encoding, BulkReply
public void write(ByteBuf os) throws IOException {
os.writeByte(MARKER);
os.writeBytes(numToBytes(capacity, true));
if (capacity > 0) {
os.writeBytes(bytes);
os.writeBytes(CRLF);
}
}
System overview
Core class diagram
command entry
protected void channelRead0(ChannelHandlerContext ctx, Command msg) throws Exception {
Reply reply = invoker.handlerEvent(ctx, msg);
if (reply == QUIT) {
ctx.close();
} else {
if (msg.isInline()) {
if (reply == null) {
reply = new InlineReply(null);
} else {
reply = new InlineReply(reply.data());
}
}
if (reply == null) {
reply = NYI_REPLY;
}
if (reply instanceof MultiBulkReply) {
MultiBulkReply multiBulkReply = (MultiBulkReply) reply;
if (multiBulkReply == MultiBulkReply.BLOCKING_QUEUE) {
return;
}
}
ctx.write(reply);
}
}
request processing
Multi-db implementation, similar to threadlocal, but here is the session level (channel), not the thread
public StatusReply select(byte[] index0) throws RedisException {
DatabaseRouter.RedisDB store = databaseRouter.select(Integer.parseInt(new String(index0)));
Attribute attribute = channelHandlerContext.channel().attr(session);
if (null == store) {
attribute.set(null);
throw new RedisException();
}
attribute.set(store);
return StatusReply.OK;
}
Transaction control, based on session control, if there is a multi command, the subsequent commands are queued, waiting for the execution/cancellation of the exec/discard command
public boolean hasOpenTx() {
return getTxAttribute().get() != null;
}
public Attribute getTxAttribute() {
return channelHandlerContext.channel().attr(transaction);
}
public Reply handlerTxOp(Command command) throws RedisException {
if (new String(command.getName()).equals("exec")) {
return exec();
}
if (new String(command.getName()).equals("discard")) {
return discard();
}
Queue queue = (Queue) getTxAttribute().get();
//declare internal event for command
command.setEventType(1);
queue.add(command);
return StatusReply.QUEUED;
}
data storage
Map files to off-heap memory
public BaseMedia(int db, String fileName, int memSize) throws Exception {
if (db == 0)
f = new File(defaultFile.getAbsolutePath() + File.separator + fileName);
else
f = new File(defaultFile.getParentFile().getAbsolutePath() + File.separator + db + File.separator + fileName);
if (!f.exists())
f.createNewFile();
fileChannel = new RandomAccessFile(f, "rw").getChannel();
long fileSize = Math.max(memSize * size, fileChannel.size());
buffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, fileSize == 0 ? memSize * size : fileSize);
}
CRUD based on off-heap memory
public DataHelper add(ByteBuffer b) throws Exception {
int pos;
if ((pos = buffer.getInt()) != 0)
buffer.position(pos);
else
buffer.position(4);
resize(pos);
buffer.put(b);
buffer.putChar(NORMAL);
DataHelper dh = new DataHelper();
dh.pos = pos == 0 ? 4 + 4 : pos + 4;
int curPos = buffer.position();
buffer.position(0);
buffer.putInt(curPos);//head 4 byte in last postion
buffer.rewind();
return dh;
}
public byte[] get(DataHelper dh) {
buffer.position(dh.pos);
byte[] data = new byte[dh.length];
buffer.get(data);
if (buffer.getChar() == DELETE)
return null;
buffer.rewind();
return data;
}
public void remove(DataHelper dh) {
buffer.position(dh.pos + dh.length);
buffer.putChar(DELETE);
buffer.rewind();
}
public DataHelper update(DataHelper dh, byte[] newBuf) {
buffer.position(dh.pos - 4);
int length = newBuf.length;
if (length > maxUnit)
throw new RuntimeException("exceed max storage limited exception");
else {
buffer.putInt(length);
buffer.put(newBuf);
dh.length = length;
buffer.rewind();
return dh;
}
}
set/get, corresponding to write and read
public boolean write(String key, String value) {
try {
if (super.write(key, value)) {
DataHelper dataHelper = (DataHelper) indexHelper.type(key);
if (dataHelper != null) {
dataHelper = dataMedia.update(dataHelper, value.getBytes(Charsets.UTF_8));
indexHelper.updateIndex(dataHelper);
return true;
} else {
ByteBuffer b = ByteBuffer.allocateDirect(128);
int length = value.getBytes().length;
b.putInt(length);
b.put(value.getBytes(Charsets.UTF_8));
b.flip();
DataHelper dh = dataMedia.add(b);
dh.setKey(key);
dh.setLength(length);
indexHelper.add(dh);
return true;
}
}
} catch (Exception e) {
log.error("write data error", e);
}
return false;
}
public byte[] read(String key) throws RedisException {
try {
if (!checkKeyType(key)) {
throw new RedisException("Operation against a key holding the wrong kind of value");
}
if (super.isExpire(key)) {
return null;
}
long start = System.currentTimeMillis();
DataHelper idx = (DataHelper) indexHelper.type(key);
if (idx == null) {
return null;
}
byte[] data = dataMedia.get(idx);
String resp = new String(data, Charsets.UTF_8);
log.debug("key={},value={} cost={}ms", key, resp, (System.currentTimeMillis() - start));
return data;
} catch (Exception e) {
log.error("read data error", e);
throw e;
}
}
High performance collection library - fastUtil
protected Map<String, Object> keyMap = new Object2ObjectAVLTreeMap<>();Automatic expansion
public void reAllocate() throws Exception {
System.err.println("reAllocate file begin");
String dir = f.getParentFile().getParentFile().getAbsolutePath();
File newFile = new File(dir + File.separator + accessIndex + File.separator + f.getName() + "_tmp");
com.google.common.io.Files.copy(f, newFile);
File newFile_ = new File(f.getAbsolutePath());
clean();
com.google.common.io.Files.copy(newFile, newFile_);
newFile.delete();
fileChannel = new RandomAccessFile(newFile_, "rw").getChannel();
buffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, buffer.capacity() * 2);
System.err.println("reAllocate file end");
}
public void resize(int pos) throws Exception {
//reallocate buffer
if (buffer.remaining() - pos < 4) {
reAllocate();
if ((pos = buffer.getInt()) != 0)
buffer.position(pos);
else
buffer.position(4);
}
}
run
java -jar hanboServer.jar
single node
configure
#服务地址
server.host=127.0.0.1
#服务端口
server.port=16379
#内存大小
memorySize=32
#db数量
dbSize=8
logging.level.root=error
master-slave configuration
Just add the following configuration items
Main placement
replication.mode=master
from config
replication.mode=slave
slaver.of=127.0.0.1:16379
project address:
https://github.com/3kuai/jredis
https://gitee.com/lmx_007/jredis
Welcome more partners to suggest and improve it
No public
{{o.name}}
{{m.name}}