Article Directory
Foreword
For Java nio ByteBuffer, we often used to do data processing buffer. If we map the sake of convenience, each data read and write operations ByteBuffer specifically allocate a relatively large capacity, this will result in unnecessary waste of the JVM heap. But if we turn to become a dynamic application of multiple small ByteBuffer, ByteBuffer will increase the management and coordination of operations. So is there any way to combine the characteristics of both the above, so that neither wasteful use JVM heap space, it does not perform complex logic ByteBuffer in business. In this paper the author describes the implementation of an internal incremental ByteBuffer Ozone implemented. Incremental ByeBuffer use on the exterior and is fully compatible with the native r ByteBuffe operation semantics, the internal operation of the increment allocate capacity to those the caller is completely transparent.
Ozone achieve incremental internal ByteBuffer
Here a brief background of Ozone, Ozone as a target storage system, when the target file is stored, the write involves a large number of small data objects, physical files stored in the form of Chunk. In the process of reading and writing data chunk, Ozone ByteBuffer used to make the intermediate data storage. In the initial implementation, the internal Ozone initial ByteBuffer allocate the capacity is relatively large, regardless of the user data is also written how much. For this reason, then the community can achieve the target capacity, while the increment specified ByteBuffer dynamic allocation size, called IncrementalChunkBuffer.
The following is IncrementalChunkBuffer implementation code, taken from the Ozone project:
/**
* Use a list of {@link ByteBuffer} to implement a single {@link ChunkBuffer}
* so that the buffer can be allocated incrementally.
*/
final class IncrementalChunkBuffer implements ChunkBuffer {
/**
* 全局Buffer的limit边界值
*/
private final int limit;
/** ByteBuffer的增量容量. */
private final int increment;
/** BytesBuffer下标数组边界值. */
private final int limitIndex;
/** 增量分配ByteBuffer数组 */
private final List<ByteBuffer> buffers;
/** Is this a duplicated buffer? (for debug only) */
private final boolean isDuplicated;
// 增量ByteBuffer总capacity(传入的limit值)，每次动态增量increment值大小
IncrementalChunkBuffer(int limit, int increment, boolean isDuplicated) {
Preconditions.checkArgument(limit >= 0);
Preconditions.checkArgument(increment > 0);
// 初始化边界值，增量ByteBuffer值
this.limit = limit;
this.increment = increment;
// 计算ByteBuffer数组下标最大值
this.limitIndex = limit/increment;
// 初始空ByteBuffer数组
this.buffers = new ArrayList<>(limitIndex + (limit%increment == 0? 0: 1));
this.isDuplicated = isDuplicated;
}
/** @return the capacity for the buffer at the given index. */
private int getBufferCapacityAtIndex(int i) {
Preconditions.checkArgument(i >= 0);
Preconditions.checkArgument(i <= limitIndex);
return i < limitIndex? increment: limit%increment;
}
private void assertInt(int expected, int computed, String name, int i) {
ChunkBuffer.assertInt(expected, computed,
() -> this + ": Unexpected " + name + " at index " + i);
}
/** @return the i-th buffer if it exists; otherwise, return null. */
private ByteBuffer getAtIndex(int i) {
Preconditions.checkArgument(i >= 0);
Preconditions.checkArgument(i <= limitIndex);
final ByteBuffer ith = i < buffers.size() ? buffers.get(i) : null;
if (ith != null) {
// assert limit/capacity
if (!isDuplicated) {
assertInt(getBufferCapacityAtIndex(i), ith.capacity(), "capacity", i);
} else {
if (i < limitIndex) {
assertInt(increment, ith.capacity(), "capacity", i);
} else if (i == limitIndex) {
assertInt(getBufferCapacityAtIndex(i), ith.limit(), "capacity", i);
} else {
assertInt(0, ith.limit(), "capacity", i);
}
}
}
return ith;
}
/** @return the i-th buffer. It may allocate buffers. */
private ByteBuffer getAndAllocateAtIndex(int index) {
Preconditions.checkArgument(index >= 0);
// never allocate over limit
if (limit % increment == 0) {
Preconditions.checkArgument(index < limitIndex);
} else {
Preconditions.checkArgument(index <= limitIndex);
}
int i = buffers.size();
if (index < i) {
return getAtIndex(index);
}
// allocate upto the given index
ByteBuffer b = null;
for (; i <= index; i++) {
b = ByteBuffer.allocate(getBufferCapacityAtIndex(i));
buffers.add(b);
}
return b;
}
/** @return the buffer containing the position. It may allocate buffers. */
private ByteBuffer getAndAllocateAtPosition(int position) {
Preconditions.checkArgument(position >= 0);
Preconditions.checkArgument(position < limit);
// 计算需要获取ByteBuffer的下标
final int i = position / increment;
// 得到此下标对应的ByteBuffer，如果没有创建，则进行ByteBuffer的创建操作
final ByteBuffer ith = getAndAllocateAtIndex(i);
assertInt(position%increment, ith.position(), "position", i);
return ith;
}
/** @return the index of the first non-full buffer. */
private int firstNonFullIndex() {
for (int i = 0; i < buffers.size(); i++) {
if (getAtIndex(i).position() != increment) {
return i;
}
}
return buffers.size();
}
@Override
public int position() {
// The buffers list must be in the following orders:
// full buffers, buffer containing the position, empty buffers, null buffers
// 1）寻找第一个不满的ByteBuffer下标
final int i = firstNonFullIndex();
// 2）获取此ByteBuffer项
final ByteBuffer ith = getAtIndex(i);
// 3）计算当前全局position位置，前面i个满ByteBuffer的长度+当前不满的ByteBuffer的position位置
final int position = i * increment + Optional.ofNullable(ith)
.map(ByteBuffer::position).orElse(0);
// remaining buffers must be empty
assert assertRemainingList(ith, i);
return position;
}
private boolean assertRemainingList(ByteBuffer ith, int i) {
if (ith != null) {
// buffers must be empty
for (i++; i < buffers.size(); i++) {
ith = getAtIndex(i);
if (ith == null) {
break; // found the first non-null
}
assertInt(0, ith.position(), "position", i);
}
}
final int j = i;
ChunkBuffer.assertInt(buffers.size(), i,
() -> "i = " + j + " != buffers.size() = " + buffers.size());
return true;
}
@Override
public int remaining() {
// remaining操作和原ByteBuffer语义一致
return limit - position();
}
@Override
public int limit() {
return limit;
}
@Override
public ChunkBuffer rewind() {
buffers.forEach(ByteBuffer::rewind);
return this;
}
@Override
public ChunkBuffer clear() {
buffers.forEach(ByteBuffer::clear);
return this;
}
@Override
public ChunkBuffer put(ByteBuffer that) {
// 1）判断待put操作的ByteBuffer中的剩余数据空间是否超过增量ByteBuffer当前剩余空间
// 如果超过，则抛出BufferOverflowException异常
if (that.remaining() > this.remaining()) {
final BufferOverflowException boe = new BufferOverflowException();
boe.initCause(new IllegalArgumentException(
"Failed to put since that.remaining() = " + that.remaining()
+ " > this.remaining() = " + this.remaining()));
throw boe;
}
// 2）得到待写入ByteBuffer的边界值
final int thatLimit = that.limit();
for(int p = position(); that.position() < thatLimit;) {
// 3）得到当前增量Buffer内正在提供写操作的ByteBuffer
final ByteBuffer b = getAndAllocateAtPosition(p);
// 4）比较剩余数据空间的大小，选取上述2个Buffer的较小值
final int min = Math.min(b.remaining(), thatLimit - that.position());
that.limit(that.position() + min);
// 进行ByteBuffer的写入操作
b.put(that);
// 更新增量ByteBuffer当前的position位置
p += min;
}
return this;
}
@Override
public ChunkBuffer duplicate(int newPosition, int newLimit) {
Preconditions.checkArgument(newPosition >= 0);
Preconditions.checkArgument(newPosition <= newLimit);
Preconditions.checkArgument(newLimit <= limit);
final IncrementalChunkBuffer duplicated = new IncrementalChunkBuffer(
newLimit, increment, true);
final int pi = newPosition / increment;
final int pr = newPosition % increment;
final int li = newLimit / increment;
final int lr = newLimit % increment;
final int newSize = lr == 0? li: li + 1;
for (int i = 0; i < newSize; i++) {
final int pos = i < pi ? increment : i == pi ? pr : 0;
final int lim = i < li ? increment : i == li ? lr : 0;
duplicated.buffers.add(duplicate(i, pos, lim));
}
return duplicated;
}
private ByteBuffer duplicate(int i, int pos, int lim) {
final ByteBuffer ith = getAtIndex(i);
Objects.requireNonNull(ith, () -> "buffers[" + i + "] == null");
final ByteBuffer b = ith.duplicate();
b.position(pos).limit(lim);
return b;
}
/** Support only when bufferSize == increment. */
@Override
public Iterable<ByteBuffer> iterate(int bufferSize) {
if (bufferSize != increment) {
throw new UnsupportedOperationException(
"Buffer size and increment mismatched: bufferSize = " + bufferSize
+ " but increment = " + increment);
}
return asByteBufferList();
}
@Override
public List<ByteBuffer> asByteBufferList() {
return Collections.unmodifiableList(buffers);
}
@Override
public long writeTo(GatheringByteChannel channel) throws IOException {
return channel.write(buffers.toArray(new ByteBuffer[0]));
}
@Override
public ByteString toByteStringImpl(Function<ByteBuffer, ByteString> f) {
return buffers.stream().map(f).reduce(ByteString.EMPTY, ByteString::concat);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
} else if (!(obj instanceof IncrementalChunkBuffer)) {
return false;
}
final IncrementalChunkBuffer that = (IncrementalChunkBuffer)obj;
return this.limit == that.limit && this.buffers.equals(that.buffers);
}
@Override
public int hashCode() {
return buffers.hashCode();
}
@Override
public String toString() {
return getClass().getSimpleName()
+ ":limit=" + limit + ",increment=" + increment;
}
}
Quote
[1].https://github.com/apache/hadoop-ozone/blob/master/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/IncrementalChunkBuffer.java
