netty 的 ChannelOutboundBuffer

netty, each channel has a write buffer ChannelOutboundBuffer

ChannelOutboundBuffer maintaining a class Entry list, the list is a node Entry, encapsulates ByteBuf to be written, and the socket is netty ByteBuffer finally written, it will eventually turn ByteBuf ByteBuffer

static  Final  class the Entry {
     // Not surprisingly, object pooling 
    Private  static  Final the ObjectPool <the Entry> RECYCLER = ObjectPool.newPool ( new new ObjectCreator <the Entry> () { 
        @Override 
        public the Entry the newObject (the Handle <the Entry> handle) {
             return  new new the Entry (handle); 
        } 
    }); 

    Private  Final the handle <the Entry> handle;
     // next node 
    the Entry next;
     // message content, i.e. ByteBuf 
    Object MSG;
     // general, corresponds to a bottom of a ByteBuf the ByteBuffer
     //So bufs is empty most of the time, only buf will be assigned 
    ByteBuffer [] bufs;
     // actually written to the socket data structure 
    ByteBuffer buf;
     // corresponds to write a successful callback 
    ChannelPromise Promise;
     // ByteBuf word has been written in socket section number 
    Long Progress;
     // number of bytes ByteBuf readable 
    Long Total;
     int pendingSize;
     int COUNT = -1 ;
     Boolean Canceled; 
}

// no need to write the pointer in the socket Entry 
Private Entry unflushedEntry; 

// to be written to the pointer of the socket Entry 
Private Entry flushedEntry; 

// tail 
Private Entry tailEntry; 

// Number Entry socket is to be written
 // equal Entry from the number flushedEntry between unflushedEntry, not including unflushedEntry 
Private  int flushed;

Each call HeadContext.write final trigger addMessage, the data behind the increase in tailEntry

Add Entry

public void addMessage(Object msg, int size, ChannelPromise promise) {
    Entry entry = Entry.newInstance(msg, size, total(msg), promise);
    if (tailEntry == null) {
        flushedEntry = null;
    } else {
        Entry tail = tailEntry;
        tail.next = entry;
    }
    tailEntry = entry;
    if (unflushedEntry == null) {
        unflushedEntry = entry;
    }

    incrementPendingOutboundBytes(entry.pendingSize, false);
}

Each call HeadContext.flush final trigger and flush addFlush

// io.netty.channel.AbstractChannel.AbstractUnsafe#flush
public final void flush() {
    assertEventLoop();

    ChannelOutboundBuffer outboundBuffer = this.outboundBuffer;
    if (outboundBuffer == null) {
        return;
    }
    
    // 移动 flushedEntry 和 unflushedEntry 指针
    outboundBuffer.addFlush();
    // 真正写 socket
    flush0();
}

FlushedEntry pointer movement and unflushedEntry

public  void addFlush () { 
    the Entry entry = unflushedEntry;
     IF (! entry = null ) {
         IF (flushedEntry == null ) {
             // If flushedEntry pointer is empty, directly to unflushedEntry, finally unflushedEntry blanking 
            flushedEntry = entry; 
        } 
        // If flushedEntry pointer is not empty, directly to the blanking unflushedEntry 
        do { 
            flushed ++ ;
             IF (! entry.promise.setUncancellable ()) {
                 int Pending = entry.cancel ();
                decrementPendingOutboundBytes(pending, false, true);
            }
            entry = entry.next;
        } while (entry != null);

        // All flushed so reset unflushedEntry
        unflushedEntry = null;
    }
}

Incidentally, only a buffer linked list, need to write to the socket between Entry from flushedEntry to unflushedEntry, not including unflushedEntry

We know that
after the flush, if the data is sufficient, and each successful write, netty default will continue to write 16 times

// io.netty.channel.socket.nio.NioSocketChannel#doWrite
protected void doWrite(ChannelOutboundBuffer in) throws Exception {
    SocketChannel ch = javaChannel();
    // 默认 16 次
    int writeSpinCount = config().getWriteSpinCount();
    do {
        // 当 ChannelOutboundBuffer 无可写的数据，返回
        if (in.isEmpty()) {
            // All written so clear OP_WRITE
            clearOpWrite();
            // Directly return here so incompleteWrite(...) is not called.
            return;
        }

        // Ensure the pending writes are made of ByteBufs only.
        int maxBytesPerGatheringWrite = ((NioSocketChannelConfig) config).getMaxBytesPerGatheringWrite();
        // 把 ChannelOutboundBuffer 中的 msg，转换成 ByteBuffer
        ByteBuffer[] nioBuffers = in.nioBuffers(1024, maxBytesPerGatheringWrite);
        // ByteBuffer 的数量
        int nioBufferCnt = in.nioBufferCount();

        switch (nioBufferCnt) {
            case 0:
                // We have something else beside ByteBuffers to write so fallback to normal writes.
                writeSpinCount -= doWrite0(in);
                BREAK ;
             Case . 1 : {
                 // the simplest case 
                ByteBuffer nioBuffers = Buffer [0 ];
                 int attemptedBytes = buffer.remaining ();
                 // the write ByteBuffer Socket 
                Final  int localWrittenBytes = ch.write (Buffer);
                 IF (localWrittenBytes <= 0 ) {
                     // If the socket is not written, OP_WRITE the registration event 
                    incompleteWrite ( to true );
                     return ; 
                } 
                // the amount of adjustment of the next number of bytes written according to the write
                adjustMaxBytesPerGatheringWrite(attemptedBytes, localWrittenBytes, maxBytesPerGatheringWrite);
                // 删除 ChannelOutboundBuffer 中的 Entry
                in.removeBytes(localWrittenBytes);
                --writeSpinCount;
                break;
            }
            default: {
                // Zero length buffers are not added to nioBuffers by ChannelOutboundBuffer, so there is no need
                // to check if the total size of all the buffers is non-zero.
                // We limit the max amount to int above so cast is safe
                long attemptedBytes = in.nioBufferSize();
                final long localWrittenBytes = ch.write(nioBuffers, 0, nioBufferCnt);
                if (localWrittenBytes <= 0) {
                    incompleteWrite(true);
                    return;
                }
                // Casting to int is safe because we limit the total amount of data in the nioBuffers to int above.
                adjustMaxBytesPerGatheringWrite((int) attemptedBytes, (int) localWrittenBytes,
                        maxBytesPerGatheringWrite);
                in.removeBytes(localWrittenBytes);
                --writeSpinCount;
                break;
            }
        }
    } while (writeSpinCount > 0);

    incompleteWrite(writeSpinCount < 0);
}

Convert all of ByteBuf flushedEntry into ByteBuffer

// io.netty.channel.ChannelOutboundBuffer#nioBuffers(int, long)
public ByteBuffer[] nioBuffers(int maxCount, long maxBytes) {
    assert maxCount > 0;
    assert maxBytes > 0;
    long nioBufferSize = 0;
    int nioBufferCount = 0;
    final InternalThreadLocalMap threadLocalMap = InternalThreadLocalMap.get();
    ByteBuffer[] nioBuffers = NIO_BUFFERS.get(threadLocalMap);
    Entry entry = flushedEntry;
    // 遍历 flushedEntry
    while (isFlushedEntry(entry) && entry.msg instanceof ByteBuf) {
        if (!entry.cancelled) {
            ByteBuf buf = (ByteBuf) entry.msg;
            final int readerIndex = buf.readerIndex();
            final int readableBytes = buf.writerIndex() - readerIndex;

            if (readableBytes > 0) {
                if (maxBytes - readableBytes < nioBufferSize && nioBufferCount != 0) {
                    break;
                }
                nioBufferSize += readableBytes;
                int count = entry.count;
                if (count == -1) {
                    entry.count = count = buf.nioBufferCount();
                }
                int neededSpace = min(maxCount, nioBufferCount + count);
                if (neededSpace > nioBuffers.length) {
                    nioBuffers = expandNioBufferArray(nioBuffers, neededSpace, nioBufferCount);
                    NIO_BUFFERS.set(threadLocalMap, nioBuffers);
                }
                if (count == 1) {
                    ByteBuffer nioBuf = entry.buf;
                    if (nioBuf == null) {
                        entry.buf = nioBuf = buf.internalNioBuffer(readerIndex, readableBytes);
                    }
                    nioBuffers[nioBufferCount++] = nioBuf;
                } else {
                    nioBufferCount = nioBuffers(entry, buf, nioBuffers, nioBufferCount, maxCount);
                }
                if (nioBufferCount == maxCount) {
                    break;
                }
            }
        }
        entry = entry.next;
    }
    this.nioBufferCount = nioBufferCount;
    this.nioBufferSize = nioBufferSize;

    return nioBuffers;
}

Delete Entry
according to the number of bytes written, deleted Entry

public void removeBytes(long writtenBytes) {
    for (;;) {
        // 当前 flushedEntry 节点
        Object msg = current();
        if (!(msg instanceof ByteBuf)) {
            assert writtenBytes == 0;
            break;
        }

        final ByteBuf buf = (ByteBuf) msg;
        final int readerIndex = buf.readerIndex();
        final int readableBytes = buf.writerIndex() - readerIndex;

        //Writing data larger than the current data flushedEntry, i.e. the finished flushedEntry 
        IF (readableBytes <= writtenBytes) {
             IF (writtenBytes = 0! ) {
                 // update progress 
                Progress (readableBytes); 
                writtenBytes - = readableBytes; 
            } 
            // remove flushedEntry the node pointed to by moving backward flushedEntry 
            Remove (); 
        } the else { // readableBytes> writtenBytes
             // the flushedEntry not finished, only the renewal of the progress 
            IF (writtenBytes = 0! ) { 
                buf.readerIndex (readerIndex + (int) writtenBytes);
                progress(writtenBytes);
            }
            break;
        }
    }
    clearNioBuffers();
}

High-water mark and low-water mark

netty pending statistical data, more than the high-water mark of the sign change, attention, changed the logo, you can also write your own judgment needs to continue to write or not to write.

). IsWritable () Retrieves whether writable by ctx.channel (

// 利用 cas 设置 unwritable 的值
private static final AtomicIntegerFieldUpdater<ChannelOutboundBuffer> UNWRITABLE_UPDATER =
        AtomicIntegerFieldUpdater.newUpdater(ChannelOutboundBuffer.class, "unwritable");

// 0 可写，1 不可写
private volatile int unwritable;


private void incrementPendingOutboundBytes(long size, boolean invokeLater) {
    if (size == 0) {
        return;
    }

    long newWriteBufferSize = TOTAL_PENDING_SIZE_UPDATER.addAndGet(this, size);
    if (newWriteBufferSize > channel.config().getWriteBufferHighWaterMark()) {
        setUnwritable(invokeLater);
    }
}

private void setUnwritable(boolean invokeLater) {
    for (;;) {
        final int oldValue = unwritable;
        final int newValue = oldValue | 1;
        if (UNWRITABLE_UPDATER.compareAndSet(this, oldValue, newValue)) {
            if (oldValue == 0 && newValue != 0) {
                fireChannelWritabilityChanged(invokeLater);
            }
            break;
        }
    }
}

Once set to not write, only when the water level falls below the low water mark, symbol will change back to write

private void decrementPendingOutboundBytes(long size, boolean invokeLater, boolean notifyWritability) {
    if (size == 0) {
        return;
    }

    long newWriteBufferSize = TOTAL_PENDING_SIZE_UPDATER.addAndGet(this, -size);
    if (notifyWritability && newWriteBufferSize < channel.config().getWriteBufferLowWaterMark()) {
        setWritable(invokeLater);
    }
}