In this section we see next allocation process
. 1 PooledByteBuf <T> the allocate (PoolThreadCache Cache, int reqCapacity, int maxCapacity) { 2 PooledByteBuf <T> buf = newByteBuf (maxCapacity); // initialize a capacity of 2 ^ 31 - ByteBuf 1 in . 3 the allocate (Cache, buf, reqCapacity ); // reqCapacity = 1024 into the distribution logic . 4 return buf; . 5 }
. 1 Private void the allocate (PoolThreadCache Cache, PooledByteBuf <T> buf, Final int reqCapacity) { 2 Final int normCapacity = normalizeCapacity (reqCapacity); // size of the request to be calibrated for a particular look . 3 IF (isTinyOrSmall (normCapacity)) { // Capacity <determines the pageSize size after calibration request is smaller than 8K . 4 int tableIdx; . 5 PoolSubpage <T> [] Table; . 6 Boolean Tiny = isTiny (normCapacity); // is smaller than the requested size is determined 512B . 7 iF (Tiny) { // <512 . 8 IF(cache.allocateTiny ( the this , buf, reqCapacity, normCapacity)) {// allocated from the cache, if it is the first time there is certainly no corresponding size of the cache, unless previously been released, and then the memory space is cached . . 9 // WAS OUT of the allocate Able to Move The Cache ON SO 10 return ; . 11 } 12 is tableIdx = tinyIdx (normCapacity); obtaining a corresponding subscripts // tiny array 13 is Table = tinySubpagePools; // tiny Subpage array 14 } the else 512 // {<= normCapacity <8K 15 IF (cache.allocateSmall ( the this , buf, reqCapacity, normCapacity)) {// the same dispensing from the cache, this last one introduced in detail 16 // was able to allocate out of the cache so move on 17 return; 18 } 19 tableIdx = smallIdx(normCapacity); // 获取small subPage数组下标 20 table = smallSubpagePools; // subPage 数组 21 } 22 23 final PoolSubpage<T> head = table[tableIdx];// 取得对应下标的subPage 24 25 /** 26 * Synchronize on the head. This is needed as {@link PoolChunk#allocateSubpage(int)} and 27 * {@link# Free PoolChunk (Long)} The On May Modify doubly linked List AS Well. 28 * / 29 the synchronized (head) {// lock prevention modifying other operating head junction 30 Final PoolSubpage <T> S = head.next; 31 is IF ( ! S = head) {// If the doubly linked list has been initialized, then the subpage dispensing from 32 assert s.doNotDestroy && s.elemSize == normCapacity; // make sure elemSize assert this current magnitude must subPage and after proofreading as the size of the request 33 is Long handle = s.allocate (); // dispensing from the subPage 34 is Assert handle> = 0 ; 35 s.chunk.initBufWithSubpage (buf, handle, reqCapacity); 36 incTinySmallAllocation (Tiny); 37 [ return ; 38 is } 39 } 40 the synchronized ( the this ) { 41 is allocateNormal (buf, reqCapacity, normCapacity); // initialize bidirectional not circular linked list, using the normal distribution 42 is } 43 is 44 is incTinySmallAllocation (Tiny); 45 return ; 46 is } 47 IF (normCapacity <= chunkSize) { 48 IF (cache.allocateNormal ( the this , buf, reqCapacity, normCapacity)) { 49 // was able to allocate out of the cache so move on 50 return; 51 } 52 synchronized (this) { 53 allocateNormal(buf, reqCapacity, normCapacity); 54 ++allocationsNormal; 55 } 56 } else { 57 // Huge allocations are never served via the cache so just call allocateHuge 58 allocateHuge(buf, reqCapacity); 59 } 60 }
int normalizeCapacity ( int reqCapacity) { IF (reqCapacity <0 ) { the throw new new an IllegalArgumentException ( "Capacity:" + reqCapacity + "(expected: 0+)" ); } IF (reqCapacity> = chunkSize) { return directMemoryCacheAlignment == 0? reqCapacity:; alignCapacity (reqCapacity) } IF (! isTiny (reqCapacity)) { // if it is> = 512 memory application, then normalized to the power of 2 size, such as 1000 becomes 1024, can put themselves Dan operate their own main method to try. // Doubled int normalizedCapacity = reqCapacity; normalizedCapacity--; normalizedCapacity |= normalizedCapacity >>> 1; normalizedCapacity |= normalizedCapacity >>> 2; normalizedCapacity |= normalizedCapacity >>> 4; normalizedCapacity |= normalizedCapacity >>> 8; normalizedCapacity |= normalizedCapacity >>> 16; normalizedCapacity ++; if (normalizedCapacity < 0) { normalizedCapacity >>>= 1; } assert== 0 || directMemoryCacheAlignment (normalizedCapacity & directMemoryCacheAlignmentMask) == 0 ; return normalizedCapacity; } IF (directMemoryCacheAlignment> 0 ) { return alignCapacity (reqCapacity); } // the Quantum-Spaced in IF ((reqCapacity & 15) == 0 ) { // If <512 determines if it is a multiple of 16, then returned directly return reqCapacity; } return (reqCapacity & ~ 15) + 16 ; // if it is not aligned in multiples of 16 }
Private void allocateNormal (PooledByteBuf <T> buf, int reqCapacity, int normCapacity) { IF (q050.allocate (buf, reqCapacity, normCapacity) || q025.allocate (buf, reqCapacity, normCapacity) || q000.allocate (buf, reqCapacity , normCapacity) || qInit.allocate (buf, reqCapacity, normCapacity) || q075.allocate (buf, reqCapacity, normCapacity)) { return ; } // first try chunk allocated from the list, there is a place of a note // allocation sequence to allocation q50 is 50% to 100%, then the q25 is 25% to 75%, then q000 0% ~ 50%, qInit 0% ~ 25%, q75 75% ~ 100% // the Add A new new chunk. PoolChunk <T> C = newChunk (the pageSize, maxOrder, pageShifts, chunkSize); // Create a the Chunk Long handle = c.allocate (normCapacity); // Memory allocation Assert handle> 0 ; c.initBuf (buf, handle, reqCapacity ); qInit.add (C); // add to the list initialization chunk }
The above allocation sequence, think about why is not allocated from q000 it? I'm looking for a good period of analysis.
When analyzing PoolChunkList, we know that a chunk of memory with the release of non-stop, which itself will keep moving towards it in the chunk list of prev list, until it is fully released and then recovered. If there is an attempt to allocate from q000 start, although the distribution of speed may be faster (because of the greater chance of successful distribution), but a chunk when there is a greater chance of redistribution within a utilization rate of 25%, which is a chunk be the chance of recovery is greatly reduced. This poses a problem, our application will exist chunk peak of a visit, this time to take up the amount of memory would be usually several times, and therefore will be more distribution out several times in the actual operation, and so on peak period after the past, due to the reduced probability chunk recycled, memory recovery progress will be very slow (because not been fully released, it can not be recycled), there is a big waste of memory.
Why try to allocate it from the beginning q050, q050 memory footprint is 50% to 100% of the chunk, speculation is hoping to improve memory usage of the entire application, because in most cases this will be used q050 memory, so memory usage is not in many cases the use of some low (<50%) chunk will slowly go out, eventually recovered. But why not start from qinit in it, chunk where utilization is low, but will not be recovered, not a waste? q075, q100 due to the high usage rate, the distribution probability of success will be smaller, and therefore put the final.
Also mentioned above, a new chunk, I would like to explain two special byte array, the array size is 4096
memoryMap = new byte[maxSubpageAllocs << 1]; depthMap = new byte[memoryMap.length]; int memoryMapIndex = 1; for (int d = 0; d <= maxOrder; ++ d) { // move down the tree one level at a time int depth = 1 << d; for (int p = 0; p < depth; ++ p) { // in each level traverse left to right and set value to the depth of subtree memoryMap[memoryMapIndex] = (byte) d; depthMap[memoryMapIndex] = (byte) d; memoryMapIndex ++; } }
After the array is how the cycle look like?
MemoryMap storage allocation information, height information depthMap storage node; initial state, and MemoryMap depthMap equal, does not change the value of the initialization depthMap, MemoryMap value is assigned as the nodes change.
The subscript number represents a balanced binary tree node, represents the height value node.
For example MemoryMap [1024] = 10, MemoryMap [2048] = 11. Well, this finished, we continue to look at:
1 long allocate(int normCapacity) { 2 if ((normCapacity & subpageOverflowMask) != 0) { // >= pageSize >= 8K 3 return allocateRun(normCapacity); 4 } else { 5 return allocateSubpage(normCapacity); // < 8K 6 } 7 }
1 private long allocateSubpage(int normCapacity) { 2 // Obtain the head of the PoolSubPage pool that is owned by the PoolArena and synchronize on it. 3 // This is need as we may add it back and so alter the linked-list structure. 4 PoolSubpage<T> head = arena.findSubpagePoolHead(normCapacity); // 根据请求内存大小，获取下标，然后获取对应的subPage 5 synchronized (head) { 6 int d = maxOrder; // 最大层数 11 7 int id = allocateNode(d); // 分配节点，修改涉及节点层数, 具体看下面的分析 8 if (id < 0) { // 无可分配节点 9 return id; 10 } 11 12 final PoolSubpage<T>[] subpages = this.subpages; 13 final int pageSize = this.pageSize; 14 15 freeBytes -= pageSize;// 16M（初始化值，后面会逐渐减少） - 8K 16 17 int subpageIdx = subpageIdx(id); // 2048 对应的偏移量是 0 ， 2049 对应的偏移量是 1 。。。4095 对应偏移量是 2047 18 PoolSubpage<T> subpage = subpages[subpageIdx]; // 取对应下标的 subpage 19 if (subpage == null) { 20 subpage = new PoolSubpage<T>(head, this, id, runOffset(id), pageSize, normCapacity); // 创建PoolSubpage 21 subpages[subpageIdx] = subpage;
// 新建或初始化subpage并加入到chunk的subpages数组，同时将subpage加入到arena的subpage双向链表中，最后完成分配请求的内存。
// 代码中，subpage != null的情况产生的原因是：subpage初始化后分配了内存，但一段时间后该subpage分配的内存释放并从arena的双向链表中删除，
// 此时subpage不为null，当再次请求分配时，只需要调用init()将其加入到areana的双向链表中即可。
22 } else { 23 subpage.init(head, normCapacity); 24 } 25 return subpage.allocate(); // 最后结果是一个long整数，其中低32位表示二叉树中的分配的节点，高32位表示subPage中分配的具体位置 26 } 27 }
修改节点对应的层数，底下这个方法涉及了很多的位运算，这里大家要仔细琢磨。
1 private int allocateNode(int d) { // d = 11 2 int id = 1; 3 int initial = - (1 << d); // -2048 4 byte val = value(id); // memoryMap[id] , 一般1对应的层数是 0 ,当第一个节点被分配完成后，这个节点的值会变成12 5 if (val > d) { // unusable // 如果分配完成，那么无法进行分配，那么就是 12 > 11 ，直接返回 -1 6 return -1; 7 } 8 while (val < d || (id & initial) == 0) { // id & initial == 1 << d for all ids at depth d, for < d it is 0 9 id <<= 1; // 左移1位 10 val = value(id); // 取到节点对应的层数 11 if (val > d) { // 如果当前的值大于层数，也就是说左子节点用完了 12 id ^= 1; // 当前节点值 +1 ，取右节点 13 val = value(id); // 取右子节点 14 } 15 } 16 byte value = value(id); 17 assert value == d && (id & initial) == 1 << d : String.format("val = %d, id & initial = %d, d = %d", 18 value, id & initial, d); 19 setValue(id, unusable); // mark as unusable //将该节点的值设置为12，不可用 20 updateParentsAlloc(id); // 将该节点对应的所有父节点，层数全部 + 1 21 return id; 22 }
1 private void updateParentsAlloc(int id) { 2 while (id > 1) { 3 int parentId = id >>> 1; // 取父级节点 4 byte val1 = value(id); // 5 byte val2 = value(id ^ 1); // 如果 id = 2048 那么 id ^ 1 = 2049 ， 如果 id = 2049 这里 id ^ 1 = 2048 这里一定要注意！！！ 6 byte val = val1 < val2 ? val1 : val2; 7 setValue(parentId, val); // 修改父级对应层数 8 id = parentId; 9 } 10 }
上面这个方法很有意思，大家要仔细体会，我举个例子
最开始是这样的，我们开始第一次分配，那么就是2048号节点，分配完成后会变成
这时候由于2048变成了12，不可用状态，那么取右节点进行分配
下一次再进行分配的时候， 发现 1024 是 12 ，那么取右子节点， 是 1025 = 10 ，因为 10 < 11 所以再次进入循环，找到2050节点，对应层数是 11 < 12 ，则分配 2050节点分配后变成
然后依次类推。
1 private long allocateRun(int normCapacity) { 2 int d = maxOrder - (log2(normCapacity) - pageShifts); // 计算满足需求的节点的高度 比如 16K , 那么 就是 d= 11 - (14 - 13) = 10 3 int id = allocateNode(d); // 从第十层找空闲节点， 原理跟上面相同，就不说了 4 if (id < 0) { 5 return id; 6 } 7 freeBytes -= runLength(id); // 分配后剩余的字节数 8 return id; 9 }
说到这里，内存模型的重点部分算是说完了。如果整个Netty系列有任何不对的地方，欢迎大家指出，我会虚心改正。
后面呢，我将打算将Spring的源码的关键部分进行分析，期间也会穿插着对Redis的底层数据结构等重要内容进行分析。有需要的童鞋们，记得关注我，随时可以获取最新的消息。
谢谢。