在HEVC中只使用了左边1列和上边1行像素作为参考像素，而在VVC中允许使用多个参考行 Multiple reference line (MRL) 。
上图是HEVC中参考像素，用p(x,y)表示，x表示横坐标，y表示纵坐标。左边参考像素x=-1,y=0,-1,-2...-2N+1。上边参考像素x=0,1,2...2N-1,y=-1。
上图MRL是一个有4个参考行的例子，其中segments A和segments F不是由重建像素得来，而是分别由segments B和segments E内最近的像素填充。
在HEVC中只使用reference line 0，而VVC中可以使用reference line 0，1，3。这是因为在索引号为0~3的参考行中，索引号为0和1的参考像素行与当前预测单元中的像素距离相对较近，有较高的相关性，因此需要保留。索引为3的参考像素行与当前预测单元的距离相对较远，可以提供较为不同的参考像素，而索引为2的参考像素行，很难提供额外的参考像素信息。
参考行索引号mrl_idx需要放入码流传到解码端。对于mrl_idx>0的情况，只有MPM中的模式能够使用并且只需要传输MPM的索引号。在传输时mrl_idx要在帧内编码模式之前。
当处理CTU的第一行块时不允许使用MRL。且当使用MRL时不允许使用PDPC。
下面是VTM5中参考像素填充的相关代码：
//!<帧内预测参考像素
void IntraPrediction::xFillReferenceSamples( const CPelBuf &recoBuf, Pel* refBufUnfiltered, const CompArea &area, const CodingUnit &cu )
{
const ChannelType chType = toChannelType( area.compID );
const CodingStructure &cs = *cu.cs;
const SPS &sps = *cs.sps;
const PreCalcValues &pcv = *cs.pcv;
const int multiRefIdx = (area.compID == COMPONENT_Y) ? cu.firstPU->multiRefIdx : 0;
const int tuWidth = area.width;
const int tuHeight = area.height;
const int predSize = m_topRefLength;
const int predHSize = m_leftRefLength;
const int cuWidth = cu.blocks[area.compID].width;
const int cuHeight = cu.blocks[area.compID].height;
const int whRatio = cu.ispMode && isLuma(area.compID) ? std::max(1, cuWidth / cuHeight) : std::max(1, tuWidth / tuHeight);
const int hwRatio = cu.ispMode && isLuma(area.compID) ? std::max(1, cuHeight / cuWidth) : std::max(1, tuHeight / tuWidth);
const int predStride = predSize + 1 + (whRatio + 1) * multiRefIdx;
const bool noShift = pcv.noChroma2x2 && area.width == 4; // don't shift on the lowest level (chroma not-split)
const int unitWidth = tuWidth <= 2 && cu.ispMode && isLuma(area.compID) ? tuWidth : pcv.minCUWidth >> (noShift ? 0 : getComponentScaleX(area.compID, sps.getChromaFormatIdc()));
const int unitHeight = tuHeight <= 2 && cu.ispMode && isLuma(area.compID) ? tuHeight : pcv.minCUHeight >> (noShift ? 0 : getComponentScaleY(area.compID, sps.getChromaFormatIdc()));
const int totalAboveUnits = (predSize + (unitWidth - 1)) / unitWidth;
const int totalLeftUnits = (predHSize + (unitHeight - 1)) / unitHeight;
const int totalUnits = totalAboveUnits + totalLeftUnits + 1; //+1 for top-left
const int numAboveUnits = std::max<int>( tuWidth / unitWidth, 1 );
const int numLeftUnits = std::max<int>( tuHeight / unitHeight, 1 );
const int numAboveRightUnits = totalAboveUnits - numAboveUnits;
const int numLeftBelowUnits = totalLeftUnits - numLeftUnits;
CHECK( numAboveUnits <= 0 || numLeftUnits <= 0 || numAboveRightUnits <= 0 || numLeftBelowUnits <= 0, "Size not supported" );
// ----- Step 1: analyze neighborhood -----
const Position posLT = area;
const Position posRT = area.topRight();
const Position posLB = area.bottomLeft();
bool neighborFlags[4 * MAX_NUM_PART_IDXS_IN_CTU_WIDTH + 1];
int numIntraNeighbor = 0;
memset( neighborFlags, 0, totalUnits );
neighborFlags[totalLeftUnits] = isAboveLeftAvailable( cu, chType, posLT );
numIntraNeighbor += neighborFlags[totalLeftUnits] ? 1 : 0;
numIntraNeighbor += isAboveAvailable ( cu, chType, posLT, numAboveUnits, unitWidth, (neighborFlags + totalLeftUnits + 1) );
numIntraNeighbor += isAboveRightAvailable( cu, chType, posRT, numAboveRightUnits, unitWidth, (neighborFlags + totalLeftUnits + 1 + numAboveUnits) );
numIntraNeighbor += isLeftAvailable ( cu, chType, posLT, numLeftUnits, unitHeight, (neighborFlags + totalLeftUnits - 1) );
numIntraNeighbor += isBelowLeftAvailable ( cu, chType, posLB, numLeftBelowUnits, unitHeight, (neighborFlags + totalLeftUnits - 1 - numLeftUnits) );
// ----- Step 2: fill reference samples (depending on neighborhood) -----
CHECK((predHSize + 1) * predStride > m_iYuvExtSize, "Reference sample area not supported");
const Pel* srcBuf = recoBuf.buf;
const int srcStride = recoBuf.stride;
Pel* ptrDst = refBufUnfiltered;
const Pel* ptrSrc;
const Pel valueDC = 1 << (sps.getBitDepth( chType ) - 1);
if( numIntraNeighbor == 0 )
{ //!<若所有参考像素都不可用则用固定值填充
// Fill border with DC value
for (int j = 0; j <= predSize + multiRefIdx; j++) { ptrDst[j] = valueDC; }
for (int i = 1; i <= predHSize + multiRefIdx; i++) { ptrDst[i*predStride] = valueDC; }
}
else if( numIntraNeighbor == totalUnits )
{ //!<srcBuf指向当前块左上角，需要将指针移到上面行或左边列
// Fill top-left border and top and top right with rec. samples
ptrSrc = srcBuf - (1 + multiRefIdx) * srcStride - (1 + multiRefIdx);
for (int j = 0; j <= predSize + multiRefIdx; j++) { ptrDst[j] = ptrSrc[j]; }
ptrSrc = srcBuf - multiRefIdx * srcStride - (1 + multiRefIdx);
for (int i = 1; i <= predHSize + multiRefIdx; i++) { ptrDst[i*predStride] = *(ptrSrc); ptrSrc += srcStride; }
}
else // reference samples are partially available
{
// Fill top-left sample(s) if available
ptrSrc = srcBuf - (1 + multiRefIdx) * srcStride - (1 + multiRefIdx);
ptrDst = refBufUnfiltered;
if (neighborFlags[totalLeftUnits])
{
ptrDst[0] = ptrSrc[0]; //!<填充左上角蓝色块
for (int i = 1; i <= multiRefIdx; i++)
{
ptrDst[i] = ptrSrc[i]; //!<填充Segment D
ptrDst[i*predStride] = ptrSrc[i*srcStride]; //!<填充Segment C
}
}
//!<填充Segment B
// Fill left & below-left samples if available (downwards)
ptrSrc += (1 + multiRefIdx) * srcStride;
ptrDst += (1 + multiRefIdx) * predStride;
for (int unitIdx = totalLeftUnits - 1; unitIdx > 0; unitIdx--)
{
if (neighborFlags[unitIdx])
{
for (int i = 0; i < unitHeight; i++)
{
ptrDst[i*predStride] = ptrSrc[i*srcStride];
}
}
ptrSrc += unitHeight * srcStride;
ptrDst += unitHeight * predStride;
}
// Fill last below-left sample(s)
if (neighborFlags[0])
{
int lastSample = (predHSize % unitHeight == 0) ? unitHeight : predHSize % unitHeight;
for (int i = 0; i < lastSample; i++)
{
ptrDst[i*predStride] = ptrSrc[i*srcStride];
}
}
//!<填充Segment E
// Fill above & above-right samples if available (left-to-right)
ptrSrc = srcBuf - srcStride * (1 + multiRefIdx);
ptrDst = refBufUnfiltered + 1 + multiRefIdx;
for (int unitIdx = totalLeftUnits + 1; unitIdx < totalUnits - 1; unitIdx++)
{
if (neighborFlags[unitIdx])
{
for (int j = 0; j < unitWidth; j++)
{
ptrDst[j] = ptrSrc[j];
}
}
ptrSrc += unitWidth;
ptrDst += unitWidth;
}
// Fill last above-right sample(s)
if (neighborFlags[totalUnits - 1])
{
int lastSample = (predSize % unitWidth == 0) ? unitWidth : predSize % unitWidth;
for (int j = 0; j < lastSample; j++)
{
ptrDst[j] = ptrSrc[j];
}
}
// pad from first available down to the last below-left
ptrDst = refBufUnfiltered;
int lastAvailUnit = 0;
if (!neighborFlags[0])
{
int firstAvailUnit = 1;
while (firstAvailUnit < totalUnits && !neighborFlags[firstAvailUnit])
{
firstAvailUnit++;
}
// first available sample
int firstAvailRow = 0;
int firstAvailCol = 0;
if (firstAvailUnit < totalLeftUnits)
{
firstAvailRow = (totalLeftUnits - firstAvailUnit) * unitHeight + multiRefIdx;
}
else if (firstAvailUnit == totalLeftUnits)
{
firstAvailRow = multiRefIdx;
}
else
{
firstAvailCol = (firstAvailUnit - totalLeftUnits - 1) * unitWidth + 1 + multiRefIdx;
}
const Pel firstAvailSample = ptrDst[firstAvailCol + firstAvailRow * predStride];
// last sample below-left (n.a.)
int lastRow = predHSize + multiRefIdx;
//!<将第一个有效像素下的像素都用该值填充
// fill left column
for (int i = lastRow; i > firstAvailRow; i--)
{
ptrDst[i*predStride] = firstAvailSample;
}//!<如果第一个有效像素的列大于0，其左侧的像素都用该值填充
// fill top row
if (firstAvailCol > 0)
{
for (int j = 0; j < firstAvailCol; j++)
{
ptrDst[j] = firstAvailSample;
}
}
lastAvailUnit = firstAvailUnit;
}
// pad all other reference samples.
int currUnit = lastAvailUnit + 1;
while (currUnit < totalUnits)
{
if (!neighborFlags[currUnit]) // samples not available
{
// last available sample
int lastAvailRow = 0;
int lastAvailCol = 0;
if (lastAvailUnit < totalLeftUnits)
{
lastAvailRow = (totalLeftUnits - lastAvailUnit - 1) * unitHeight + multiRefIdx + 1;
}
else if (lastAvailUnit == totalLeftUnits)
{
lastAvailCol = multiRefIdx;
}
else
{
lastAvailCol = (lastAvailUnit - totalLeftUnits) * unitWidth + multiRefIdx;
}
const Pel lastAvailSample = ptrDst[lastAvailCol + lastAvailRow * predStride];
//!<用最近的像素填充
// fill current unit with last available sample
if (currUnit < totalLeftUnits)
{
for (int i = lastAvailRow - 1; i >= lastAvailRow - unitHeight; i--)
{
ptrDst[i*predStride] = lastAvailSample;
}
}
else if (currUnit == totalLeftUnits)
{
for (int i = 1; i < multiRefIdx + 1; i++)
{
ptrDst[i*predStride] = lastAvailSample;
}
for (int j = 0; j < multiRefIdx + 1; j++)
{
ptrDst[j] = lastAvailSample;
}
}
else
{
int numSamplesInUnit = (currUnit == totalUnits - 1) ? ((predSize % unitWidth == 0) ? unitWidth : predSize % unitWidth) : unitWidth;
for (int j = lastAvailCol + 1; j <= lastAvailCol + numSamplesInUnit; j++)
{
ptrDst[j] = lastAvailSample;
}
}
}
lastAvailUnit = currUnit;
currUnit++;
}
}
// padding of extended samples above right with the last sample
int lastSample = multiRefIdx + predSize;
for (int j = 1; j <= whRatio * multiRefIdx; j++) { ptrDst[lastSample + j] = ptrDst[lastSample]; } //!<填充Segment F
// padding of extended samples below left with the last sample
lastSample = multiRefIdx + predHSize;
for (int i = 1; i <= hwRatio * multiRefIdx; i++) { ptrDst[(lastSample + i)*predStride] = ptrDst[lastSample*predStride]; } //!<填充Segment A
}
参考
[1]https://cloud.tencent.com/developer/article/1442620
[2]JVET-N1002
感兴趣的请关注微信公众号Video Coding
