H.266 / VVC transformation code Learning: xT function

Disclaimer: This article is a blogger original article, follow the CC 4.0 BY-SA copyright agreement, reproduced, please attach the original source link and this statement.

This link: https://blog.csdn.net/BigDream123/article/details/102748739

Transformation code H.266 / VVC in, xT primary transform function is a function, primarily to horizontal and vertical transform kernels were selected and the horizontal and vertical transform.

The basic process is as follows:

1. By calling getTrTypes () Gets the vertical and horizontal transform nuclear type (the default is a function of the DCT-2 );
2. The vertical and horizontal transform nuclear type and the width and height determining SkipWidth and SkipHeight (mainly used for a high-frequency zero: For use DCT2 large size (64) for zero-frequency transform block, leaving only the low-frequency coefficients (32 ) ; using MTS large size (32) transform block zero frequency, i.e., retaining only low frequency coefficients (16) )
3. By whether LFNST quadratic transformation adjustment SkipWidth and SkipHeight (for shape 4xN , Nx4 (N>. 8) block, LFNST applied only to the upper left corner of the 4x4 region; the shape is NxN (N> 8) block, LFNST only applied to the upper left corner of the 8x8 block)? ? ?
4. The vertical or horizontal type and size of the transform kernel selection function respectively corresponding to the transform (DCT-2 / DST7 / DCT8 ) and were transformed (vertical and horizontal transform are separate, i.e., two-dimensional transform is called twice fastFwdTrans )

code show as below:

void TrQuant::xT( const TransformUnit &tu, const ComponentID &compID, const CPelBuf &resi, CoeffBuf &dstCoeff, const int width, const int height )
{
  const unsigned maxLog2TrDynamicRange  = tu.cs->sps->getMaxLog2TrDynamicRange( toChannelType( compID ) );//15
  const unsigned bitDepth               = tu.cs->sps->getBitDepth(              toChannelType( compID ) );
  const int      TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift[TRANSFORM_FORWARD];
  const uint32_t transformWidthIndex    = g_aucLog2[width ] - 1;  // nLog2WidthMinus1, since transform start from 2-point
  const uint32_t transformHeightIndex   = g_aucLog2[height] - 1;  // nLog2HeightMinus1, since transform start from 2-point


  int trTypeHor = DCT2;
  int trTypeVer = DCT2;
  //获取水平和垂直的变换类型（根据是否是隐性MTS和ISP模式或者SBT或者显性MTS来获取）
  getTrTypes ( tu, compID, trTypeHor, trTypeVer ); 

#if !JVET_O0094_LFNST_ZERO_PRIM_COEFFS
  const int      skipWidth  = ( trTypeHor != DCT2 && width  == 32 ) ? 16 : width  > JVET_C0024_ZERO_OUT_TH ? width  - JVET_C0024_ZERO_OUT_TH : 0;
  const int      skipHeight = ( trTypeVer != DCT2 && height == 32 ) ? 16 : height > JVET_C0024_ZERO_OUT_TH ? height - JVET_C0024_ZERO_OUT_TH : 0;
#else
  //对于使用DCT2的大尺寸(64x64)变换块进行高频调零
  //对于使用MTS的大尺寸(32x32)变换块进行高频调零，即只保留16x16低频系数
  int  skipWidth  = ( trTypeHor != DCT2 && width  == 32 ) ? 16 : width  > JVET_C0024_ZERO_OUT_TH ? width  - JVET_C0024_ZERO_OUT_TH : 0;
  int  skipHeight = ( trTypeVer != DCT2 && height == 32 ) ? 16 : height > JVET_C0024_ZERO_OUT_TH ? height - JVET_C0024_ZERO_OUT_TH : 0;
  
  if( tu.cs->sps->getUseLFNST() && tu.cu->lfnstIdx )
  {
    //如果使用LFNST则只对部分块进行变换
    //对于形状为4xN、Nx4和N > 8的块，建议的限制意味着LFNST现在只应用一次，而只应用于左上角的4x4区域，即将第一4x4子组外的所有主系数都归零
    if( (width == 4 && height > 4) || (width > 4 && height == 4) )
    {
      skipWidth  = width  - 4;
      skipHeight = height - 4;
    }
    else if( (width >= 8 && height >= 8) )
    {
      skipWidth  = width  - 8;
      skipHeight = height - 8;
    }
  }
#endif

#if RExt__DECODER_DEBUG_TOOL_STATISTICS
  if ( trTypeHor != DCT2 )
  {
    CodingStatistics::IncrementStatisticTool( CodingStatisticsClassType{ STATS__TOOL_EMT, uint32_t( width ), uint32_t( height ), compID } );
  }
#endif
  //__declspec(align(nBytes))
  ALIGN_DATA( MEMORY_ALIGN_DEF_SIZE, TCoeff block[MAX_TB_SIZEY * MAX_TB_SIZEY] );

  const Pel *resiBuf    = resi.buf; //像素矩阵
  const int  resiStride = resi.stride; //每行的使用的像素数目

  for( int y = 0; y < height; y++ )
  {
    for( int x = 0; x < width; x++ )
    {
      block[( y * width ) + x] = resiBuf[( y * resiStride ) + x];//将像素矩阵赋值给block矩阵
    }
  }

  if( width > 1 && height > 1 ) // 2-D transform
  {
    //TRANSFORM_MATRIX_SHIFT=6, COM16_C806_TRANS_PREC=0
    const int      shift_1st              = ((g_aucLog2[width ]) + bitDepth + TRANSFORM_MATRIX_SHIFT) - maxLog2TrDynamicRange + COM16_C806_TRANS_PREC;
    const int      shift_2nd              =  (g_aucLog2[height])            + TRANSFORM_MATRIX_SHIFT                          + COM16_C806_TRANS_PREC;
    CHECK( shift_1st < 0, "Negative shift" );
    CHECK( shift_2nd < 0, "Negative shift" );
  TCoeff *tmp = ( TCoeff * ) alloca( width * height * sizeof( TCoeff ) );   
  //fastFwdTrans函数是一个DCT2、DST7、DCT8的3x6的函数数组，第一行代表DCT2(2/4/8/16/32/64)第二行代表DCT8(null/4/8/16/32/null)第三行代表DST7(null/4/8/16/32/null)
  fastFwdTrans[trTypeHor][transformWidthIndex ](block,        tmp, shift_1st, height,        0, skipWidth);//horizontal transform
  fastFwdTrans[trTypeVer][transformHeightIndex](tmp, dstCoeff.buf, shift_2nd, width, skipWidth, skipHeight);//vertical transform
  }
  else if( height == 1 ) //1-D horizontal transform
  {
    const int      shift              = ((g_aucLog2[width ]) + bitDepth + TRANSFORM_MATRIX_SHIFT) - maxLog2TrDynamicRange + COM16_C806_TRANS_PREC;
    CHECK( shift < 0, "Negative shift" );
    CHECKD( ( transformWidthIndex < 0 ), "There is a problem with the width." );
    fastFwdTrans[trTypeHor][transformWidthIndex]( block, dstCoeff.buf, shift, 1, 0, skipWidth );
  }
  else //if (iWidth == 1) //1-D vertical transform
  {
    int shift = ( ( g_aucLog2[height] ) + bitDepth + TRANSFORM_MATRIX_SHIFT ) - maxLog2TrDynamicRange + COM16_C806_TRANS_PREC;
    CHECK( shift < 0, "Negative shift" );
    CHECKD( ( transformHeightIndex < 0 ), "There is a problem with the height." );
    fastFwdTrans[trTypeVer][transformHeightIndex]( block, dstCoeff.buf, shift, 1, 0, skipHeight );
  }
}

Wherein, fastFwdTrans function is a two-dimensional array, which is defined as follows:

FwdTrans *fastFwdTrans[NUM_TRANS_TYPE][g_numTransformMatrixSizes] =
{
  { fastForwardDCT2_B2, fastForwardDCT2_B4, fastForwardDCT2_B8, fastForwardDCT2_B16, fastForwardDCT2_B32, fastForwardDCT2_B64 },
  { nullptr,            fastForwardDCT8_B4, fastForwardDCT8_B8, fastForwardDCT8_B16, fastForwardDCT8_B32, nullptr },
  { nullptr,            fastForwardDST7_B4, fastForwardDST7_B8, fastForwardDST7_B16, fastForwardDST7_B32, nullptr },
};

The first index is a transformation type (DCT-2, DCT-8, DST-7), the second index is the corresponding transform block size.

Function getTrTypes () is used to obtain horizontal and vertical transform kernel type, whether it is mainly based on the MTS dominant, recessive MTS, ISP mode, MIP mode, the SBT selected transform kernel, wherein MTS, SBT transform kernel principle Reference : https://blog.csdn.net/BigDream123/article/details/102599000

code show as below:

void TrQuant::getTrTypes(const TransformUnit tu, const ComponentID compID, int &trTypeHor, int &trTypeVer)
{
  const bool isExplicitMTS = (CU::isIntra(*tu.cu) ? tu.cs->sps->getUseIntraMTS() : tu.cs->sps->getUseInterMTS() && CU::isInter(*tu.cu)) && isLuma(compID);
#if JVET_O0529_IMPLICIT_MTS_HARMONIZE
  const bool isImplicitMTS = CU::isIntra(*tu.cu) && tu.cs->sps->getUseImplicitMTS() && isLuma(compID) && tu.cu->lfnstIdx == 0 && tu.cu->mipFlag == 0;
#else
  const bool isImplicitMTS = CU::isIntra(*tu.cu) && tu.cs->sps->getUseImplicitMTS() && isLuma(compID);
#endif
  const bool isISP = CU::isIntra(*tu.cu) && tu.cu->ispMode && isLuma(compID);
  const bool isSBT = CU::isInter(*tu.cu) && tu.cu->sbtInfo && isLuma(compID);//对于SBT仅亮度块的转换核与位置有关，色度块都用DCT-2

  trTypeHor = DCT2;
  trTypeVer = DCT2;

#if JVET_O0538_SPS_CONTROL_ISP_SBT
  if (!tu.cs->sps->getUseMTS())
    return;
#endif

  if (isImplicitMTS || isISP)
  {
    int  width = tu.blocks[compID].width;
    int  height = tu.blocks[compID].height;
    bool widthDstOk = width >= 4 && width <= 16;
    bool heightDstOk = height >= 4 && height <= 16;

    if (widthDstOk)
      trTypeHor = DST7;
    if (heightDstOk)
      trTypeVer = DST7;
    return;
  }


  if (isSBT)
  {
    uint8_t sbtIdx = tu.cu->getSbtIdx();
    uint8_t sbtPos = tu.cu->getSbtPos();

    if( sbtIdx == SBT_VER_HALF || sbtIdx == SBT_VER_QUAD )
    {
      assert( tu.lwidth() <= MTS_INTER_MAX_CU_SIZE ); //垂直分割时如果宽度大于32则出错
      if( tu.lheight() > MTS_INTER_MAX_CU_SIZE )
      {
        trTypeHor = trTypeVer = DCT2;//若残差的一侧大于32则两个维度变换均设为DCT-2
      }
      else
      {
        if( sbtPos == SBT_POS0 )  { trTypeHor = DCT8;  trTypeVer = DST7; }
        else                      { trTypeHor = DST7;  trTypeVer = DST7; }
      }
    }
    else
    {
      assert( tu.lheight() <= MTS_INTER_MAX_CU_SIZE ); //水平分割时如果高度大于32则出错
      if( tu.lwidth() > MTS_INTER_MAX_CU_SIZE )
      {
        trTypeHor = trTypeVer = DCT2;////若残差的一侧大于32则两个维度变换均设为DCT-2
      }
      else
      {
        if( sbtPos == SBT_POS0 )  { trTypeHor = DST7;  trTypeVer = DCT8; }
        else                      { trTypeHor = DST7;  trTypeVer = DST7; }
      }
    }
    return;
  }

  if (isExplicitMTS)
  {
    if (tu.mtsIdx > MTS_SKIP)
    {
      int indHor = (tu.mtsIdx - MTS_DST7_DST7) & 1;
      int indVer = (tu.mtsIdx - MTS_DST7_DST7) >> 1;

      trTypeHor = indHor ? DCT8 : DST7;
      trTypeVer = indVer ? DCT8 : DST7;
    }
  }
}