Most of the players are facing RTSP live scene, in addition to H.264, also need to support H.265, H.265 RTSP currently on the market more and more cameras, support RTSP H.265 player is imminent. In addition, a simple play H.265 enough, you also need to be able to H.265 data can be recorded.
Green rhinoceros team access to a lot of developers, hoping to play at the same time, access to YUV or RGB data, facial matching algorithm analysis, so the callback optional audio and video.
EasyPlayer-RTSP implementation supports H265 coding format
EasyPlayer-RTSP-Win has full support for decoding and playing RTSP streams H265 and H265 here to support the process of doing a brief introduction;
A, libEasyRTSPClient library has pulled support RTSP stream H265 video sources and analytical
Two, H265 header parsing
H265 and H264 is similar, but its NAL type format is more diversified, in addition to the SPS, PPS, also increased the VPS, do the following simple analysis for H265 frame nal head;
First, look at the definition of H265nal header format X265 in the source code:
//H265 NAL type
//this enum have been defined in x265.h
typedef enum tagH265NalUnitType
{
NAL_UNIT_CODED_SLICE_TRAIL_N = 0, // 0
NAL_UNIT_CODED_SLICE_TRAIL_R, // 1
NAL_UNIT_CODED_SLICE_TSA_N, // 2
NAL_UNIT_CODED_SLICE_TLA, // 3 // Current name in the spec: TSA_R
NAL_UNIT_CODED_SLICE_STSA_N, // 4
NAL_UNIT_CODED_SLICE_STSA_R, // 5
NAL_UNIT_CODED_SLICE_RADL_N, // 6
NAL_UNIT_CODED_SLICE_DLP, // 7 // Current name in the spec: RADL_R
NAL_UNIT_CODED_SLICE_RASL_N, // 8
NAL_UNIT_CODED_SLICE_TFD, // 9 // Current name in the spec: RASL_R
NAL_UNIT_RESERVED_10,
NAL_UNIT_RESERVED_11,
NAL_UNIT_RESERVED_12,
NAL_UNIT_RESERVED_13,
NAL_UNIT_RESERVED_14,
NAL_UNIT_RESERVED_15,
NAL_UNIT_CODED_SLICE_BLA, // 16 // Current name in the spec: BLA_W_LP
NAL_UNIT_CODED_SLICE_BLANT, // 17 // Current name in the spec: BLA_W_DLP
NAL_UNIT_CODED_SLICE_BLA_N_LP, // 18
NAL_UNIT_CODED_SLICE_IDR, // 19 // Current name in the spec: IDR_W_DLP
NAL_UNIT_CODED_SLICE_IDR_N_LP, // 20
NAL_UNIT_CODED_SLICE_CRA, // 21
NAL_UNIT_RESERVED_22,
NAL_UNIT_RESERVED_23,
NAL_UNIT_RESERVED_24,
NAL_UNIT_RESERVED_25,
NAL_UNIT_RESERVED_26,
NAL_UNIT_RESERVED_27,
NAL_UNIT_RESERVED_28,
NAL_UNIT_RESERVED_29,
NAL_UNIT_RESERVED_30,
NAL_UNIT_RESERVED_31,
NAL_UNIT_VPS, // 32
NAL_UNIT_SPS, // 33
NAL_UNIT_PPS, // 34
NAL_UNIT_ACCESS_UNIT_DELIMITER, // 35
NAL_UNIT_EOS, // 36
NAL_UNIT_EOB, // 37
NAL_UNIT_FILLER_DATA, // 38
NAL_UNIT_SEI, // 39 Prefix SEI
NAL_UNIT_SEI_SUFFIX, // 40 Suffix SEI
NAL_UNIT_RESERVED_41,
NAL_UNIT_RESERVED_42,
NAL_UNIT_RESERVED_43,
NAL_UNIT_RESERVED_44,
NAL_UNIT_RESERVED_45,
NAL_UNIT_RESERVED_46,
NAL_UNIT_RESERVED_47,
NAL_UNIT_UNSPECIFIED_48,
NAL_UNIT_UNSPECIFIED_49,
NAL_UNIT_UNSPECIFIED_50,
NAL_UNIT_UNSPECIFIED_51,
NAL_UNIT_UNSPECIFIED_52,
NAL_UNIT_UNSPECIFIED_53,
NAL_UNIT_UNSPECIFIED_54,
NAL_UNIT_UNSPECIFIED_55,
NAL_UNIT_UNSPECIFIED_56,
NAL_UNIT_UNSPECIFIED_57,
NAL_UNIT_UNSPECIFIED_58,
NAL_UNIT_UNSPECIFIED_59,
NAL_UNIT_UNSPECIFIED_60,
NAL_UNIT_UNSPECIFIED_61,
NAL_UNIT_UNSPECIFIED_62,
NAL_UNIT_UNSPECIFIED_63,
NAL_UNIT_INVALID,
}H265NalUnitType;
#endif
Where we can see the definition of VPS, SPS and PPS:
NAL_UNIT_VPS, // 32
NAL_UNIT_SPS, // 33 is
NAL_UNIT_PPS, // 34 is
Similarly, we know that it is easy to define a P-frame NAL type is 0-9, I-frame is defined 16-21; NAL type definition as found H265 than H264 diversification, it is determined not limited to one type;
Meanwhile, the test was found, the actual frame data H265 VPS = 0x40, SPS = 0x42, PPS = 0x44, through the conversion, we can obtain: NALtype * 2 = actual flow NaLType;
Specific analysis procedure is as follows:
//输入的pbuf必须包含start code(00 00 00 01)
int GetH265VPSandSPSandPPS(char *pbuf, int bufsize, char *_vps, int *_vpslen, char *_sps, int *_spslen, char *_pps, int *_ppslen)
{
char vps[512]={0}, sps[512] = {0}, pps[128] = {0};
int vpslen=0, spslen=0, ppslen=0, i=0, iStartPos=0, ret=-1;
int iFoundVPS=0, iFoundSPS=0, iFoundPPS=0, iFoundSEI=0;
if (NULL == pbuf || bufsize<4) return -1;
for (i=0; i<bufsize; i++)
{
if ( (unsigned char)pbuf[i] == 0x00 && (unsigned char)pbuf[i+1] == 0x00 &&
(unsigned char)pbuf[i+2] == 0x00 && (unsigned char)pbuf[i+3] == 0x01 )
{
printf("0x%X\n", (unsigned char)pbuf[i+4]);
switch ((unsigned char)pbuf[i+4])
{
case 0x40: //VPS
{
iFoundVPS = 1;
iStartPos = i+4;
}
break;
case 0x42: //SPS
{
if (iFoundVPS == 0x01 && i>4)
{
vpslen = i-iStartPos;
if (vpslen>256) return -1; //vps长度超出范围
memset(vps, 0x00, sizeof(vps));
memcpy(vps, pbuf+iStartPos, vpslen);
}
iStartPos = i+4;
iFoundSPS = 1;
}
break;
case 0x44: //PPS
{
if (iFoundSPS == 0x01 && i>4)
{
spslen = i-iStartPos;
if (spslen>256) return -1;
memset(sps, 0x0, sizeof(sps));
memcpy(sps, pbuf+iStartPos, spslen);
}
iStartPos = i+4;
iFoundPPS = 1;
}
break;
case 0x4E: //Prefix SEI
case 0x50: //Suffix SEI
case 0x20: //I frame 16
case 0x22: //I frame 17
case 0x24: //I frame 18
case 0x26: //I frame 19
case 0x28: //I frame 20
case 0x2A: //I frame 21(acturally we should find naltype 16-21)
{
if (iFoundPPS == 0x01 && i>4)
{
ppslen = i-iStartPos;
if (ppslen>256) return -1;
memset(pps, 0x0, sizeof(pps));
memcpy(pps, pbuf+iStartPos, ppslen);
}
iStartPos = i+4;
iFoundSEI = 1;
}
break;
default:
break;
}
}
if (iFoundSEI == 0x01) break;
}
if (iFoundVPS == 0x01)
{
if (vpslen < 1)
{
if (bufsize < sizeof(vps))
{
vpslen = bufsize-4;
memset(vps, 0x00, sizeof(vps));
memcpy(vps, pbuf+4, vpslen);
}
}
if (vpslen > 0)
{
if (NULL != _vps) memcpy(_vps, vps, vpslen);
if (NULL != _vpslen) *_vpslen = vpslen;
}
ret = 0;
}
if (iFoundSPS == 0x01)
{
if (spslen < 1)
{
if (bufsize < sizeof(sps))
{
spslen = bufsize-4;
memset(sps, 0x00, sizeof(sps));
memcpy(sps, pbuf+4, spslen);
}
}
if (spslen > 0)
{
if (NULL != _sps) memcpy(_sps, sps, spslen);
if (NULL != _spslen) *_spslen = spslen;
}
ret = 0;
}
if (iFoundPPS == 0x01)
{
if (ppslen < 1)
{
if (bufsize < sizeof(pps))
{
ppslen = bufsize-4;
memset(pps, 0x00, sizeof(pps));
memcpy(pps, pbuf+4, ppslen); //pps
}
}
if (ppslen > 0)
{
if (NULL != _pps) memcpy(_pps, pps, ppslen);
if (NULL != _ppslen) *_ppslen = ppslen;
}
ret = 0;
}
return ret;
}
Third, the decoder needs to support H265
The decoder directly use the latest FFMPEG library is to support H265 decoding, and efficient software solutions can also, if you do not know how to use, can go and see ffplay source code, here is not described in detail here; the problem encountered here EasyPlayer call do a simple explanation:
- Older ffmpeg and the like live555 definition of H265 is to "H265" do substring string format combination, sequentially define new custom FFMPEG use of enumerated type, it may not correspond occur during use case, for example, in the definition of H265 libEasyRTSPClient library as: #define EASY_SDK_VIDEO_CODEC_H265 0x48323635 / * 1211250229 * /
while FFMPEG defined H265 (the HEVC) format 174
EasyPlayer in a unified format code is as follows:
//H265 codecID改成FFMPEG新版的
int nCodec = (_frameinfo->codec == EASY_SDK_VIDEO_CODEC_H265) ? 174 : _frameinfo->codec;
- Before EasyPlayer key frames in the processing of decoding failure will be the I-frame is a critical frame is P frame dropping all relying, of course, the video can be avoided to some extent, but the test found decoding H265, H265 of the first I-frame will often fails to decode, the commissioning discovered that in fact it is time FFNPEG decoding function returns no results are decoded to decode judgment program failed, but this time without any treatment should wait for the next time will be able to get to return correct return the results, EasyPlayer process as follows:
nRet = FFD_DecodeVideo3(pDecoderObj->ffDecoder, pbuf, frameinfo.length, pThread->yuvFrame[pThread->decodeYuvIdx].pYuvBuf, frameinfo.width, frameinfo.height, lTimestamp, lTimestamp);
if (0 != nRet)
{
if(nRet == -4)//-4表示为当前帧尚未解码完成，不作为错误判断
{
_TRACE("视频帧解码尚未完成[%d]... framesize:%d %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", nRet, frameinfo.length,
(unsigned char)pbuf[0], (unsigned char)pbuf[1], (unsigned char)pbuf[2], (unsigned char)pbuf[3], (unsigned char)pbuf[4],
(unsigned char)pbuf[5], (unsigned char)pbuf[6], (unsigned char)pbuf[7], (unsigned char)pbuf[8], (unsigned char)pbuf[9]);
}
else
{
_TRACE("视频帧解解码失败[%d]... framesize:%d %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", nRet, frameinfo.length,
(unsigned char)pbuf[0], (unsigned char)pbuf[1], (unsigned char)pbuf[2], (unsigned char)pbuf[3], (unsigned char)pbuf[4],
(unsigned char)pbuf[5], (unsigned char)pbuf[6], (unsigned char)pbuf[7], (unsigned char)pbuf[8], (unsigned char)pbuf[9]);
if (frameinfo.type == EASY_SDK_VIDEO_FRAME_I) //关键帧
{
_TRACE("[ch%d]当前关键帧解码失败...\n", pThread->channelId);
#ifdef _DEBUG
FILE *f = fopen("keyframe.txt", "wb");
if (NULL != f)
{
fwrite(pbuf, 1, frameinfo.length, f);
fclose(f);
}
#endif
}
else
{
#ifdef _DEBUG
FILE *f = fopen("pframe.txt", "wb");
if (NULL != f)
{
fwrite(pbuf, 1, frameinfo.length, f);
fclose(f);
}
#endif
}
pThread->findKeyframe = 0x01;
}
After testing, after the above code changes can effectively avoid the H265 video decoder when the initial frame will always be the case of the video frame or a local card appears.
Four, H265 video format MP4 write
Then here before EasyPlayer series of articles written MP4 speaking, the H265 MP4 package;
-
Analytical H265 head, or VPS, SPS and PPS
taken from the H265 frame header NAL above explanation has been made here do not described in detail here; -
Write VPS, SPS and PPS and other key decoding information
bool EasyMP4Writer::WriteH265VPSSPSandPPS(unsigned char*vps, int vpslen, unsigned char*sps, int spslen,
unsigned char*pps, int ppslen, int width, int height)
{
if (m_nCreateFileFlag&ZOUTFILE_FLAG_VIDEO)
{
m_videtrackid = gf_isom_new_track(p_file, 0, GF_ISOM_MEDIA_VISUAL, 1000);
gf_isom_set_track_enabled(p_file, m_videtrackid, 1);
}
else
{
return false;
}
p_videosample = gf_isom_sample_new();
p_videosample->data = (char*)malloc(1024 * 1024);
p_hevc_config = gf_odf_hevc_cfg_new();
p_hevc_config->nal_unit_size = 32 / 8;
//gf_isom_avc_config_new(p_file, m_videtrackid, p_config, NULL, NULL, &i_videodescidx);
//gf_isom_set_visual_info(p_file, m_videtrackid, i_videodescidx, width, height);
//初始化配置
gf_isom_hevc_config_new(p_file, m_videtrackid, p_hevc_config, NULL, NULL, &i_videodescidx);
gf_isom_set_nalu_extract_mode(p_file, m_videtrackid, GF_ISOM_NALU_EXTRACT_INSPECT);
gf_isom_set_cts_packing(p_file, m_videtrackid, GF_TRUE);
HEVCState hevc = { 0 };
m_slotsps = { 0 };
m_slotpps = { 0 };
m_slotvps = { 0 };
m_spss = { 0 };
m_ppss = { 0 };
m_vpss = { 0 };
p_hevc_config->configurationVersion = 1;
//Config vps
int idx = gf_media_hevc_read_vps((char*)vps, vpslen, &hevc);
hevc.vps[idx].crc = gf_crc_32((char*)vps, vpslen);
p_hevc_config->avgFrameRate = hevc.vps[idx].rates[0].avg_pic_rate;
p_hevc_config->constantFrameRate = hevc.vps[idx].rates[0].constand_pic_rate_idc;
p_hevc_config->numTemporalLayers = hevc.vps[idx].max_sub_layers;
p_hevc_config->temporalIdNested = hevc.vps[idx].temporal_id_nesting;
m_vpss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_vpss);
m_vpss.array_completeness = 1;
m_vpss.type = GF_HEVC_NALU_VID_PARAM;// naltype = VPS
m_slotvps.id = idx;
m_slotvps.size = vpslen;
m_slotvps.data = (char*)malloc(vpslen);
memcpy(m_slotvps.data, vps, vpslen);
gf_list_add(m_vpss.nalus, &m_slotvps);
//Config sps
idx = gf_media_hevc_read_sps((char*)sps, spslen, &hevc);
hevc.sps[idx].crc = gf_crc_32((char*)sps, spslen);
p_hevc_config->profile_space = hevc.sps[idx].ptl.profile_space;
p_hevc_config->tier_flag = hevc.sps[idx].ptl.tier_flag;
p_hevc_config->profile_idc = hevc.sps[idx].ptl.profile_idc;
m_spss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_spss);
m_spss.array_completeness = 1;
m_spss.type = GF_HEVC_NALU_SEQ_PARAM;// naltype = SPS
m_slotsps.id = idx;
m_slotsps.size = spslen;
m_slotsps.data = (char*)malloc(spslen);
memcpy(m_slotsps.data, sps, spslen);
gf_list_add(m_spss.nalus, &m_slotsps);
int act_width = hevc.sps[idx].width;
int act_height = hevc.sps[idx].height;
//Config pps
idx = gf_media_hevc_read_pps((char*)pps, ppslen, &hevc);
hevc.pps[idx].crc = gf_crc_32((char*)pps, ppslen);
m_ppss.nalus = gf_list_new();
gf_list_add(p_hevc_config->param_array, &m_ppss);
m_ppss.array_completeness = 1;
m_ppss.type = GF_HEVC_NALU_PIC_PARAM;// naltype = PPS
m_slotpps.id = idx;
m_slotpps.size = ppslen;
m_slotpps.data = (char*)malloc(ppslen);
memcpy(m_slotpps.data, pps, ppslen);
gf_list_add(m_ppss.nalus, &m_slotpps);
gf_isom_set_visual_info(p_file, m_videtrackid, i_videodescidx, act_width, act_height);
gf_isom_hevc_config_update(p_file, m_videtrackid, 1, p_hevc_config);
//销毁申请的内存资源
gf_list_del(m_vpss.nalus);
gf_list_del(m_spss.nalus);
gf_list_del(m_ppss.nalus);
free(m_slotvps.data);
free(m_slotsps.data);
free(m_slotpps.data);
p_hevc_config->param_array = NULL;
return true;
}
MP4 binding write articles we can see, on MP4Box encapsulated H265 special structural function gf_isom_hevc_config_new () to set the parameters of the H265, H264 and methods provided similar, but more detailed treatment of H265, MP4Box the VPS, SPS, PPS is split out of each parameter assignment, writing decoded parameter information gf_isom_hevc_config_update.