How to develop an audio and video player (II)

Continue last lecture content. How to develop an audio and video player (ffmpeg3.2 + sdl2.0, video + audio)

Fourth, the audio and video packets extracted from the queue, and decodes

Decode and play audio frame

There is little change in the way audio decoding.

// decode_audio_thread 解码音频packet的线程
void decode_audio_thread(PlayerContext* playerCtx) {
	AVFrame *pFrame = nullptr;
	while (playerCtx && !playerCtx->quit) {
		AVPacket *pkt;
		if (!playerCtx->audio_queue.pull(pkt)) { // 从音频队列中取出音频包
			SDL_Delay(1);
			continue;
		}
		
		// ------------------------ 根据音频包解码出一个音频帧 ------------------------ //
		if (avcodec_send_packet(playerCtx->audioCodecCtx, pkt)) {// 发送一个pkt给解码器,ffmpeg3之后的写法
			exit(1);
			return;
		}
		int ret = 0;
		while (ret >= 0) {
			if (!pFrame) {
				if (!(pFrame = av_frame_alloc())) {
					fprintf(stderr, "Could not allocate audio frame\n");
					exit(1);
				}
			}
			// 从编码器接收一个frame
			ret = avcodec_receive_frame(playerCtx->audioCodecCtx, pFrame);
			if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
				break;
			else if (ret < 0)
			{
				fprintf(stderr, "Error during decoding\n");
				exit(1);
				return;
			}
		// ----------------------- 对音频帧进行重采样，并给系统缓冲区数据 -------------------------- //
			
			swr_convert(playerCtx->au_convert_ctx, &playerCtx->out_buffer, playerCtx->out_buffer_size, // 对音频的采样率进行转换
				(const uint8_t * *)pFrame->data, playerCtx->audioCodecCtx->frame_size);
			// 上一帧音频数据还没有播完
			while (playerCtx->audio_len > 0) {
				SDL_Delay(1);
			}
			// 重新设置音频缓存
			playerCtx->audio_pos = (Uint8*)playerCtx->out_buffer;
			playerCtx->audio_len = playerCtx->out_buffer_size;
		// -------------------------------- 更新音频时间戳 ----------------------------------- //
			if (pFrame->pts != AV_NOPTS_VALUE) { 
				playerCtx->audio_clk = av_q2d(playerCtx->audio_stream->time_base) * pFrame->pts; /* time_base 是一个分数 av_q2d是一个将分数转换成小数的函数。
																									在此处是用pts乘以time_base得到时间戳 */
			}
			playerCtx->audio_pts_duration = av_q2d(playerCtx->audio_stream->time_base) * pFrame->pkt_duration; // 更新音频帧的持续时间
			av_frame_unref(pFrame);
		}
		// ------------------------------ end ------------------------------------- //
		av_frame_free(&pFrame);
		av_packet_unref(pkt);
		av_packet_free(&pkt);
	}
}
// sdl_audio_callback SDL 的系统回调函数
void sdl_audio_callback(void* userdata, Uint8* stream, int len) {
	PlayerContext* playerCtx = (PlayerContext*)userdata;
	if (!playerCtx || playerCtx->quit) {
		return;
	}
	// 清空stream中的内存 SDL 2.0  
	SDL_memset(stream, 0, len);
	if (playerCtx->audio_len == 0)
		return;
	// 尽可能的给系统传递音频buffer，但不会超出已经解析的大小(audio_len),也不会多给超出系统索要的大小(len)
	Uint32 streamlen = ((Uint32)len > playerCtx->audio_len ? playerCtx->audio_len : len);
	SDL_MixAudio(stream, playerCtx->audio_pos, streamlen, SDL_MIX_MAXVOLUME); // 给系统要分配的控件赋值
	playerCtx->audio_pos += streamlen; // 音频缓存的位置向前
	playerCtx->audio_len -= streamlen; // 音频缓存去掉已经分配掉的大小
}

It should be noted in the decoded audio frames at the calculated audio time. And stored in the audio_clk in. This will be the basis of when the video frame is displayed.

playerCtx->audio_clk = av_q2d(playerCtx->audio_stream->time_base) * pFrame->pts;

Is the current time stamp is calculated pts (i.e., the first few frames of audio) multiplied by the time base time_baseto obtain the current time stamp pts (where the group is a fraction of the time its value is equal to 1 sec / frame rate video).

Decodes and plays the video frame

The process of decoding video frames and audio similar, but the lack of playing video and audio is completely different. Because the need to render the video playback video. But also according to the audio time stamp to display video frames.

// decode_video_thread 解码视频packet的线程
void decode_video_thread(PlayerContext* playerCtx) {
	AVFrame* pFrame = nullptr;
	while (playerCtx && !playerCtx->quit) {
		AVPacket *pkt;
		if (!playerCtx->video_queue.pull(pkt)) { // 从视频队列中取出视频包
			SDL_Delay(1);
			continue;
		}
		// ------------------------ 根据视频包解码出一个视频帧 ------------------------ //
		if (avcodec_send_packet(playerCtx->videoCodecCtx, pkt)) {
			exit(1);
			return;
		}
		int ret = 0;
		while(ret >= 0){
			if (!pFrame) {
				if (!(pFrame = av_frame_alloc())) {
					fprintf(stderr, "Could not allocate audio frame\n");
					exit(1);
				}
			}
			// 从编码器接收一个frame
			ret = avcodec_receive_frame(playerCtx->videoCodecCtx, pFrame);
			if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
				break;
			// --------------------------- 计算视频时间戳 ------------------------------ //
			double pts = 0; 
			int64_t f_pts = pFrame->pts;
			int64_t pkt_dts = pFrame->pkt_dts;
			if (pkt_dts != AV_NOPTS_VALUE) {
				pts = av_frame_get_best_effort_timestamp(pFrame);  // 尽力获取视频帧的时间戳
			}
			else {
				pts = pFrame->pts; // 如果获取时间戳失败，则将当前的pts作为视频的时间戳
			}
			pts *= av_q2d(playerCtx->video_stream->time_base);/* time_base 是一个分数 av_q2d是一个将分数转换成小数的函数。
																 在此处是用pts乘以time_base得到时间戳 */
			if (pts == 0.0) {
				pts = playerCtx->video_clk;
			}
			//计算视频流的时间基。av_q2d将一个分数转换成小数
			double frame_delay = av_q2d(playerCtx->video_stream->time_base); 
			frame_delay += pFrame->repeat_pict * (frame_delay * 0.5); /* 计算视频延迟。 根据 pFrame->repeat_pict的注释：
																	     extra_delay = repeat_pict / (2*fps) => extra_delay = repeat_pict * fps * 0.5 */
			pts += frame_delay; // 视频帧的时间戳+视频延迟等于视频的实际时间戳
			playerCtx->video_clk = pts; // 更新视频时间戳

			// ---------------------------- 音视频同步 -------------------------- //
			int64_t delay = (playerCtx->video_clk - (playerCtx->audio_clk + playerCtx->audio_pts_duration)) * 1000; // 计算视频时间戳和音频的差距。因为这里的单位是微妙。所以*1000 转换成毫秒
			if (delay > 0) { // delay大于0说明视频时间戳提前了，等待一段时间再显示视频。
				SDL_Delay(delay);
			}

			// ---------------------------- SDL 显示视频 ----------------------------//
			sws_scale(playerCtx->vi_convert_ctx, pFrame->data, pFrame->linesize, 0 // 对视频数据进行缩放
				, playerCtx->videoCodecCtx->height, playerCtx->pFrameYUV->data, playerCtx->pFrameYUV->linesize);
			// 更细屏幕的纹理，参数：(texture纹理，rect区域nullptr为更新全局，plane像素数据，pitch数据大小)
			SDL_UpdateYUVTexture(playerCtx->texture, nullptr,
				playerCtx->pFrameYUV->data[0], playerCtx->pFrameYUV->linesize[0], //y
				playerCtx->pFrameYUV->data[1], playerCtx->pFrameYUV->linesize[1], //u
				playerCtx->pFrameYUV->data[2], playerCtx->pFrameYUV->linesize[2]); //v
			//重置渲染器
			SDL_RenderClear(playerCtx->renderer);
			//将纹理信息拷贝给渲染器
			SDL_RenderCopy(playerCtx->renderer, playerCtx->texture, nullptr, &playerCtx->sdlRect);
			// 显示
			SDL_RenderPresent(playerCtx->renderer);
			// --------------------------- end --------------------------------- //
			av_frame_unref(pFrame);
		}
		
		av_frame_free(&pFrame);
		av_packet_unref(pkt);
		av_packet_free(&pkt);
	}
	
}

When calculating the video time stamps, according to the different formats of the video frame in the pts may not actually played pts (herein relate to an I frame , P frame problems, we speak in the next chapter, or self Baidu), it is generally by av_frame_get_best_effort_timestamppts to get video frames. Video stream is multiplied by this time-yl pts time_baseto give a current video time stamp ().
Because sdl audio system automatically play the audio frame rate. So his play rate is generally correct. And we know that the time stamp of audio and video. According to the time frame of audio can feel whether the line to wait after each decoded video frames.

Two examples of the former two can be said is the most basic video and audio players. But they all contain a player must process. Next might write two complete player. With video fast forward. Speed ​​and other aspects of the play, written or broadcast about two, pushing the stream. Stay tuned.