Reprinted from the original address: https://segmentfault.com/a/1190000042611851
Previously we introduced ffplay's debugging environment integration, ffplay's overall architecture, ffplay's reading thread and other related content. Today we will introduce the ffplay decoding thread workflow.
Because the process of video decoding and audio decoding are roughly the same, this article mainly introduces the content of the video decoding thread, and ignores the decoding of subtitles...
Let’s start with this picture:
As can be seen from the figure, the main work of the decoding thread is to take the resource packet out of the queue to be decoded, then send it to the decoder, and finally put the decoded data frame into the frame queue and wait for SDL to get played.
decoding process
The decoding thread is created when the stream is opened, that is, created in the function stream_component_open. The working function of the video decoding thread is video_thread.
The following is the content of the function video_thread. You can see that after removing the filter-related processing, this function is very streamlined. After obtaining a frame of image data through the function get_video_frame in the for loop, adjust the pts of the data frame, and then convert the data frame Put in the frame queue:
/**
* 视频解码
* @param arg
* @return
*/
static int video_thread(void *arg)
{
VideoState *is = arg;
// 分配frame
AVFrame *frame = av_frame_alloc();
double pts;
double duration;
int ret;
AVRational tb = is->video_st->time_base;
AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);
#if CONFIG_AVFILTER
AVFilterGraph *graph = NULL;
AVFilterContext *filt_out = NULL, *filt_in = NULL;
int last_w = 0;
int last_h = 0;
enum AVPixelFormat last_format = -2;
int last_serial = -1;
int last_vfilter_idx = 0;
#endif
if (!frame)
return AVERROR(ENOMEM);
for (;;) {
// 获取一帧视频图像，返回负值会退出线程，什么时候会返回负值？问管家VideoState
ret = get_video_frame(is, frame);
if (ret < 0)
goto the_end;
if (!ret)
continue;
#if CONFIG_AVFILTER
if ( last_w != frame->width
|| last_h != frame->height
|| last_format != frame->format
|| last_serial != is->viddec.pkt_serial
|| last_vfilter_idx != is->vfilter_idx) {
av_log(NULL, AV_LOG_DEBUG,
"Video frame changed from size:%dx%d format:%s serial:%d to size:%dx%d format:%s serial:%d\n",
last_w, last_h,
(const char *)av_x_if_null(av_get_pix_fmt_name(last_format), "none"), last_serial,
frame->width, frame->height,
(const char *)av_x_if_null(av_get_pix_fmt_name(frame->format), "none"), is->viddec.pkt_serial);
avfilter_graph_free(&graph);
graph = avfilter_graph_alloc();
if (!graph) {
ret = AVERROR(ENOMEM);
goto the_end;
}
graph->nb_threads = filter_nbthreads;
if ((ret = configure_video_filters(graph, is, vfilters_list ? vfilters_list[is->vfilter_idx] : NULL, frame)) < 0) {
SDL_Event event;
event.type = FF_QUIT_EVENT;
event.user.data1 = is;
SDL_PushEvent(&event);
goto the_end;
}
filt_in = is->in_video_filter;
filt_out = is->out_video_filter;
last_w = frame->width;
last_h = frame->height;
last_format = frame->format;
last_serial = is->viddec.pkt_serial;
last_vfilter_idx = is->vfilter_idx;
frame_rate = av_buffersink_get_frame_rate(filt_out);
}
ret = av_buffersrc_add_frame(filt_in, frame);
if (ret < 0)
goto the_end;
while (ret >= 0) {
is->frame_last_returned_time = av_gettime_relative() / 1000000.0;
ret = av_buffersink_get_frame_flags(filt_out, frame, 0);
if (ret < 0) {
if (ret == AVERROR_EOF)
is->viddec.finished = is->viddec.pkt_serial;
ret = 0;
break;
}
is->frame_last_filter_delay = av_gettime_relative() / 1000000.0 - is->frame_last_returned_time;
if (fabs(is->frame_last_filter_delay) > AV_NOSYNC_THRESHOLD / 10.0)
is->frame_last_filter_delay = 0;
tb = av_buffersink_get_time_base(filt_out);
#endif
// 计算持续播放时间
duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){
frame_rate.den, frame_rate.num}) : 0);
pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
// 将解码得到的帧放进队列
ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
av_frame_unref(frame);
#if CONFIG_AVFILTER
if (is->videoq.serial != is->viddec.pkt_serial)
break;
}
#endif
if (ret < 0)
goto the_end;
}
the_end:
#if CONFIG_AVFILTER
avfilter_graph_free(&graph);
#endif
av_frame_free(&frame);
return 0;
}
Let's take a look at what the function get_video_frame does:
/**
* 获取一帧图像
* @param is
* @param frame
* @return
*/
static int get_video_frame(VideoState *is, AVFrame *frame)
{
int got_picture;
// 获取解码数据
if ((got_picture = decoder_decode_frame(&is->viddec, frame, NULL)) < 0)
return -1;
// 分析是否需要丢帧，例如解码出来的图片已经比真正播放的音频都慢了，那就要丢帧了
if (got_picture) {
double dpts = NAN;
if (frame->pts != AV_NOPTS_VALUE)
// 计算pts
dpts = av_q2d(is->video_st->time_base) * frame->pts;
frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame);
// 同步时钟不以视频为基准时
if (framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) {
if (frame->pts != AV_NOPTS_VALUE) {
// 理论上如果需要连续接上播放的话 dpts + diff = get_master_clock(is)
// 所以可以算出diff 注意绝对值
double diff = dpts - get_master_clock(is);
if (!isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD &&
diff - is->frame_last_filter_delay < 0 &&
is->viddec.pkt_serial == is->vidclk.serial &&
is->videoq.nb_packets) {
is->frame_drops_early++;
av_frame_unref(frame);
got_picture = 0;
}
}
}
}
return got_picture;
}
This function internally calls the function decoder_decode_frame to obtain the decoded frame, and then determines whether frame loss processing is required based on the synchronized clock. Then let’s look at the function decoder_decode_frame
static int decoder_decode_frame(Decoder *d, AVFrame *frame, AVSubtitle *sub) {
int ret = AVERROR(EAGAIN);
for (;;) {
// 解码器的序列需要和解码包队列的序列一致
if (d->queue->serial == d->pkt_serial) {
do {
// 请求退出了则返回-1
if (d->queue->abort_request)
return -1;
switch (d->avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO:
// 获取1帧解码数据
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
// 更新pts为AVPacket的pts
if (decoder_reorder_pts == -1) {
frame->pts = frame->best_effort_timestamp;
} else if (!decoder_reorder_pts) {
frame->pts = frame->pkt_dts;
}
}
break;
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
AVRational tb = (AVRational){
1, frame->sample_rate};
if (frame->pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb);
else if (d->next_pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
if (frame->pts != AV_NOPTS_VALUE) {
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb;
}
}
break;
}
if (ret == AVERROR_EOF) {
d->finished = d->pkt_serial;
// 刷新解码器
avcodec_flush_buffers(d->avctx);
return 0;
}
if (ret >= 0)
return 1;
} while (ret != AVERROR(EAGAIN));
}
do {
if (d->queue->nb_packets == 0)
// 队列空了，唤醒读取线程，赶紧读取数据
SDL_CondSignal(d->empty_queue_cond);
if (d->packet_pending) {
// 处理有缓冲的数据
d->packet_pending = 0;
} else {
int old_serial = d->pkt_serial;
if (packet_queue_get(d->queue, d->pkt, 1, &d->pkt_serial) < 0)
return -1;
if (old_serial != d->pkt_serial) {
// 刷新解码器
avcodec_flush_buffers(d->avctx);
d->finished = 0;
d->next_pts = d->start_pts;
d->next_pts_tb = d->start_pts_tb;
}
}
if (d->queue->serial == d->pkt_serial)
break;
av_packet_unref(d->pkt);
} while (1);
if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
int got_frame = 0;
ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, d->pkt);
if (ret < 0) {
ret = AVERROR(EAGAIN);
} else {
if (got_frame && !d->pkt->data) {
d->packet_pending = 1;
}
ret = got_frame ? 0 : (d->pkt->data ? AVERROR(EAGAIN) : AVERROR_EOF);
}
av_packet_unref(d->pkt);
} else {
// 数据送进解码器失败，遇到EAGAIN怎么办？
if (avcodec_send_packet(d->avctx, d->pkt) == AVERROR(EAGAIN)) {
av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
d->packet_pending = 1;
} else {
av_packet_unref(d->pkt);
}
}
}
}
It turns out that the function decoder_decode_frame is the real core of decoding. You can see that video, audio and subtitles are all decoded by calling this function. The main job of this function is to continuously retrieve data packets in the for loop and then call FFmpeg's API for decoding.
1. First, determine whether the playback sequence of the queue to be decoded is consistent with the playback sequence of the decoder:
if (d->queue->serial == d->pkt_serial)
If consistent, obtain the data frame through the FFmpeg function avcodec_receive_frame, and then update the data frame pts.
There is a question here. Didn’t we say before that the decoding process is to first avcodec_send_packet and then n avcodec_receive_frame to obtain the data frame? How can I get the data frame directly from avcodec_receive_frame here?
This is because the function decoder_decode_frame is a for loop that is constantly called. Calling avcodec_receive_frame first can prevent the decoder from buffering too much data internally and causing the decoded packet to be unable to be sent to the decoder.
2. Whether there is data in the queue to be decoded, if not, wake up the reading thread
if (d->queue->nb_packets == 0)
// 队列空了，唤醒读取线程，赶紧读取数据
SDL_CondSignal(d->empty_queue_cond);
3. Determine whether there are any packets that failed to be sent to the decoder. If so, process them first.
if (d->packet_pending) {
// 处理有缓冲的数据
d->packet_pending = 0;
}
This packet is cached when calling avcodec_send_packet returns AVERROR (EAGAIN).
4. If there is no cached packet, get packet_queue_get from the queue.
int old_serial = d->pkt_serial;
if (packet_queue_get(d->queue, d->pkt, 1, &d->pkt_serial) < 0)
return -1;
if (old_serial != d->pkt_serial) {
// 刷新解码器
avcodec_flush_buffers(d->avctx);
d->finished = 0;
d->next_pts = d->start_pts;
d->next_pts_tb = d->start_pts_tb;
}
5. Send the packet to the decoder. If the returned value is AVERROR (EAGAIN), cache the packet and send it to the decoder first for processing next time.
At the same time, the reason why avcodec_receive_frame is called first in the first step can also be analyzed here, just so that the data can be sent to the decoder smoothly.
// 数据送进解码器失败，遇到EAGAIN怎么办？
if (avcodec_send_packet(d->avctx, d->pkt) == AVERROR(EAGAIN)) {
av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
d->packet_pending = 1;
} else {
av_packet_unref(d->pkt);
}
>>> 音视频开发 视频教程： https://ke.qq.com/course/3202131?flowToken=1031864
>>> 音视频开发学习资料、教学视频，免费分享有需要的可以自行添加学习交流群 739729163 领取