Article directory
1. Decoding thread framework
The decoding thread of ffplay is independent of the data reading thread, and each stream is assigned its own decoding thread.
- video_thread is used to decode video_stream
- audio_thread is used to decode audio_stream
- subtitle_thread is used to decode subtitle_stream
Decoder packaging structure
typedef struct Decoder {
AVPacket pkt;
PacketQueue *queue; // 数据包队列
AVCodecContext *avctx; // 解码器上下文
int pkt_serial; // 包序列
int finished; // =0，解码器处于工作状态；=非0，解码器处于空闲状态
int packet_pending; // =0，解码器处于异常状态，需要考虑重置解码器；=1，解码器处于正常状态
SDL_cond *empty_queue_cond; // 检查到packet队列空时发送 signal缓存read_thread读取数据
int64_t start_pts; // 初始化时是stream的start time
AVRational start_pts_tb; // 初始化时是stream的time_base
int64_t next_pts; // 记录最近一次解码后的frame的pts，当解出来的部分帧没有有效的pts时则使用next_pts进行推算
AVRational next_pts_tb; // next_pts的单位
SDL_Thread *decoder_tid; // 线程句柄
} Decoder;
Decoder related operation functions
- 初始化解码器
static void decoder_init(Decoder *d, AVCodecContext *avctx, PacketQueue *queue, SDL_cond *empty_queue_cond)
- 启用解码器线程
static int decoder_start(Decoder *d, int (*fn)(void *), const char *thread_name, void* arg)
- 解帧
static int decoder_decode_frame(Decoder *d, AVFrame *frame, AVSubtitle *sub)
- 终止解码器
static void decoder_abort(Decoder *d, FrameQueue *fq)
- 销毁解码器
static void decoder_destroy(Decoder *d)
Start the decoder thread:
decoder_init
decoder_start
The specific process of decoding thread:
decoder_decode_frame
Exit the decoding thread:
decoder_abort
decoder_destroy
2. Video decoding thread function
Data source: The avpacket read by read_thread is put into the corresponding packet video queue.
Data processing: The video_thread reads the AVPacket in the packet video queue for decoding, obtains the AVFrame and puts it into the corresponding frame video queue.
Data export: Read the frame in the frame video queue in video_refresh for display.
// 视频解码线程
static int video_thread(void *arg)
{
VideoState *is = arg;
AVFrame *frame = av_frame_alloc(); // 分配解码帧
double pts; // pts
double duration; // 帧持续时间
int ret;
//1 获取stream timebase
AVRational tb = is->video_st->time_base; // 获取stream timebase
//2 获取帧率，以便计算每帧picture的duration
AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);
if (!frame)
return AVERROR(ENOMEM);
for (;;) {
// 循环取出视频解码的帧数据
// 3 获取解码后的视频帧
ret = get_video_frame(is, frame);
if (ret < 0)
goto the_end; //解码结束, 什么时候会结束
if (!ret) //没有解码得到画面, 什么情况下会得不到解后的帧
continue;
// 4 计算帧持续时间和换算pts值为秒
// 1/帧率 = duration 单位秒, 没有帧率时则设置为0, 有帧率帧计算出帧间隔
duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){
frame_rate.den, frame_rate.num}) : 0);
// 根据AVStream timebase计算出pts值, 单位为秒
pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
// 5 将解码后的视频帧插入队列
ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
// 6 释放frame对应的数据
av_frame_unref(frame);
if (ret < 0) // 返回值小于0则退出线程
goto the_end;
}
the_end:
av_frame_free(&frame);
return 0;
}
The above code is the source code in ffplay, but I deleted the filter part of the code for a better reading experience.
You can see from the code that the process is very simple
It mainly obtains the frame through get_video_frame, calculates the real pts and duration, and then inserts it into the frame video queue through queue_picture.
Why do you still need to use queue_picture instead of inserting it directly?
Since we want the code to be highly usable, we encapsulate the AVFrame into the Frame structure through the queue_picture function and put the Frame structure into the queue, so that video, audio, and subtitle frames can be stored using a queue interface.
Let’s take a look at the get_video_frame function first:
static int get_video_frame(VideoState *is, AVFrame *frame)
{
int got_picture;
// 1. 获取解码后的视频帧
if ((got_picture = decoder_decode_frame(&is->viddec, frame, NULL)) < 0) {
return -1; // 返回-1意味着要退出解码线程, 所以要分析decoder_decode_frame什么情况下返回-1
}
if (got_picture) {
// 2. 分析获取到的该帧是否要drop掉, 该机制的目的是在放入帧队列前先drop掉过时的视频帧
double dpts = NAN;
if (frame->pts != AV_NOPTS_VALUE)
dpts = av_q2d(is->video_st->time_base) * frame->pts; //计算出秒为单位的pts
frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame);
if (framedrop>0 || // 允许drop帧
(framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER))//非视频同步模式
{
if (frame->pts != AV_NOPTS_VALUE) {
// pts值有效
double diff = dpts - get_master_clock(is);
if (!isnan(diff) && // 差值有效
fabs(diff) < AV_NOSYNC_THRESHOLD && // 差值在可同步范围呢
diff - is->frame_last_filter_delay < 0 && // 和过滤器有关系
is->viddec.pkt_serial == is->vidclk.serial && // 同一序列的包
is->videoq.nb_packets) {
// packet队列至少有1帧数据
is->frame_drops_early++;
printf("%s(%d) diff:%lfs, drop frame, drops:%d\n",
__FUNCTION__, __LINE__, diff, is->frame_drops_early);
av_frame_unref(frame);
got_picture = 0;
}
}
}
}
return got_picture;
}
After reading the frame through decoder_decode_frame, it is judged whether to discard the frame. The principle is to look at the synchronization difference between the video frame and other frames. get_master_clock means to get the pts of other frames, and then compare it with the pts of the current video frame. The value is said to be diff, and then the judgment condition is used to determine whether to discard it. This will be discussed in detail when synchronizing audio and video.
Looking at the code, I found that decoder_decode_frame was called again to read the frame. Why?
Because the function that actually reads the frame is decoder_decode_frame, this function controls the frame type.
Take a look at the code:
static int decoder_decode_frame(Decoder *d, AVFrame *frame, AVSubtitle *sub) {
int ret = AVERROR(EAGAIN);
for (;;) {
AVPacket pkt;
// 1. 流连续情况下获取解码后的帧
if (d->queue->serial == d->pkt_serial) {
// 1.1 先判断是否是同一播放序列的数据
do {
if (d->queue->abort_request)
return -1; // 是否请求退出
// 1.2. 获取解码帧
switch (d->avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO:
ret = avcodec_receive_frame(d->avctx, frame);
//printf("frame pts:%ld, dts:%ld\n", frame->pts, frame->pkt_dts);
if (ret >= 0) {
if (decoder_reorder_pts == -1) {
frame->pts = frame->best_effort_timestamp;
} else if (!decoder_reorder_pts) {
frame->pts = frame->pkt_dts;
}
}
break;
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
AVRational tb = (AVRational){
1, frame->sample_rate}; //
if (frame->pts != AV_NOPTS_VALUE) {
// 如果frame->pts正常则先将其从pkt_timebase转成{1, frame->sample_rate}
// pkt_timebase实质就是stream->time_base
frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb);
}
else if (d->next_pts != AV_NOPTS_VALUE) {
// 如果frame->pts不正常则使用上一帧更新的next_pts和next_pts_tb
// 转成{1, frame->sample_rate}
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
}
if (frame->pts != AV_NOPTS_VALUE) {
// 根据当前帧的pts和nb_samples预估下一帧的pts
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb; // 设置timebase
}
}
break;
}
// 1.3. 检查解码是否已经结束，解码结束返回0
if (ret == AVERROR_EOF) {
d->finished = d->pkt_serial;
printf("avcodec_flush_buffers %s(%d)\n", __FUNCTION__, __LINE__);
avcodec_flush_buffers(d->avctx);
return 0;
}
// 1.4. 正常解码返回1
if (ret >= 0)
return 1;
} while (ret != AVERROR(EAGAIN)); // 1.5 没帧可读时ret返回EAGIN，需要继续送packet
}
// 2 获取一个packet，如果播放序列不一致(数据不连续)则过滤掉“过时”的packet
do {
// 2.1 如果没有数据可读则唤醒read_thread, 实际是continue_read_thread SDL_cond
if (d->queue->nb_packets == 0) // 没有数据可读
SDL_CondSignal(d->empty_queue_cond);// 通知read_thread放入packet
// 2.2 如果还有pending的packet则使用它
if (d->packet_pending) {
av_packet_move_ref(&pkt, &d->pkt);
d->packet_pending = 0;
} else {
// 2.3 阻塞式读取packet
if (packet_queue_get(d->queue, &pkt, 1, &d->pkt_serial) < 0)
return -1;
}
} while (d->queue->serial != d->pkt_serial);// 如果不是同一播放序列(流不连续)则继续读取
// 3 将packet送入解码器
if (pkt.data == flush_pkt.data) {
//
// when seeking or when switching to a different stream
avcodec_flush_buffers(d->avctx); //清空里面的缓存帧
d->finished = 0; // 重置为0
d->next_pts = d->start_pts; // 主要用在了audio
d->next_pts_tb = d->start_pts_tb;// 主要用在了audio
} else {
if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
int got_frame = 0;
ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt);
if (ret < 0) {
ret = AVERROR(EAGAIN);
} else {
if (got_frame && !pkt.data) {
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF);
}
} else {
if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) {
av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
}
av_packet_unref(&pkt); // 一定要自己去释放音视频数据
}
}
}
This function first checks whether the packet queue serial number and the decoding serial number are consistent. If it is always distinguished by the decoder type, obtain the frame through avcodec_receive_frame. If the frame is obtained, it will return. If the frame is not obtained and there is no EOF, then send a packet. However, Before that, it will check whether the packet queue serial number and the decoding serial number are consistent. If they are inconsistent, the packet will be discarded until they are consistent, and then the packet will be sent. If the send is flush_pkt, avcodec_flush_buffers will be called to clear the cache. If it is a subtitle package, avcodec_decode_subtitle2 will be called. Otherwise, avcodec_decode_subtitle2 will be called. Just call avcodec_send_packet directly.
static int queue_picture(VideoState *is, AVFrame *src_frame, double pts,
double duration, int64_t pos, int serial)
{
Frame *vp;
if (!(vp = frame_queue_peek_writable(&is->pictq))) // 检测队列是否有可写空间
return -1; // 请求退出则返回-1
// 执行到这步说已经获取到了可写入的Frame
vp->sar = src_frame->sample_aspect_ratio;
vp->uploaded = 0;
vp->width = src_frame->width;
vp->height = src_frame->height;
vp->format = src_frame->format;
vp->pts = pts;
vp->duration = duration;
vp->pos = pos;
vp->serial = serial;
set_default_window_size(vp->width, vp->height, vp->sar);
av_frame_move_ref(vp->frame, src_frame); // 将src中所有数据转移到dst中，并复位src。
frame_queue_push(&is->pictq); // 更新写索引位置
return 0;
}
From this part of the code, we can know that the read AVFrame is encapsulated into a Frame, and then the pts and duration are set in seconds, and then some default window sizes are set and put into the queue.
This ends the video decoding thread~~~
3. Audio decoding thread function
// 音频解码线程
static int audio_thread(void *arg)
{
VideoState *is = arg;
AVFrame *frame = av_frame_alloc(); // 分配解码帧
Frame *af;
#if CONFIG_AVFILTER
int last_serial = -1;
int64_t dec_channel_layout;
int reconfigure;
#endif
int got_frame = 0; // 是否读取到帧
AVRational tb; // timebase
int ret = 0;
if (!frame)
return AVERROR(ENOMEM);
do {
// 1. 读取解码帧
if ((got_frame = decoder_decode_frame(&is->auddec, frame, NULL)) < 0)
goto the_end;
if (got_frame) {
tb = (AVRational){
1, frame->sample_rate}; // 设置为sample_rate为timebase
// 2. 获取可写Frame
if (!(af = frame_queue_peek_writable(&is->sampq))) // 获取可写帧
goto the_end;
// 3. 设置Frame并放入FrameQueue
af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
af->pos = frame->pkt_pos;
af->serial = is->auddec.pkt_serial;
af->duration = av_q2d((AVRational){
frame->nb_samples, frame->sample_rate});
av_frame_move_ref(af->frame, frame);
frame_queue_push(&is->sampq);
}
} while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF);
the_end:
av_frame_free(&frame);
return ret;
}
The audio process and the video process are basically the same, but one point is why tb = (AVRational){1, frame->sample_rate} is used when converting pts and duration in seconds here instead of using stream->time_base?
The main reason is that when processing audio in decoder_decode_frame, time_base is converted into (AVRational){1, frame->sample_rate}
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
AVRational tb = (AVRational){
1, frame->sample_rate}; //
if (frame->pts != AV_NOPTS_VALUE) {
// 如果frame->pts正常则先将其从pkt_timebase转成{1, frame->sample_rate}
// pkt_timebase实质就是stream->time_base
frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb);
}
else if (d->next_pts != AV_NOPTS_VALUE) {
// 如果frame->pts不正常则使用上一帧更新的next_pts和next_pts_tb
// 转成{1, frame->sample_rate}
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
}
if (frame->pts != AV_NOPTS_VALUE) {
// 根据当前帧的pts和nb_samples预估下一帧的pts
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb; // 设置timebase
}
}
break;