Article directory
1.ffplay introduction
ffplay is a player provided by FFmpeg source code. It is a player implemented by the API of FFmpeg and SDL. It has reference significance for the secondary development of subsequent players, such as Bilibili's ijkplayer.
2.FFplay framework process
3.Data structure analysis
3.1 The general manager of ffplay
typedef struct VideoState {
SDL_Thread *read_tid; // 读线程句柄
AVInputFormat *iformat; // 指向demuxer
int abort_request; // =1时请求退出播放
int force_refresh; // =1时需要刷新画面，请求立即刷新画面的意思
int paused; // =1时暂停，=0时播放
int last_paused; // 暂存“暂停”/“播放”状态
int queue_attachments_req;
int seek_req; // 标识一次seek请求
int seek_flags; // seek标志，诸如AVSEEK_FLAG_BYTE等
int64_t seek_pos; // 请求seek的目标位置(当前位置+增量)
int64_t seek_rel; // 本次seek的位置增量
int read_pause_return;
AVFormatContext *ic; // iformat的上下文
int realtime; // =1为实时流
Clock audclk; // 音频时钟
Clock vidclk; // 视频时钟
Clock extclk; // 外部时钟
FrameQueue pictq; // 视频Frame队列
FrameQueue subpq; // 字幕Frame队列
FrameQueue sampq; // 采样Frame队列
Decoder auddec; // 音频解码器
Decoder viddec; // 视频解码器
Decoder subdec; // 字幕解码器
int audio_stream ; // 音频流索引
int av_sync_type; // 音视频同步类型, 默认audio master
double audio_clock; // 当前音频帧的PTS+当前帧Duration
int audio_clock_serial; // 播放序列，seek可改变此值
// 以下4个参数 非audio master同步方式使用
double audio_diff_cum; // used for AV difference average computation
double audio_diff_avg_coef;
double audio_diff_threshold;
int audio_diff_avg_count;
// end
AVStream *audio_st; // 音频流
PacketQueue audioq; // 音频packet队列
int audio_hw_buf_size; // SDL音频缓冲区的大小(字节为单位)
// 指向待播放的一帧音频数据，指向的数据区将被拷入SDL音频缓冲区。若经过重采样则指向audio_buf1，
// 否则指向frame中的音频
uint8_t *audio_buf; // 指向需要重采样的数据
uint8_t *audio_buf1; // 指向重采样后的数据
unsigned int audio_buf_size; // 待播放的一帧音频数据(audio_buf指向)的大小
unsigned int audio_buf1_size; // 申请到的音频缓冲区audio_buf1的实际尺寸
int audio_buf_index; // 更新拷贝位置 当前音频帧中已拷入SDL音频缓冲区
// 的位置索引(指向第一个待拷贝字节)
// 当前音频帧中尚未拷入SDL音频缓冲区的数据量:
// audio_buf_size = audio_buf_index + audio_write_buf_size
int audio_write_buf_size;
int audio_volume; // 音量
int muted; // =1静音，=0则正常
struct AudioParams audio_src; // 音频frame的参数
#if CONFIG_AVFILTER
struct AudioParams audio_filter_src;
#endif
struct AudioParams audio_tgt; // SDL支持的音频参数，重采样转换：audio_src->audio_tgt
struct SwrContext *swr_ctx; // 音频重采样context
int frame_drops_early; // 丢弃视频packet计数
int frame_drops_late; // 丢弃视频frame计数
enum ShowMode {
SHOW_MODE_NONE = -1, // 无显示
SHOW_MODE_VIDEO = 0, // 显示视频
SHOW_MODE_WAVES, // 显示波浪，音频
SHOW_MODE_RDFT, // 自适应滤波器
SHOW_MODE_NB
} show_mode;
// 音频波形显示使用
int16_t sample_array[SAMPLE_ARRAY_SIZE]; // 采样数组
int sample_array_index; // 采样索引
int last_i_start; // 上一开始
RDFTContext *rdft; // 自适应滤波器上下文
int rdft_bits; // 自使用比特率
FFTSample *rdft_data; // 快速傅里叶采样
int xpos;
double last_vis_time;
SDL_Texture *vis_texture; // 音频Texture
SDL_Texture *sub_texture; // 字幕显示
SDL_Texture *vid_texture; // 视频显示
int subtitle_stream; // 字幕流索引
AVStream *subtitle_st; // 字幕流
PacketQueue subtitleq; // 字幕packet队列
double frame_timer; // 记录最后一帧播放的时刻
double frame_last_returned_time; // 上一次返回时间
double frame_last_filter_delay; // 上一个过滤器延时
int video_stream; // 视频流索引
AVStream *video_st; // 视频流
PacketQueue videoq; // 视频队列
double max_frame_duration; // 一帧最大间隔. above this, we consider the jump a timestamp discontinuity
struct SwsContext *img_convert_ctx; // 视频尺寸格式变换
struct SwsContext *sub_convert_ctx; // 字幕尺寸格式变换
int eof; // 是否读取结束
char *filename; // 文件名
int width, height, xleft, ytop; // 宽、高，x起始坐标，y起始坐标
int step; // =1 步进播放模式, =0 其他模式
#if CONFIG_AVFILTER
int vfilter_idx;
AVFilterContext *in_video_filter; // the first filter in the video chain
AVFilterContext *out_video_filter; // the last filter in the video chain
AVFilterContext *in_audio_filter; // the first filter in the audio chain
AVFilterContext *out_audio_filter; // the last filter in the audio chain
AVFilterGraph *agraph; // audio filter graph
#endif
// 保留最近的相应audio、video、subtitle流的steam index
int last_video_stream, last_audio_stream, last_subtitle_stream;
SDL_cond *continue_read_thread; // 当读取数据队列满了后进入休眠时，可以通过该condition唤醒读线程
} VideoState;
3.2 struct Clock clock package
typedef struct Clock {
double pts; // 时钟基础, 当前帧(待播放)显示时间戳，播放后，当前帧变成上一帧
// 当前pts与当前系统时钟的差值, audio、video对于该值是独立的
double pts_drift; // clock base minus time at which we updated the clock
// 当前时钟(如视频时钟)最后一次更新时间，也可称当前时钟时间
double last_updated; // 最后一次更新的系统时钟
double speed; // 时钟速度控制，用于控制播放速度
// 播放序列，所谓播放序列就是一段连续的播放动作，一个seek操作会启动一段新的播放序列
int serial; // clock is based on a packet with this serial
int paused; // = 1 说明是暂停状态
// 指向packet_serial
int *queue_serial; /* pointer to the current packet queue serial, used for obsolete clock detection */
} Clock;
3.3 Packet queue data data structure design
typedef struct MyAVPacketList {
AVPacket pkt; //解封装后的数据
struct MyAVPacketList *next; //下一个节点
int serial; //播放序列
} MyAVPacketList;
typedef struct PacketQueue {
MyAVPacketList *first_pkt, *last_pkt; // 队首，队尾指针
int nb_packets; // 包数量，也就是队列元素数量
int size; // 队列所有元素的数据大小总和
int64_t duration; // 队列所有元素的数据播放持续时间
int abort_request; // 用户退出请求标志
int serial; // 播放序列号，和MyAVPacketList的serial作用相同，但改变的时序稍微有点不同
SDL_mutex *mutex; // 用于维持PacketQueue的多线程安全(SDL_mutex可以按pthread_mutex_t理解）
SDL_cond *cond; // 用于读、写线程相互通知(SDL_cond可以按pthread_cond_t理解)
} PacketQueue;
According to the code, we can see that struct MyAVPacketList is a node of an AVPacket, and PacketQueue is used to manage the queue of this group of nodes.
Next, let’s take a look at some functions that operate this queue.
3.3.1 packet_queue_init
static int packet_queue_init(PacketQueue *q)
{
memset(q, 0, sizeof(PacketQueue));
q->mutex = SDL_CreateMutex();
if (!q->mutex) {
av_log(NULL, AV_LOG_FATAL, "SDL_CreateMutex(): %s\n", SDL_GetError());
return AVERROR(ENOMEM);
}
q->cond = SDL_CreateCond();
if (!q->cond) {
av_log(NULL, AV_LOG_FATAL, "SDL_CreateCond(): %s\n", SDL_GetError());
return AVERROR(ENOMEM);
}
q->abort_request = 1;
return 0;
}
It can be seen from the source code that this function is used to create a new queue and create locks and condition variables for it.
This function allows us to directly create AVPacket queues for video, audio and subtitles
3.3.2 packet_queue_destroy
static void packet_queue_destroy(PacketQueue *q)
{
packet_queue_flush(q); //先清除所有的节点
SDL_DestroyMutex(q->mutex);
SDL_DestroyCond(q->cond);
}
This function is the opposite of packet_queue_init. It is used to release the packet queue. The function packet_queue_flush is called. This function releases each node in the queue and then releases the lock and condition variable. You may have a question here. Why release PacketQueue? Because this variable is stored on the stack, we can see it by looking at the PacketQueue variable in VideoState
3.3.3 packet_queue_flush
static void packet_queue_flush(PacketQueue *q)
{
MyAVPacketList *pkt, *pkt1;
SDL_LockMutex(q->mutex);
for (pkt = q->first_pkt; pkt; pkt = pkt1) {
pkt1 = pkt->next;
av_packet_unref(&pkt->pkt);
av_freep(&pkt);
}
q->last_pkt = NULL;
q->first_pkt = NULL;
q->nb_packets = 0;
q->size = 0;
q->duration = 0;
SDL_UnlockMutex(q->mutex);
}
This function is very simple: it traverses the queue and releases all the nodes in the queue.
3.3.4 packet_queue_start
static void packet_queue_start(PacketQueue *q)
{
SDL_LockMutex(q->mutex);
q->abort_request = 0;
packet_queue_put_private(q, &flush_pkt); //这里放入了一个flush_pkt
SDL_UnlockMutex(q->mutex);
}
This function is to start the queue. The step is to change abort_request to 0. It was 1 when we initialized it at the beginning.
Then call packet_queue_put_private to put a flush_pkt into the queue, which is used as a dividing line between two pieces of non-continuous data. Inserting flush_pkt will start the serial increment operation by 1, and will start the decoder to clear its own cache, avcodec_flush_buffers, in the decoder_decode_frame function Reflected
3.3.5 packet_queue_abort
static void packet_queue_abort(PacketQueue *q)
{
SDL_LockMutex(q->mutex);
q->abort_request = 1; // 请求退出
SDL_CondSignal(q->cond); //释放一个条件信号
SDL_UnlockMutex(q->mutex);
}
This function is to terminate the queue, that is, it will change the variable in PacketQueue, set it to 1 and request to exit. SDL_CondSignal is called here to prevent other threads from blocking while waiting. This function will be called under the decoder_abort function.
3.3.6 packet_queue_put
static int packet_queue_put(PacketQueue *q, AVPacket *pkt)
{
int ret;
SDL_LockMutex(q->mutex);
ret = packet_queue_put_private(q, pkt);//主要实现
SDL_UnlockMutex(q->mutex);
if (pkt != &flush_pkt && ret < 0)
av_packet_unref(pkt); //放入失败，释放AVPacket
return ret;
}
This function will call packet_queue_put_private. If the insertion fails, the AVPacket will be released.
3.3.7 packet_queue_get
static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block, int *serial)
{
MyAVPacketList *pkt1;
int ret;
SDL_LockMutex(q->mutex); // 加锁
for (;;) {
if (q->abort_request) {
ret = -1;
break;
}
pkt1 = q->first_pkt; //MyAVPacketList *pkt1; 从队头拿数据
if (pkt1) {
//队列中有数据
q->first_pkt = pkt1->next; //队头移到第二个节点
if (!q->first_pkt)
q->last_pkt = NULL;
q->nb_packets--; //节点数减1
q->size -= pkt1->pkt.size + sizeof(*pkt1); //cache大小扣除一个节点
q->duration -= pkt1->pkt.duration; //总时长扣除一个节点
//返回AVPacket，这里发生一次AVPacket结构体拷贝，AVPacket的data只拷贝了指针
*pkt = pkt1->pkt;
if (serial) //如果需要输出serial，把serial输出
*serial = pkt1->serial;
av_free(pkt1); //释放节点内存,只是释放节点，而不是释放AVPacket
ret = 1;
break;
} else if (!block) {
//队列中没有数据，且非阻塞调用
ret = 0;
break;
} else {
//队列中没有数据，且阻塞调用
//这里没有break。for循环的另一个作用是在条件变量满足后重复上述代码取出节点
SDL_CondWait(q->cond, q->mutex);
}
}
SDL_UnlockMutex(q->mutex); // 释放锁
return ret;
}
This function obtains pkt. From the code, we can see that it will traverse the queue. If there is no data in the queue, it will block and wait until the data arrives before exiting.
3.3.8 packet_queue_put_nullpacket
static int packet_queue_put_nullpacket(PacketQueue *q, int stream_index)
{
AVPacket pkt1, *pkt = &pkt1;
av_init_packet(pkt);
pkt->data = NULL;
pkt->size = 0;
pkt->stream_index = stream_index;
return packet_queue_put(q, pkt);
}
Inserting an empty packet means that the file data has been read, and the decoder has to read out all frames.
3.4 Frame queue data structure design
typedef struct Frame {
AVFrame *frame; // 指向数据帧
AVSubtitle sub; // 用于字幕
int serial; // 帧序列，在seek的操作时serial会变化
double pts; // 时间戳，单位为秒
double duration; // 该帧持续时间，单位为秒
int64_t pos; // 该帧在输入文件中的字节位置
int width; // 图像宽度
int height; // 图像高读
int format; // 对于图像为(enum AVPixelFormat)，
// 对于声音则为(enum AVSampleFormat)
AVRational sar; // 图像的宽高比（16:9，4:3...），如果未知或未指定则为0/1
int uploaded; // 用来记录该帧是否已经显示过？
int flip_v; // =1则垂直翻转， = 0则正常播放
} Frame;
/* 这是一个循环队列，windex是指其中的首元素，rindex是指其中的尾部元素. */
typedef struct FrameQueue {
Frame queue[FRAME_QUEUE_SIZE]; // FRAME_QUEUE_SIZE 最大size, 数字太大时会占用大量的内存，需要注意该值的设置
int rindex; // 读索引。待播放时读取此帧进行播放，播放后此帧成为上一帧
int windex; // 写索引
int size; // 当前总帧数
int max_size; // 可存储最大帧数
int keep_last; // = 1说明要在队列里面保持最后一帧的数据不释放，只在销毁队列的时候才将其真正释放
int rindex_shown; // 初始化为0，配合keep_last=1使用
SDL_mutex *mutex; // 互斥量
SDL_cond *cond; // 条件变量
PacketQueue *pktq; // 数据包缓冲队列
} FrameQueue;
The Frame structure encapsulates the decoded AVFrame data. The AVSubtitle is used to store subtitle frames. The Frame is designed for audio, video and subtitles. FrameQueue uses a circular queue. This lock-free queue can be
better to improve performance.
3.4.1 frame_queue_init
static int frame_queue_init(FrameQueue *f, PacketQueue *pktq, int max_size, int keep_last)
{
int i;
memset(f, 0, sizeof(FrameQueue));
if (!(f->mutex = SDL_CreateMutex())) {
av_log(NULL, AV_LOG_FATAL, "SDL_CreateMutex(): %s\n", SDL_GetError());
return AVERROR(ENOMEM);
}
if (!(f->cond = SDL_CreateCond())) {
av_log(NULL, AV_LOG_FATAL, "SDL_CreateCond(): %s\n", SDL_GetError());
return AVERROR(ENOMEM);
}
f->pktq = pktq;
f->max_size = FFMIN(max_size, FRAME_QUEUE_SIZE);
f->keep_last = !!keep_last;
for (i = 0; i < f->max_size; i++)
if (!(f->queue[i].frame = av_frame_alloc())) // 分配AVFrame结构体
return AVERROR(ENOMEM);
return 0;
}
Initialize the queue, create locks and condition variables, and assign avframe to them
Note: max_size cannot be set too large for video frames. Assuming the format is YUV420p, 1.5 * 1080 * 720= 1,166,400byte=1.112Mb for 1080p
3.4.2 frame_queue_destory
static void frame_queue_destory(FrameQueue *f)
{
int i;
for (i = 0; i < f->max_size; i++) {
Frame *vp = &f->queue[i];
// 释放对vp->frame中的数据缓冲区的引用，注意不是释放frame对象本身
frame_queue_unref_item(vp);
// 释放vp->frame对象
av_frame_free(&vp->frame);
}
SDL_DestroyMutex(f->mutex);
SDL_DestroyCond(f->cond);
}
static void frame_queue_unref_item(Frame *vp)
{
av_frame_unref(vp->frame); /* 释放数据 */
avsubtitle_free(&vp->sub);
}
3.4.3 frame_queue_peek_writable
// 获取可写指针
static Frame *frame_queue_peek_writable(FrameQueue *f)
{
/* wait until we have space to put a new frame */
SDL_LockMutex(f->mutex);
while (f->size >= f->max_size &&
!f->pktq->abort_request) {
/* 检查是否需要退出 */
SDL_CondWait(f->cond, f->mutex);
}
SDL_UnlockMutex(f->mutex);
if (f->pktq->abort_request) /* 检查是不是要退出 */
return NULL;
return &f->queue[f->windex];
}
This function is to obtain the Frame pointer for the next writing operation. If the queue is full, it will return empty.
3.4.4 frame_queue_push
// 更新写指针
static void frame_queue_push(FrameQueue *f)
{
if (++f->windex == f->max_size)
f->windex = 0;
SDL_LockMutex(f->mutex);
f->size++;
SDL_CondSignal(f->cond); // 当_readable在等待时则可以唤醒
SDL_UnlockMutex(f->mutex);
}
This function updates the write index
3.4.5 queue_picture
static int queue_picture(VideoState *is, AVFrame *src_frame, double pts,double duration, int64_t pos, int serial)
{
Frame *vp;
#if defined(DEBUG_SYNC)
printf("frame_type=%c pts=%0.3f\n",
av_get_picture_type_char(src_frame->pict_type), pts);
#endif
if (!(vp = frame_queue_peek_writable(&is->pictq))) // 检测队列是否有可写空间,如果没有就会阻塞这边
return -1; // 请求退出则返回-1
// 执行到这步说已经获取到了可写入的Frame
vp->sar = src_frame->sample_aspect_ratio;
vp->uploaded = 0;
vp->width = src_frame->width;
vp->height = src_frame->height;
vp->format = src_frame->format;
vp->pts = pts;
vp->duration = duration;
vp->pos = pos;
vp->serial = serial;
set_default_window_size(vp->width, vp->height, vp->sar);
av_frame_move_ref(vp->frame, src_frame); // 将src中所有数据转移到dst中，并复位src。
frame_queue_push(&is->pictq); // 更新写索引位置
return 0;
}
This function is essentially used to write frames. It obtains the writable Frame position through the frame_queue_peek_writable function, then initializes it and other operations, and finally calls frame_queue_push to point the write index to the next position.
3.4.6 frame_queue_peek_readable
static Frame *frame_queue_peek_readable(FrameQueue *f)
{
/* wait until we have a readable a new frame */
SDL_LockMutex(f->mutex);
while (f->size - f->rindex_shown <= 0 &&
!f->pktq->abort_request) {
SDL_CondWait(f->cond, f->mutex);
}
SDL_UnlockMutex(f->mutex);
if (f->pktq->abort_request)
return NULL;
return &f->queue[(f->rindex + f->rindex_shown) % f->max_size];
}
Get an object from the queue, and if the queue is empty, wait for the write queue to be put in.
3.4.7 frame_queue_next
static void frame_queue_next(FrameQueue *f)
{
if (f->keep_last && !f->rindex_shown) {
f->rindex_shown = 1; // 第一次进来没有更新，对应的frame就没有释放
return;
}
frame_queue_unref_item(&f->queue[f->rindex]);
if (++f->rindex == f->max_size)
f->rindex = 0;
SDL_LockMutex(f->mutex);
f->size--;
SDL_CondSignal(f->cond);
SDL_UnlockMutex(f->mutex);
}
This function updates the write-read index, that is, frame_queue_peek_readable gets the Frame, and then unrefs the current Frame.
3.4.8 frame_queue_peek_last/frame_queue_peek/frame_queue_peek_next
/* 获取队列当前Frame, 在调用该函数前先调用frame_queue_nb_remaining确保有frame可读 */
static Frame *frame_queue_peek(FrameQueue *f)
{
return &f->queue[(f->rindex + f->rindex_shown) % f->max_size];
}
/* 获取当前Frame的下一Frame, 此时要确保queue里面至少有2个Frame */
// 不管你什么时候调用，返回来肯定不是 NULL
static Frame *frame_queue_peek_next(FrameQueue *f)
{
return &f->queue[(f->rindex + f->rindex_shown + 1) % f->max_size];
}
/* 获取last Frame：
* 当rindex_shown=0时，和frame_queue_peek效果一样
* 当rindex_shown=1时，读取的是已经显示过的frame
*/
static Frame *frame_queue_peek_last(FrameQueue *f)
{
return &f->queue[f->rindex]; // 这时候才有意义
}
3.4.9 frame_queue_nb_remaining
static int frame_queue_nb_remaining(FrameQueue *f)
{
return f->size - f->rindex_shown; // 注意这里为什么要减去f->rindex_shown
}
Get the unshown frame, subtract rindex_shown because if it is set to retain the previous frame, then the unshown frame will be -1
3.5 AudioParams audio parameters
typedef struct AudioParams {
int freq; // 采样率
int channels; // 通道数
int64_t channel_layout; // 通道布局，比如2.1声道，5.1声道等
enum AVSampleFormat fmt; // 音频采样格式，比如AV_SAMPLE_FMT_S16表示为有符号16bit深度，交错排列模式。
int frame_size; // 一个采样单元占用的字节数（比如2通道时，则左右通道各采样一次合成一个采样单元）
int bytes_per_sec; // 一秒时间的字节数，比如采样率48Khz，2 channel，16bit，则一秒48000*2*16/8=192000
} AudioParams;
3.6 Decoder decoder structure
typedef struct Decoder {
AVPacket pkt;
PacketQueue *queue; // 数据包队列
AVCodecContext *avctx; // 解码器上下文
int pkt_serial; // 包序列
int finished; // =0，解码器处于工作状态；=非0，解码器处于空闲状态
int packet_pending; // =0，解码器处于异常状态，需要考虑重置解码器；=1，解码器处于正常状态
SDL_cond *empty_queue_cond; // 检查到packet队列空时发送 signal缓存read_thread读取数据
int64_t start_pts; // 初始化时是stream的start time
AVRational start_pts_tb; // 初始化时是stream的time_base
int64_t next_pts; // 记录最近一次解码后的frame的pts，当解出来的部分帧没有有效的pts时则使用next_pts进行推算
AVRational next_pts_tb; // next_pts的单位
SDL_Thread *decoder_tid; // 线程句柄
} Decoder;