ffplay.c source code analysis【1】

overview

　　ffplay is a player that comes with ffmpeg. It is a very reference player implemented by calling ffmpeg and SDL API. Even the famous open source project ijkplayer of station B is also developed on ffplay.c. ffplay realizes the general functions of the player. Mastering its principles is very meaningful for player development. The structure of ffplay is as follows.

　　(1) Initialization: buffer queues before audio and video decoding (PacketQueue audioq, PacketQueue videoq), buffer queues after audio and video decoding (FrameQueue sampq, FrameQueue pictq), clocks (audio, video, external), create data reading threads, etc. for initialization.

The benefits of this article, free C++ audio and video learning materials package, technical video/code, including (audio and video development, interview questions, FFmpeg, webRTC, rtmp, hls, rtsp, ffplay, codec, push-pull stream, srs) ↓↓↓↓↓↓See below↓↓Click at the bottom of the article to get it for free↓↓

　　(2) Data reading thread

　　　　open media file

　　　 Open the decoder corresponding to the code stream, create an audio and video decoding thread, and the decoding thread waits for the buffer queue (PacketQueue audioq, PacketQueue videoq) to have data and wakes up to start decoding

　　　　Call av_read_frame to read the packet, and put it into the packet queue corresponding to different streams according to steam_index, at this time the decoding thread is awakened

　　(3) Audio and video decoding threads

　　　　Read the packet from the audio audioq queue, decode the frame and put it into the audio sampq

　　　　Read the packet from the video videoq queue, decode the frame and put it into the video pictq

　　(4) Play audio and video

　　　　Audio playback, through the SDL callback function, output audio

　　　　Video playback, play on the main thread (event_loop-->refresh_loop_wait_event--->video_refresh)

　　　　Through audio and video synchronization, the default is to synchronize with the audio clock

data structure

　　(1) VideoState player package

typedef struct VideoState {
    SDL_Thread *read_tid;//读线程句柄
    AVInputFormat *iformat;//指向demuxer，解复用器格式：dshow、flv
    int abort_request;//=1请求退出播放
    int force_refresh;//=1需要刷新画面
    int paused;//=1暂停，=0播放
    int last_paused;//保存暂停/播放状态
    int queue_attachments_req;//mp3、acc音频文件附带的专辑封面，所以需要注意的是音频文件不一定只存在音频流本身
    int seek_req;//标识一次seek请求
    int seek_flags;//seek标志，按字节还是时间seek，诸如AVSEEK_BYTE等
    int64_t seek_pos;//请求seek的目标位置（当前位置+增量）
    int64_t seek_rel;//本次seek的增量
    int read_pause_return;
    AVFormatContext *ic;//iformat的上下文
    int realtime;//=1为实时流，还是本地文件
//三种同步的时钟
    Clock audclk;//音频时钟
    Clock vidclk;//视频时钟
    Clock extclk;//外部时钟
     
    FrameQueue pictq;//视频frame队列
    FrameQueue subpq;//字幕frame队列
    FrameQueue sampq;//采样frame队列
 
    Decoder auddec;//音频解码器
    Decoder viddec;//视频解码器
    Decoder subdec;//字幕解码器
 
    int audio_stream;//音频流索引
 
    int av_sync_type;//音视频同步类型，默认audio master
 
    double audio_clock;//当前音频帧的pts+当前帧Duration
    int audio_clock_serial;//播放序列，seek可改变此值
     /* 以下四个参数，非audio master 同步方式使用*/
    double audio_diff_cum;
    double audio_diff_avg_coef;
    double audio_diff_threshold;
    int audio_diff_avg_count;
     
    AVStream *audio_st;//音频流
    PacketQueue audioq;//音频packet队列
    int audio_hw_buf_size;//SDL音频缓冲区的大小
    uint8_t *audio_buf;//指向需要重采样的数据
    uint8_t *audio_buf1;//指向重采样后的数据
    unsigned int audio_buf_size; //带播放的一帧音频数据（audio_buf）大小
    unsigned int audio_buf1_size;//申请到的音频缓冲区audio_buf1实际大小
    int audio_buf_index; //更新拷贝位置，当前音频帧中已拷贝入SDL音频缓冲区的位置索引
   //当前音频帧中尚未拷贝入SDL音频缓冲区的数据量
　　 int audio_write_buf_size;
    int audio_volume;//音量
    int muted;//=1静音，=0正常
    struct AudioParams audio_src;//音频frame的参数
#if CONFIG_AVFILTER
    struct AudioParams audio_filter_src;
#endif
    struct AudioParams audio_tgt;//SDL支持的音频参数，重采样转换
    struct SwrContext *swr_ctx;//音频重采样context
    int frame_drops_early;//丢弃视频packet计数
    int frame_drops_late;//丢弃视频frame计数
 
    enum ShowMode {
        SHOW_MODE_NONE = -1, SHOW_MODE_VIDEO = 0, SHOW_MODE_WAVES, SHOW_MODE_RDFT, SHOW_MODE_NB
    } show_mode;
    //音频波形显示使用
    int16_t sample_array[SAMPLE_ARRAY_SIZE];
    int sample_array_index;
    int last_i_start;
    RDFTContext *rdft;
    int rdft_bits;
    FFTSample *rdft_data;
    int xpos;
    double last_vis_time;
    SDL_Texture *vis_texture;
    SDL_Texture *sub_texture;//字幕显示
    SDL_Texture *vid_texture;//视频显示
 
    int subtitle_stream;//字幕流索引
    AVStream *subtitle_st;//字幕流
    PacketQueue subtitleq;//字幕packet队列
 
    double frame_timer;//记录最后一帧播放时间
    double frame_last_returned_time;
    double frame_last_filter_delay;
    int video_stream;//视频流索引
    AVStream *video_st;//视频流
    PacketQueue videoq;//视频packet队列
    double max_frame_duration; // 一帧最大的间隔
    struct SwsContext *sub_convert_ctx;//字幕尺寸格式变换
    int eof;//是否读取结束
 
    char *filename;//文件名
    int width, height, xleft, ytop;//宽，高，x起始坐标，y起始坐标
    int step;//=1步进播放模式，=0其他模式
 
#if CONFIG_AVFILTER
    int vfilter_idx;
    AVFilterContext *in_video_filter;   // the first filter in the video chain
    AVFilterContext *out_video_filter;  // the last filter in the video chain
    AVFilterContext *in_audio_filter;   // the first filter in the audio chain
    AVFilterContext *out_audio_filter;  // the last filter in the audio chain
    AVFilterGraph *agraph;              // audio filter graph
#endif
    //保存最近的相应audio、video、subtitle流的stream_index
    int last_video_stream, last_audio_stream, last_subtitle_stream;
 
    SDL_cond *continue_read_thread;//当读取线程队列满后进入休眠，可通过condition唤醒读取线程
} VideoState;

　(2) Clock clock package

typedef struct Clock {
    double pts;//时钟基础, 当前帧(待播放)显示时间戳，播放后,当前帧变成上一帧
    double pts_drift;//当前pts与当前系统时钟的差值, audio、video对于该值是独立的
    double last_updated;//最后一次更新的系统时钟
    double speed;//时钟速度控制，用于控制播放速度
    int serial;//播放序列，所谓播放序列就是一段连续的播放动作，一个seek操作会启动一段新的播放序列          
    int paused;//= 1 说明是暂停状态
    int *queue_serial; //指向packet_serial
} Clock;

　　（3）PacketQueue

typedef struct MyAVPacketList {
    AVPacket pkt;//解封装后的数据
    struct MyAVPacketList *next;//下一个节点
    int serial;//播放序列
} MyAVPacketList;

MyAVPacketList can be understood as a node in the queue. Serial marks the playback sequence number of the current node, which is mainly used to distinguish whether there is continuous data. Every time a seek is made, serial will be +1 to distinguish different playback sequences.

typedef struct PacketQueue {
    MyAVPacketList *first_pkt, *last_pkt;//队首，队尾指针
    int nb_packets;//包数量，也就是队列元素数量
    int size;//队列所有元素的数据大小总和
    int64_t duration;//队列所有元素的数据播放持续时间
    int abort_request;//用户退出请求标志
    int serial;//播放序列号
    SDL_mutex *mutex;//用于维持PacketQueue的多线程安全
    SDL_cond *cond;//用于读、写线程相互通知，
} PacketQueue;

　　Audio, video, and subtitle streams all have their own independent PacketQueue, and PacketQueue provides the following methods:

　　(a) packet_queue_init initialization

　　(b) packet_queue_destroy destruction

　　(c) packet_queue_start enabled

//启动队列
static void packet_queue_start(PacketQueue *q)
{
    SDL_LockMutex(q->mutex);
    q->abort_request = 0;
    //这里放入了一个flush_pkt,目的是什么？
    /*
        1.插入flush_pkt 触发PacketQueue其对应的serial，加1操作
        2.触发解码器清空自身缓存，avcodec_flush_buffers()，以备新序列的数据进⾏新解码
    */
    packet_queue_put_private(q, &flush_pkt);//flush_pkt 是⼀个特殊的packet
    SDL_UnlockMutex(q->mutex);
}

　　(d) packet_queue_abort terminated

//终止队列
static void packet_queue_abort(PacketQueue *q)
{
    SDL_LockMutex(q->mutex);
 
    q->abort_request = 1;//请求退出
    /*
    SDL_CondSignal的作⽤在于确保当前等待该条件的线程能被激活并继续执⾏退出流程，并唤醒者会
    检测abort_request标志确定⾃⼰的退出流程。
    */
    SDL_CondSignal(q->cond);//释放一个条件信号
 
    SDL_UnlockMutex(q->mutex);
}

　　(e) packet_queue_get gets a node

/*
参数：
    队列
    输出参数
    调⽤者是否需要在没节点可取的情况下阻塞等待
    输出参数，即MyAVPacketList.serial
*/
static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block, int *serial)
{
    MyAVPacketList *pkt1;
    int ret;
 
    SDL_LockMutex(q->mutex);
 
    for (;;) {
        if (q->abort_request) {//判断是否退出
            ret = -1;
            break;
        }
 
        pkt1 = q->first_pkt;//从队头拿数据
        if (pkt1) {//队列中有数据
            q->first_pkt = pkt1->next;//队头移动到第二个节点
            if (!q->first_pkt)
                q->last_pkt = NULL;
            q->nb_packets--;//节点数减1
            q->size -= pkt1->pkt.size + sizeof(*pkt1);//cache大小扣除一个节点
            q->duration -= pkt1->pkt.duration;//总时长扣除一个节点
            *pkt = pkt1->pkt;//返回AVPacket，这⾥发⽣⼀次AVPacket结构体拷⻉，AVPacket的data只拷⻉了指针
            if (serial)
                *serial = pkt1->serial;
            av_free(pkt1);//释放节点内存，只是释放了节点，而不是释放AVPacket的内存
            ret = 1;
            break;
        } else if (!block) {///队列中没有数据，且⾮阻塞调⽤
            ret = 0;
            break;
        } else {<br>　　　　　　 //这⾥没有break，for循环的另⼀个作⽤是在条件变量满⾜后重复上述代码取出节点
            SDL_CondWait(q->cond, q->mutex);
        }
    }
    SDL_UnlockMutex(q->mutex);
    return ret;
}

The benefits of this article, free C++ audio and video learning materials package, technical video/code, including (audio and video development, interview questions, FFmpeg, webRTC, rtmp, hls, rtsp, ffplay, codec, push-pull stream, srs) ↓↓↓↓↓↓See below↓↓Click at the bottom of the article to get it for free↓↓ 

　　(f) packet_queue_put is stored in a node

　　　　If the insertion fails, the AVPacket needs to be released

//往队列中放入一个节点
static int packet_queue_put(PacketQueue *q, AVPacket *pkt)
{
    int ret;
 
    SDL_LockMutex(q->mutex);
    ret = packet_queue_put_private(q, pkt);//主要的实现
    SDL_UnlockMutex(q->mutex);
 
    if (pkt != &flush_pkt && ret < 0)
        av_packet_unref(pkt);//添加失败，需要释放
 
    return ret;
}

　　Mainly implemented in the function packet_queue_put_private

static int packet_queue_put_private(PacketQueue *q, AVPacket *pkt)
{
    MyAVPacketList *pkt1;
 
    if (q->abort_request)//如果终止，则放入失败
       return -1;
 
    pkt1 = av_malloc(sizeof(MyAVPacketList));//分配内存节点
    if (!pkt1)
        return -1;
    pkt1->pkt = *pkt;//拷⻉AVPacket(浅拷⻉，AVPacket.data等内存并没有拷贝)
    pkt1->next = NULL;
    if (pkt == &flush_pkt)//如果放入的是flush_pkt，需要增加队列的播放序列号，以区分不连续的两段数据
        q->serial++;
    pkt1->serial = q->serial;//用队列序列号标记节点
　　//如果last_pkt为空，说明队列是空，新增节点为头节点
    if (!q->last_pkt)
        q->first_pkt = pkt1;
    else//否则，队列有数据，则让原队尾的next为新增节点
        q->last_pkt->next = pkt1;
    q->last_pkt = pkt1;<br>//队列属性操作：增加节点数、cache大小、cache总时间
    q->nb_packets++;
    q->size += pkt1->pkt.size + sizeof(*pkt1);
    q->duration += pkt1->pkt.duration;
    //发出信号，表明当前队列中有数据了，通知等待中的读线程可以取数据了
    SDL_CondSignal(q->cond);
    return 0;
}

　　(g) packet_queue_put_nullpacket is stored in an empty node

//放入空包意味着流的结束，一般在媒体数据读取完成的时候放入空包
static int packet_queue_put_nullpacket(PacketQueue *q, int stream_index)
{
    AVPacket pkt1, *pkt = &pkt1;
    av_init_packet(pkt);
    pkt->data = NULL;
    pkt->size = 0;
    pkt->stream_index = stream_index;
    return packet_queue_put(q, pkt);
}

　　After the media stream data is read, the encoder still has data cached at this time, and an empty node (empty AVPacket) is added to the linked list, and the decoding thread puts the empty AVPacket into the decoder. At this time, the decoder will consider the end of the stream and flush out all the frames in the cache.

　　（h）packet_queue_flush

　　Clear all nodes in the queue, including the AVPacket corresponding to the node, which is mainly used to exit playback and seek playback.

static void packet_queue_flush(PacketQueue *q)
{
    MyAVPacketList *pkt, *pkt1;
 
    SDL_LockMutex(q->mutex);
    for (pkt = q->first_pkt; pkt; pkt = pkt1) {
        pkt1 = pkt->next;
        av_packet_unref(&pkt->pkt);//释放AVPacket的数据
        av_freep(&pkt);//释放节点
    }
    q->last_pkt = NULL;
    q->first_pkt = NULL;
    q->nb_packets = 0;
    q->size = 0;
    q->duration = 0;
    SDL_UnlockMutex(q->mutex);
}

　　The memory of MyAVPacketList is maintained by PacketQueue, malloc when put, free (only node memory is released) when get, node memory is released by av_freep, and the memory pointed to by the node AVPacket field is released by av_packet_unref

　　(4) FrameQueue

　　Frame is a common structure for audio, video, and subtitles. AVFrame actually stores decoded audio and video data, and AVSubtitle is used to store subtitles.

typedef struct Frame {
    AVFrame *frame;//指向数据帧，音视频解码后的数据
    AVSubtitle sub;//用于字幕
    int serial;//播放序列，在seek时serial会变化
    double pts;  //时间戳，单位为秒
    double duration; //该帧持续时间，单位为秒
    int64_t pos;  //该帧在输入文件中的字节位置
    int width;
    int height;
    int format;
    AVRational sar;
    int uploaded;//记录该帧是否已经显示过
    int flip_v;//1旋转180，0正常播放
} Frame;
 
typedef struct FrameQueue {
    Frame queue[FRAME_QUEUE_SIZE];//队列大小，数字太大时占用内存就会越大，需要注意设置
    int rindex;//读索引，待播放时读取此帧进行播放，播放后此帧成为上一帧
    int windex;//写索引
    int size;//当前总帧数
    int max_size;//可存储最大帧数
    int keep_last;//=1 说明要在队列里面保持最后一帧的数据不释放，只在销毁队列的时候才真正释放
    int rindex_shown;//初始化值为0，配合kepp_last=1使用
    SDL_mutex *mutex;
    SDL_cond *cond;
    PacketQueue *pktq;//数据包缓冲队列
} FrameQueue;

　　FrameQueue is a ring buffer (ring buffer), which is a FIFO implemented by an array. Audio frame_queue, video frame_queue, and subtitle frame_queue are created in ffplay. Each frame_queue has a write end and a read end. The write end is located in the decoding thread, and the read end is located in the playback thread.

　　FrameQueue operations provide the following methods:

　　(a) frame_queue_unref_item: release the AVFrame and AVSubtitle in the Frame
　　(b) frame_queue_init: initialize the queue

static int frame_queue_init(FrameQueue *f, PacketQueue *pktq, int max_size, int keep_last)
{
    int i;
    memset(f, 0, sizeof(FrameQueue));
    if (!(f->mutex = SDL_CreateMutex())) {
        av_log(NULL, AV_LOG_FATAL, "SDL_CreateMutex(): %s\n", SDL_GetError());
        return AVERROR(ENOMEM);
    }
    if (!(f->cond = SDL_CreateCond())) {
        av_log(NULL, AV_LOG_FATAL, "SDL_CreateCond(): %s\n", SDL_GetError());
        return AVERROR(ENOMEM);
    }
    f->pktq = pktq;
    f->max_size = FFMIN(max_size, FRAME_QUEUE_SIZE);//队列大小
    f->keep_last = !!keep_last;//将int取值的keep_last转换为boot取值（0或1）
    for (i = 0; i < f->max_size; i++)
        if (!(f->queue[i].frame = av_frame_alloc()))//为每一个节点分配内存，并不是缓存区的内存
            return AVERROR(ENOMEM);
    return 0;
}

　　(c) frame_queue_destory: Destroy the queue

static void frame_queue_destory(FrameQueue *f)
{
    int i;
    for (i = 0; i < f->max_size; i++) {
        Frame *vp = &f->queue[i];
        frame_queue_unref_item(vp);//释放对vp->frame中的数据缓冲区AVBuffer的引⽤
        av_frame_free(&vp->frame);//释放vp->frame对象本身，节点
    }
    SDL_DestroyMutex(f->mutex);
    SDL_DestroyCond(f->cond);
}

　　(d) frame_queue_signal: Send a wake-up signal
　　(e) frame_queue_peek: Get the current Frame, call frame_queue_nb_remaining before calling to ensure there is a frame readable

static Frame *frame_queue_peek(FrameQueue *f)
{
    return &f->queue[(f->rindex + f->rindex_shown) % f->max_size];
}

　　(f) frame_queue_peek_next: Get the next Frame

　　　　Get the next frame of the current frame. At this time, make sure that there are at least 2 frames in the queue

static Frame *frame_queue_peek_next(FrameQueue *f)
{
    return &f->queue[(f->rindex + f->rindex_shown + 1) % f->max_size];
}

　　(g) frame_queue_nb_remaining: Get the remaining size of the queue

static int frame_queue_nb_remaining(FrameQueue *f)
{
    return f->size - f->rindex_shown;
}

　　(h) frame_queue_peek_last: Get the last frame

static Frame *frame_queue_peek_last(FrameQueue *f)
{
    return &f->queue[f->rindex];
}

　　(i) frame_queue_peek_writable: Get a writable frame, which can be done in a blocking or non-blocking manner

　　　　Apply for a writable frame space at the end of the queue. If the queue is full and there is no space to write, wait (controlled by SDL_cond *cond, triggered by frame_queue_next or frame_queue_signal)

static Frame *frame_queue_peek_writable(FrameQueue *f)
{
    /* wait until we have space to put a new frame */
    SDL_LockMutex(f->mutex);
    while (f->size >= f->max_size && !f->pktq->abort_request) {
        SDL_CondWait(f->cond, f->mutex);//frame_queue_next或frame_queue_signal触发唤醒
    }
    SDL_UnlockMutex(f->mutex);
 
    if (f->pktq->abort_request)
        return NULL;
 
    return &f->queue[f->windex];
}

　　(j) frame_queue_peek_readable: Get a readable frame, which can be done in a blocking or non-blocking manner

　　　　Read a frame from the head of the queue, only read but not delete, wait if there is no frame to read

static Frame *frame_queue_peek_readable(FrameQueue *f)
{
    /* wait until we have a readable a new frame */
    SDL_LockMutex(f->mutex);
    while (f->size - f->rindex_shown <= 0 && !f->pktq->abort_request) {
        SDL_CondWait(f->cond, f->mutex);
    }
    SDL_UnlockMutex(f->mutex);
 
    if (f->pktq->abort_request)
        return NULL;
 
    return &f->queue[(f->rindex + f->rindex_shown) % f->max_size];
}

　　(k) frame_queue_push: update and write the index, at this time the Frame is actually put into the queue, and the number of frames in the queue node is increased by 1

　　　　Push a frame to the end of the queue, only update the count and write pointer, so before using this function, the frame data should be written to the corresponding position of the queue, SDL_CondSignal wakes up the read frame_queue_peek_readable

static void frame_queue_push(FrameQueue *f)
{
    if (++f->windex == f->max_size)
        f->windex = 0;
    SDL_LockMutex(f->mutex);
    f->size++;
    SDL_CondSignal(f->cond);//当frame_queue_peek_readable在等待时，可唤醒
    SDL_UnlockMutex(f->mutex);
}

　　(l) frame_queue_next: update the read index, at this time the Frame is actually out of the queue, and the number of frames in the queue node is reduced by 1

　　　　When keep_last is enabled, if rindex_shown is 0, set it to 1 and return; at this time, the read index will not be updated, that is to say, the keep_last mechanism will actually occupy the size of the queue, which cannot be calculated into the size when calling frame_queue_nb_remaining to obtain the size; release the data corresponding to the Frame, but not release the Frame node itself; update the read index; release the wake-up signal to ring the thread waiting to write.

static void frame_queue_next(FrameQueue *f)
{
    if (f->keep_last && !f->rindex_shown) {
        f->rindex_shown = 1;
        return;
    }
    frame_queue_unref_item(&f->queue[f->rindex]);//删除frame
    if (++f->rindex == f->max_size)
        f->rindex = 0;
    SDL_LockMutex(f->mutex);
    f->size--;
    SDL_CondSignal(f->cond);
    SDL_UnlockMutex(f->mutex);
}

　　(m) Whether frame_queue_unref_item corresponds to the AVFrame and AVSubtitle
　　(o) frame_queue_last_pos: Get the position of the data corresponding to the recently played Frame in the media file, mainly used when seeking

Take a look at the write queue usage in ffplay through an example:

static int queue_picture(VideoState *is, AVFrame *src_frame, double pts, double duration, int64_t pos, int serial)
{
    Frame *vp;
 
#if defined(DEBUG_SYNC)
    printf("frame_type=%c pts=%0.3f\n",
           av_get_picture_type_char(src_frame->pict_type), pts);
#endif
 
    if (!(vp = frame_queue_peek_writable(&is->pictq)))//检测队列可写，获取可写Frame指针
        return -1;//队列已满
　　//对可写帧赋值
    vp->sar = src_frame->sample_aspect_ratio;
    vp->uploaded = 0;
 
    vp->width = src_frame->width;
    vp->height = src_frame->height;
    vp->format = src_frame->format;
 
    vp->pts = pts;
    vp->duration = duration;
    vp->pos = pos;
    vp->serial = serial;
　　
    set_default_window_size(vp->width, vp->height, vp->sar);
　　//将src_frame中所有数据拷贝到vp->frame
    av_frame_move_ref(vp->frame, src_frame);
    frame_queue_push(&is->pictq);//更新写索引位置
    return 0;
}

　　ffplay read queue usage:

//从队列读取一帧，并显示画面的过程<br><br>static void video_refresh(void *opaque, double *remaining_time){
    VideoState *is = opaque;
    double time;
 
    ......................
 
    if (is->video_st) {
retry:
        if (frame_queue_nb_remaining(&is->pictq) == 0) {// 帧队列是否为空
            // nothing to do, no picture to display in the queue
            // 队列中没有图像可显示
        } else {
            double last_duration, duration, delay;
            Frame *vp, *lastvp;
 
            // 从队列取出上一个Frame
            lastvp = frame_queue_peek_last(&is->pictq);//读取上一帧
            vp = frame_queue_peek(&is->pictq);  // 读取待显示帧
 
            if (vp->serial != is->videoq.serial) {
                // 如果不是最新的播放序列，则将其出队列，以尽快读取最新序列的帧
                frame_queue_next(&is->pictq);
                goto retry;
            }
 
            if (lastvp->serial != vp->serial) {
                // 新的播放序列重置当前时间
                is->frame_timer = av_gettime_relative() / 1000000.0;
            }
 
            if (is->paused)
            {
                goto display;
                printf("视频暂停is->paused");
            }
   
            //last_duration 计算上一帧应显示的时长
            last_duration = vp_duration(is, lastvp, vp);
 
            // 经过compute_target_delay方法，计算出待显示帧vp需要等待的时间
            // 如果以video同步，则delay直接等于last_duration。
            // 如果以audio或外部时钟同步，则需要比对主时钟调整待显示帧vp要等待的时间。
            delay = compute_target_delay(last_duration, is);
 
            time= av_gettime_relative()/1000000.0;
            // is->frame_timer 实际上就是上一帧lastvp的播放时间,
            // is->frame_timer + delay 是待显示帧vp该播放的时间
            if (time < is->frame_timer + delay) { //判断是否继续显示上一帧
                // 当前系统时刻还未到达上一帧的结束时刻，那么还应该继续显示上一帧。
                // 计算出最小等待时间
                *remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
                goto display;
            }
 
            // 走到这一步，说明已经到了或过了该显示的时间，待显示帧vp的状态变更为当前要显示的帧
 
            is->frame_timer += delay;   // 更新当前帧播放的时间
            if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX) {
                is->frame_timer = time; //如果和系统时间差距太大，就纠正为系统时间
            }
            SDL_LockMutex(is->pictq.mutex);
            if (!isnan(vp->pts))
                update_video_pts(is, vp->pts, vp->pos, vp->serial); // 更新video时钟
            SDL_UnlockMutex(is->pictq.mutex);
            //丢帧逻辑
            if (frame_queue_nb_remaining(&is->pictq) > 1) {//有nextvp才会检测是否该丢帧
                Frame *nextvp = frame_queue_peek_next(&is->pictq);
                duration = vp_duration(is, vp, nextvp);
                if(!is->step        // 非逐帧模式才检测是否需要丢帧 is->step==1 为逐帧播放
                        && (framedrop>0 ||      // cpu解帧过慢
                            (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) // 非视频同步方式
                        && time > is->frame_timer + duration // 确实落后了一帧数据
                        ) {
                    printf("%s(%d) dif:%lfs, drop frame\n", __FUNCTION__, __LINE__,
                           (is->frame_timer + duration) - time);
                    is->frame_drops_late++;             // 统计丢帧情况
                    frame_queue_next(&is->pictq);       // 这里实现真正的丢帧
                    //(这里不能直接while丢帧，因为很可能audio clock重新对时了，这样delay值需要重新计算)
                    goto retry; //回到函数开始位置，继续重试
                }
            }
 
           .........................
 
            frame_queue_next(&is->pictq);   // 当前vp帧出队列
            is->force_refresh = 1;          /* 说明需要刷新视频帧 */
 
            if (is->step && !is->paused)
                stream_toggle_pause(is);
        }
display:
        /* display picture */
        if (!display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)
            video_display(is); // 重点是显示
    }

　　(5) AudioParams audio parameters

typedef struct AudioParams {
    int freq;//采样率
    int channels;//通道数
    int64_t channel_layout;//通道布局，如：2.1声道，5.1声道
    enum AVSampleFormat fmt;//音频采样格式，如：AV_SAMPLE_FMT_S16
    int frame_size;//一个采样单元占用的字节数
    int bytes_per_sec;//一秒时间的字节数
} AudioParams;

　　(6) Decoder decoder package

typedef struct Decoder {
    AVPacket pkt;//packet缓存
    PacketQueue *queue;//packet队列，音频归音频、视频归视频的
    AVCodecContext *avctx;//解码器上下文
    int pkt_serial;//包序列
    int finished;//=0解码器处理工作状态，!=0处于空闲状态
    int packet_pending;//=0解码器处于异常状态，需要考虑重置解码器，=1解码器处于正常状态
    SDL_cond *empty_queue_cond;//检查到packet队列为空时发送signal缓存read_thread读取数据
    int64_t start_pts;//初始化时是stream的start_time
    AVRational start_pts_tb;//初始化时是stream的time_base
    int64_t next_pts;//记录最近一次解码后的frame的pts
    AVRational next_pts_tb;//next_pts的单位
    SDL_Thread *decoder_tid;//线程句柄
} Decoder;

Original address: https://www.cnblogs.com/juju-go/p/16489044.html

The benefits of this article, free C++ audio and video learning materials package, technical video/code, including (audio and video development, interview questions, FFmpeg, webRTC, rtmp, hls, rtsp, ffplay, codec, push-pull stream, srs) ↓↓↓↓↓↓See below↓↓Click at the bottom of the article to get it for free↓↓