Article directory
- 1. Video output module
-
- 1.1 Video output initialization
- 1.2 Video output logic
-
- 1.2.1 event_loop starts processing SDL events
- 1.2.2 video_refresh
-
- 1.2.2.1 Calculate the display duration of the previous frame and determine whether to continue the previous frame
- 1.2.2.2 Estimate the display duration of the current frame and determine whether to drop the frame
- 1.2.2.3 Call video_display for display
- 1.2.2.4 realloc_texture() reallocates vid_texture
- 1.2.2.5 sws_getCachedContext
- 1.2.2.6 sws_scale image conversion
- Recommended articles for flags algorithm testing:
1. Video output module
1.1 Video output initialization
1.1.1 Main process of video output initialization
- Initialize SDL, SDL_Init, mainly SDL_INIT_VIDEO support
- SDL_CreateWindow, creates the main window
- SDL_CreateRender, creates a renderer based on the main window for rendering output
- stream_open
- event_loop, the corresponding loop of the playback control event, but is also responsible for the display output of the video
int main(int argc, char **argv)
{
/* 是否显示视频 */
if (display_disable) {
video_disable = 1;
}
// 3. SDL的初始化
flags = SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER;
/* 是否运行音频 */
if (audio_disable)
flags &= ~SDL_INIT_AUDIO;
else {
/* Try to work around an occasional ALSA buffer underflow issue when the
* period size is NPOT due to ALSA resampling by forcing the buffer size. */
if (!SDL_getenv("SDL_AUDIO_ALSA_SET_BUFFER_SIZE"))
SDL_setenv("SDL_AUDIO_ALSA_SET_BUFFER_SIZE","1", 1);
}
if (display_disable)
flags &= ~SDL_INIT_VIDEO;
if (SDL_Init (flags)) {
av_log(NULL, AV_LOG_FATAL, "Could not initialize SDL - %s\n", SDL_GetError());
av_log(NULL, AV_LOG_FATAL, "(Did you set the DISPLAY variable?)\n");
exit(1);
}
SDL_EventState(SDL_SYSWMEVENT, SDL_IGNORE);
SDL_EventState(SDL_USEREVENT, SDL_IGNORE);
av_init_packet(&flush_pkt); // 初始化flush_packet
flush_pkt.data = (uint8_t *)&flush_pkt; // 初始化为数据指向自己本身
// 4. 创建窗口
if (!display_disable) {
int flags = SDL_WINDOW_HIDDEN;
if (alwaysontop)
#if SDL_VERSION_ATLEAST(2,0,5)
flags |= SDL_WINDOW_ALWAYS_ON_TOP;
#else
av_log(NULL, AV_LOG_WARNING, "Your SDL version doesn't support SDL_WINDOW_ALWAYS_ON_TOP. Feature will be inactive.\n");
#endif
if (borderless)
flags |= SDL_WINDOW_BORDERLESS;
else
flags |= SDL_WINDOW_RESIZABLE;
window = SDL_CreateWindow(program_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, default_width, default_height, flags);
SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
if (window) {
// 创建renderer
renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);
if (!renderer) {
av_log(NULL, AV_LOG_WARNING, "Failed to initialize a hardware accelerated renderer: %s\n", SDL_GetError());
renderer = SDL_CreateRenderer(window, -1, 0);
}
if (renderer) {
if (!SDL_GetRendererInfo(renderer, &renderer_info))
av_log(NULL, AV_LOG_VERBOSE, "Initialized %s renderer.\n", renderer_info.name);
}
}
if (!window || !renderer || !renderer_info.num_texture_formats) {
av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError());
do_exit(NULL);
}
}
// 5. 通过stream_open函数，开启read_thread读取线程
is = stream_open(input_filename, file_iformat);
if (!is) {
av_log(NULL, AV_LOG_FATAL, "Failed to initialize VideoState!\n");
do_exit(NULL);
}
// 6. 事件响应
event_loop(is);
/* never returns */
return 0;
}
//7 从待处理流中获取相关参数，设置显示窗口的宽度、高度及宽高比
if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) {
AVStream *st = ic->streams[st_index[AVMEDIA_TYPE_VIDEO]];
AVCodecParameters *codecpar = st->codecpar;
//根据流和帧宽高比猜测视频帧的像素宽高比（像素的宽高比，注意不是图像的）
AVRational sar = av_guess_sample_aspect_ratio(ic, st, NULL);
if (codecpar->width) {
// 设置显示窗口的大小和宽高比
set_default_window_size(codecpar->width, codecpar->height, sar);
}
}
Here we focus on the set_default_window_size function
static void set_default_window_size(int width, int height, AVRational sar)
{
SDL_Rect rect;
int max_width = screen_width ? screen_width : INT_MAX; // 确定是否指定窗口最大宽度
int max_height = screen_height ? screen_height : INT_MAX; // 确定是否指定窗口最大高度
if (max_width == INT_MAX && max_height == INT_MAX)
max_height = height; // 没有指定最大高度时则使用视频的高度
calculate_display_rect(&rect, 0, 0, max_width, max_height, width, height, sar);
default_width = rect.w; // 实际是渲染区域的宽高
default_height = rect.h;
}
screen_width and screen_height can be set using the command line when ffplay is started -x -y. If not specified, the height of the video frame will be used.
The key point is the calculate_display_rect function!
1.1.2 calculate_display_rect initializes the display window size
static void calculate_display_rect(SDL_Rect *rect,
int scr_xleft, int scr_ytop, int scr_width, int scr_height,
int pic_width, int pic_height, AVRational pic_sar)
{
AVRational aspect_ratio = pic_sar; // 比率
int64_t width, height, x, y;
if (av_cmp_q(aspect_ratio, av_make_q(0, 1)) <= 0)
aspect_ratio = av_make_q(1, 1);// 如果aspect_ratio是负数或者为0,设置为1:1
// 转成真正的播放比例
aspect_ratio = av_mul_q(aspect_ratio, av_make_q(pic_width, pic_height));
/* XXX: we suppose the screen has a 1.0 pixel ratio */
// 计算显示视频帧区域的宽高
// 先以高度为基准
height = scr_height;
// &~1, 取偶数宽度 1110
width = av_rescale(height, aspect_ratio.num, aspect_ratio.den) & ~1;
if (width > scr_width) {
// 当以高度为基准,发现计算出来的需要的窗口宽度不足时调整为以窗口宽度为基准
width = scr_width;
height = av_rescale(width, aspect_ratio.den, aspect_ratio.num) & ~1;
}
// 计算显示视频帧区域的起始坐标（在显示窗口内部的区域）
x = (scr_width - width) / 2;
y = (scr_height - height) / 2;
rect->x = scr_xleft + x;
rect->y = scr_ytop + y;
rect->w = FFMAX((int)width, 1);
rect->h = FFMAX((int)height, 1);
}
This function sets and calculates the width, height and position of the window.
The function first calculates the aspect ratio. If the aspect ratio is not set, the actual width and height are used to calculate the aspect ratio. Then the width is calculated using the av_rescale function based on the height. If the width is greater than scr_width, it is converted to the width.
Then we need to calculate the vertex coordinates, scr_width and scr_height window sizes, width and height are the video sizes, we need to calculate the upper left corner position of the video
Look at the picture:
The rect in the code is the rendered video part, which is the green part in the picture!
1.2 Video output logic
main()->
event_loop()->
refresh_loop_wait_event()->
video_refresh()->
video_display()->
video_image_display()->
upload_texture()
1.2.1 event_loop starts processing SDL events
static void event_loop(VideoState *cur_stream)
{
SDL_Event event;
double incr, pos, frac;
for (;;) {
double x;
refresh_loop_wait_event(cur_stream, &event); //video是在这里显示的
switch (event.type) {
case SDL_KEYDOWN: /* 键盘事件 */
if (exit_on_keydown || event.key.keysym.sym == SDLK_ESCAPE || event.key.keysym.sym == SDLK_q) {
do_exit(cur_stream);
break;
}
if (!cur_stream->width)
continue;
switch (event.key.keysym.sym) {
case SDLK_f:
toggle_full_screen(cur_stream);
cur_stream->force_refresh = 1;
break;
case SDLK_p:
case SDLK_SPACE: //按空格键触发暂停/恢复
toggle_pause(cur_stream);
break;
case SDLK_m:
toggle_mute(cur_stream);
break;
case SDLK_KP_MULTIPLY:
case SDLK_0:
update_volume(cur_stream, 1, SDL_VOLUME_STEP);
break;
case SDLK_KP_DIVIDE:
case SDLK_9:
update_volume(cur_stream, -1, SDL_VOLUME_STEP);
break;
case SDLK_s: // S: Step to next frame
step_to_next_frame(cur_stream);
break;
case SDLK_a:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
break;
case SDLK_v:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
break;
case SDLK_c:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
break;
case SDLK_t:
stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
break;
case SDLK_w:
#if CONFIG_AVFILTER
if (cur_stream->show_mode == SHOW_MODE_VIDEO && cur_stream->vfilter_idx < nb_vfilters - 1) {
if (++cur_stream->vfilter_idx >= nb_vfilters)
cur_stream->vfilter_idx = 0;
} else {
cur_stream->vfilter_idx = 0;
toggle_audio_display(cur_stream);
}
#else
toggle_audio_display(cur_stream);
#endif
break;
case SDLK_PAGEUP:
if (cur_stream->ic->nb_chapters <= 1) {
incr = 600.0;
goto do_seek;
}
seek_chapter(cur_stream, 1);
break;
case SDLK_PAGEDOWN:
if (cur_stream->ic->nb_chapters <= 1) {
incr = -600.0;
goto do_seek;
}
seek_chapter(cur_stream, -1);
break;
case SDLK_LEFT:
incr = seek_interval ? -seek_interval : -10.0;
goto do_seek;
case SDLK_RIGHT:
incr = seek_interval ? seek_interval : 10.0;
goto do_seek;
case SDLK_UP:
incr = 60.0;
goto do_seek;
case SDLK_DOWN:
incr = -60.0;
do_seek:
if (seek_by_bytes) {
pos = -1;
if (pos < 0 && cur_stream->video_stream >= 0)
pos = frame_queue_last_pos(&cur_stream->pictq);
if (pos < 0 && cur_stream->audio_stream >= 0)
pos = frame_queue_last_pos(&cur_stream->sampq);
if (pos < 0)
pos = avio_tell(cur_stream->ic->pb);
if (cur_stream->ic->bit_rate)
incr *= cur_stream->ic->bit_rate / 8.0;
else
incr *= 180000.0;
pos += incr;
stream_seek(cur_stream, pos, incr, 1);
} else {
pos = get_master_clock(cur_stream);
if (isnan(pos))
pos = (double)cur_stream->seek_pos / AV_TIME_BASE;
pos += incr; // 现在是秒的单位
if (cur_stream->ic->start_time != AV_NOPTS_VALUE && pos < cur_stream->ic->start_time / (double)AV_TIME_BASE)
pos = cur_stream->ic->start_time / (double)AV_TIME_BASE;
stream_seek(cur_stream, (int64_t)(pos * AV_TIME_BASE), (int64_t)(incr * AV_TIME_BASE), 0);
}
break;
default:
break;
}
break;
case SDL_MOUSEBUTTONDOWN: /* 鼠标按下事件 */
if (exit_on_mousedown) {
do_exit(cur_stream);
break;
}
if (event.button.button == SDL_BUTTON_LEFT) {
static int64_t last_mouse_left_click = 0;
if (av_gettime_relative() - last_mouse_left_click <= 500000) {
//连续鼠标左键点击2次显示窗口间隔小于0.5秒，则进行全屏或者恢复原始窗口
toggle_full_screen(cur_stream);
cur_stream->force_refresh = 1;
last_mouse_left_click = 0;
} else {
last_mouse_left_click = av_gettime_relative();
}
}
case SDL_MOUSEMOTION: /* 鼠标移动事件 */
if (cursor_hidden) {
SDL_ShowCursor(1);
cursor_hidden = 0;
}
cursor_last_shown = av_gettime_relative();
if (event.type == SDL_MOUSEBUTTONDOWN) {
if (event.button.button != SDL_BUTTON_RIGHT)
break;
x = event.button.x;
} else {
if (!(event.motion.state & SDL_BUTTON_RMASK))
break;
x = event.motion.x;
}
if (seek_by_bytes || cur_stream->ic->duration <= 0) {
uint64_t size = avio_size(cur_stream->ic->pb); // 整个文件的字节
stream_seek(cur_stream, size*x/cur_stream->width, 0, 1);
} else {
int64_t ts;
int ns, hh, mm, ss;
int tns, thh, tmm, tss;
tns = cur_stream->ic->duration / 1000000LL;
thh = tns / 3600;
tmm = (tns % 3600) / 60;
tss = (tns % 60);
frac = x / cur_stream->width;
ns = frac * tns;
hh = ns / 3600;
mm = (ns % 3600) / 60;
ss = (ns % 60);
av_log(NULL, AV_LOG_INFO,
"Seek to %2.0f%% (%2d:%02d:%02d) of total duration (%2d:%02d:%02d) \n", frac*100,
hh, mm, ss, thh, tmm, tss);
ts = frac * cur_stream->ic->duration;
if (cur_stream->ic->start_time != AV_NOPTS_VALUE)
ts += cur_stream->ic->start_time;
stream_seek(cur_stream, ts, 0, 0);
}
break;
case SDL_WINDOWEVENT: /* 窗口事件 */
switch (event.window.event) {
case SDL_WINDOWEVENT_SIZE_CHANGED:
screen_width = cur_stream->width = event.window.data1;
screen_height = cur_stream->height = event.window.data2;
if (cur_stream->vis_texture) {
SDL_DestroyTexture(cur_stream->vis_texture);
cur_stream->vis_texture = NULL;
}
case SDL_WINDOWEVENT_EXPOSED:
cur_stream->force_refresh = 1;
}
break;
case SDL_QUIT:
case FF_QUIT_EVENT: /* ffplay自定义事件,用于主动退出 */
do_exit(cur_stream);
break;
default:
break;
}
}
}
This function mainly waits for events through the refresh_loop_wait_event function, and then event_loop processes the events.
The display of video is mainly in refresh_loop_wait_event:
static void refresh_loop_wait_event(VideoState *is, SDL_Event *event) {
double remaining_time = 0.0; /* 休眠等待，remaining_time的计算在video_refresh中 */
/* 调用SDL_PeepEvents前先调用SDL_PumpEvents，将输入设备的事件抽到事件队列中 */
SDL_PumpEvents();
/*
* SDL_PeepEvents check是否事件，比如鼠标移入显示区等
* 从事件队列中拿一个事件，放到event中，如果没有事件，则进入循环中
* SDL_PeekEvents用于读取事件，在调用该函数之前，必须调用SDL_PumpEvents搜集键盘等事件
*/
while (!SDL_PeepEvents(event, 1, SDL_GETEVENT, SDL_FIRSTEVENT, SDL_LASTEVENT)) {
if (!cursor_hidden && av_gettime_relative() - cursor_last_shown > CURSOR_HIDE_DELAY) {
SDL_ShowCursor(0);
cursor_hidden = 1;
}
/*
* remaining_time就是用来进行音视频同步的。
* 在video_refresh函数中，根据当前帧显示时刻(display time)和实际时刻(actual time)
* 计算需要sleep的时间，保证帧按时显示
*/
if (remaining_time > 0.0) //sleep控制画面输出的时机
av_usleep((int64_t)(remaining_time * 1000000.0)); // remaining_time <= REFRESH_RATE
remaining_time = REFRESH_RATE;
if (is->show_mode != SHOW_MODE_NONE && // 显示模式不等于SHOW_MODE_NONE
(!is->paused // 非暂停状态
|| is->force_refresh) // 强制刷新状态
) {
video_refresh(is, &remaining_time);
}
/* 从输入设备中搜集事件，推动这些事件进入事件队列，更新事件队列的状态，
* 不过它还有一个作用是进行视频子系统的设备状态更新，如果不调用这个函数，
* 所显示的视频会在大约10秒后丢失色彩。没有调用SDL_PumpEvents，将不会
* 有任何的输入设备事件进入队列，这种情况下，SDL就无法响应任何的键盘等硬件输入。
*/
SDL_PumpEvents();
}
}
SDL_PeepEvent non-blockingly queries whether there are events in the queue through SDL_GETEVENT. If it is not 0, an event occurs (-1 means an error occurs), then the function will return and let event_loop handle it; otherwise, call video_refresh to display the picture and pass the output parameters remaining_time gets the time that should sleep in the next round to ensure stable screen output.
The preconditions for whether to call video_refresh are:
- The display mode is not SHOW_MODE_NONE (if it only contains audio pictures, it may also be its waveform diagram)
- Not currently suspended
- When force_refresh (forced refresh) is set, analyze the situation:
- Frame display in video_refresh is normal.
- SDL_WINDOWEVENT_EXPOSED, the window needs to be redrawn
- SDL_MOUSEBUTTONDOWN && SDL_BUTTON_LEFT The mouse is pressed and the continuous interval between pressing the left button is less than 0.5s
- SDLK_f, press f key to go to full screen or restore the original playback window
You may not understand what this forced refresh is. Take a look at the rendering without forced refresh:
It can be seen that the following part is incomplete because we changed the window size, but the page we rendered has not changed. At this time, we can refresh the page again and let the rendered page change according to the window size.
1.2.2 video_refresh
static void video_refresh(void *opaque, double *remaining_time)
{
VideoState *is = opaque;
double time;
Frame *sp, *sp2;
if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)
check_external_clock_speed(is);
if (!display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {
time = av_gettime_relative() / 1000000.0;
if (is->force_refresh || is->last_vis_time + rdftspeed < time) {
video_display(is);
is->last_vis_time = time;
}
*remaining_time = FFMIN(*remaining_time, is->last_vis_time + rdftspeed - time);
}
if (is->video_st) {
retry:
if (frame_queue_nb_remaining(&is->pictq) == 0) {
// 帧队列是否为空
// nothing to do, no picture to display in the queue
// 什么都不做，队列中没有图像可显示
} else {
// 重点是音视频同步
double last_duration, duration, delay;
Frame *vp, *lastvp;
/* dequeue the picture */
// 从队列取出上一个Frame
lastvp = frame_queue_peek_last(&is->pictq);//读取上一帧
vp = frame_queue_peek(&is->pictq); // 读取待显示帧
// lastvp 上一帧(正在显示的帧)
// vp 等待显示的帧
if (vp->serial != is->videoq.serial) {
// 如果不是最新的播放序列，则将其出队列，以尽快读取最新序列的帧
frame_queue_next(&is->pictq);
goto retry;
}
if (lastvp->serial != vp->serial) {
// 新的播放序列重置当前时间
is->frame_timer = av_gettime_relative() / 1000000.0;
}
if (is->paused)
{
goto display;
printf("视频暂停is->paused");
}
/* compute nominal last_duration */
//lastvp上一帧，vp当前帧 ，nextvp下一帧
//last_duration 计算上一帧应显示的时长
last_duration = vp_duration(is, lastvp, vp);
// 经过compute_target_delay方法，计算出待显示帧vp需要等待的时间
// 如果以video同步，则delay直接等于last_duration。
// 如果以audio或外部时钟同步，则需要比对主时钟调整待显示帧vp要等待的时间。
delay = compute_target_delay(last_duration, is); // 上一帧需要维持的时间
time= av_gettime_relative()/1000000.0;
// is->frame_timer 实际上就是上一帧lastvp的播放时间,
// is->frame_timer + delay 是待显示帧vp该播放的时间
if (time < is->frame_timer + delay) {
//判断是否继续显示上一帧
// 当前系统时刻还未到达上一帧的结束时刻，那么还应该继续显示上一帧。
// 计算出最小等待时间
*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
goto display;
}
// 走到这一步，说明已经到了或过了该显示的时间，待显示帧vp的状态变更为当前要显示的帧
is->frame_timer += delay; // 更新当前帧播放的时间
if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX) {
is->frame_timer = time; //如果和系统时间差距太大，就纠正为系统时间
}
SDL_LockMutex(is->pictq.mutex);
if (!isnan(vp->pts))
update_video_pts(is, vp->pts, vp->pos, vp->serial); // 更新video时钟
SDL_UnlockMutex(is->pictq.mutex);
//丢帧逻辑
if (frame_queue_nb_remaining(&is->pictq) > 1) {
//有nextvp才会检测是否该丢帧
Frame *nextvp = frame_queue_peek_next(&is->pictq);
duration = vp_duration(is, vp, nextvp);
if(!is->step // 非逐帧模式才检测是否需要丢帧 is->step==1 为逐帧播放
&& (framedrop>0 || // cpu解帧过慢
(framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) // 非视频同步方式
&& time > is->frame_timer + duration // 确实落后了一帧数据
) {
printf("%s(%d) dif:%lfs, drop frame\n", __FUNCTION__, __LINE__,
(is->frame_timer + duration) - time);
is->frame_drops_late++; // 统计丢帧情况
frame_queue_next(&is->pictq); // 这里实现真正的丢帧
//(这里不能直接while丢帧，因为很可能audio clock重新对时了，这样delay值需要重新计算)
goto retry; //回到函数开始位置，继续重试
}
}
if (is->subtitle_st) {
while (frame_queue_nb_remaining(&is->subpq) > 0) {
sp = frame_queue_peek(&is->subpq);
if (frame_queue_nb_remaining(&is->subpq) > 1)
sp2 = frame_queue_peek_next(&is->subpq);
else
sp2 = NULL;
if (sp->serial != is->subtitleq.serial
|| (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))
|| (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000))))
{
if (sp->uploaded) {
int i;
for (i = 0; i < sp->sub.num_rects; i++) {
AVSubtitleRect *sub_rect = sp->sub.rects[i];
uint8_t *pixels;
int pitch, j;
if (!SDL_LockTexture(is->sub_texture, (SDL_Rect *)sub_rect, (void **)&pixels, &pitch)) {
for (j = 0; j < sub_rect->h; j++, pixels += pitch)
memset(pixels, 0, sub_rect->w << 2);
SDL_UnlockTexture(is->sub_texture);
}
}
}
frame_queue_next(&is->subpq);
} else {
break;
}
}
}
frame_queue_next(&is->pictq); // 当前vp帧出队列
is->force_refresh = 1; /* 说明需要刷新视频帧 */
if (is->step && !is->paused)
stream_toggle_pause(is); // 逐帧的时候那继续进入暂停状态
}
display:
/* display picture */
if (!display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)
video_display(is); // 重点是显示
}
is->force_refresh = 0;
if (show_status) {
static int64_t last_time;
int64_t cur_time;
int aqsize, vqsize, sqsize;
double av_diff;
cur_time = av_gettime_relative();
if (!last_time || (cur_time - last_time) >= 30000) {
aqsize = 0;
vqsize = 0;
sqsize = 0;
if (is->audio_st)
aqsize = is->audioq.size;
if (is->video_st)
vqsize = is->videoq.size;
if (is->subtitle_st)
sqsize = is->subtitleq.size;
av_diff = 0;
if (is->audio_st && is->video_st)
av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);
else if (is->video_st)
av_diff = get_master_clock(is) - get_clock(&is->vidclk);
else if (is->audio_st)
av_diff = get_master_clock(is) - get_clock(&is->audclk);
av_log(NULL, AV_LOG_INFO,
"%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64" \r",
get_master_clock(is),
(is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : " ")),
av_diff,
is->frame_drops_early + is->frame_drops_late,
aqsize / 1024,
vqsize / 1024,
sqsize,
is->video_st ? is->viddec.avctx->pts_correction_num_faulty_dts : 0,
is->video_st ? is->viddec.avctx->pts_correction_num_faulty_pts : 0);
fflush(stdout);
last_time = cur_time;
}
}
}
flow chart:
Look at the main flow chart:
- Get the previous frame and the frame to be displayed
- Calculate the display duration of the previous frame and determine whether to continue the previous frame
- Estimate the display duration of the current frame and determine whether to drop frames
- Call video_display to display
1.2.2.1 Calculate the display duration of the previous frame and determine whether to continue the previous frame
First determine whether pictq is empty (call frame_queue_nb_remaining to determine whether there are undisplayed frames). If it is empty, continue to call video_display to display the previous frame.
Before further calculating the display time of the previous frame, it is necessary to determine whether the vp of the next frame is the latest sequence, that is, if (vp->serial!= is->videoq.serial), if the condition is true, seek and other operations have occurred. , lastvp should be discarded at this time. Therefore, after calling frame_queue_next to discard lastvp, return to the beginning of the process and try again.
Next, you can calculate the display duration of lastvp. The calculation code is:
last_duration = vp_duration(is, lastvp, vp);
delay = compute_target_delay(last_duration, is);//返回当前显示帧要持续播放的时间
The essence is calculated through the pts of the previous frame and the frame to be displayed. If synchronization is taken into account, the gap between the current and the main clock needs to be considered to decide whether to repeat the previous frame, lose the frame, or display the next frame normally (to be displayed). display frame)
time= av_gettime_relative()/1000000.0;
// is->frame_timer 实际上就是上一帧lastvp的播放时间,
// is->frame_timer + delay 是待显示帧vp该播放的时间
if (time < is->frame_timer + delay) {
//判断是否继续显示上一帧
// 当前系统时刻还未到达上一帧的结束时刻，那么还应该继续显示上一帧。
// 计算出最小等待时间
*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
goto display;
}
frame_timer is the current frame display time. If the current frame display time + delay display time is greater than the current system time, the previous frame will continue to be displayed!
1.2.2.2 Estimate the display duration of the current frame and determine whether to drop the frame
is->frame_timer += delay; // 更新当前帧播放的时间
if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX) {
is->frame_timer = time; //如果和系统时间差距太大，就纠正为系统时间
}
Determine whether frames need to be dropped
if (frame_queue_nb_remaining(&is->pictq) > 1) {
//有nextvp才会检测是否该丢帧
Frame *nextvp = frame_queue_peek_next(&is->pictq);
duration = vp_duration(is, vp, nextvp);
if(!is->step // 非逐帧模式才检测是否需要丢帧 is->step==1 为逐帧播放
&& (framedrop>0 || // cpu解帧过慢
(framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) // 非视频同步方式
&& time > is->frame_timer + duration // 确实落后了一帧数据
) {
printf("%s(%d) dif:%lfs, drop frame\n", __FUNCTION__, __LINE__,
(is->frame_timer + duration) - time);
is->frame_drops_late++; // 统计丢帧情况
frame_queue_next(&is->pictq); // 这里实现真正的丢帧
//(这里不能直接while丢帧，因为很可能audio clock重新对时了，这样delay值需要重新计算)
goto retry; //回到函数开始位置，继续重试
}
}
Calculate the display time of the current frame through the frame to be displayed and the next frame (provided there is a next frame)
And frames will be dropped only if the following conditions are met:
- Not in step state, frame by frame state
- Enable framedrop mode, which means frames need to be dropped when the CPU is too slow and the video is not used as the synchronization clock.
- The current time is >frame_timer+duration
1.2.2.3 Call video_display for display
static void video_display(VideoState *is)
{
if (!is->width)
video_open(is); //如果窗口未显示，则显示窗口
SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
SDL_RenderClear(renderer);
if (is->audio_st && is->show_mode != SHOW_MODE_VIDEO)
video_audio_display(is); //图形化显示仅有音轨的文件
else if (is->video_st)
video_image_display(is); //显示一帧视频画面
SDL_RenderPresent(renderer);
}
static void video_image_display(VideoState *is)
{
Frame *vp;
Frame *sp = NULL;
SDL_Rect rect;
// keep_last的作用就出来了,我们是有调用frame_queue_next, 但最近出队列的帧并没有真正销毁
// 所以这里可以读取出来显示
vp = frame_queue_peek_last(&is->pictq); //
if (is->subtitle_st) {
if (frame_queue_nb_remaining(&is->subpq) > 0) {
sp = frame_queue_peek(&is->subpq);
if (vp->pts >= sp->pts + ((float) sp->sub.start_display_time / 1000)) {
if (!sp->uploaded) {
uint8_t* pixels[4];
int pitch[4];
int i;
if (!sp->width || !sp->height) {
sp->width = vp->width;
sp->height = vp->height;
}
if (realloc_texture(&is->sub_texture, SDL_PIXELFORMAT_ARGB8888, sp->width, sp->height, SDL_BLENDMODE_BLEND, 1) < 0)
return;
for (i = 0; i < sp->sub.num_rects; i++) {
AVSubtitleRect *sub_rect = sp->sub.rects[i];
sub_rect->x = av_clip(sub_rect->x, 0, sp->width );
sub_rect->y = av_clip(sub_rect->y, 0, sp->height);
sub_rect->w = av_clip(sub_rect->w, 0, sp->width - sub_rect->x);
sub_rect->h = av_clip(sub_rect->h, 0, sp->height - sub_rect->y);
is->sub_convert_ctx = sws_getCachedContext(is->sub_convert_ctx,
sub_rect->w, sub_rect->h, AV_PIX_FMT_PAL8,
sub_rect->w, sub_rect->h, AV_PIX_FMT_BGRA,
0, NULL, NULL, NULL);
if (!is->sub_convert_ctx) {
av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");
return;
}
if (!SDL_LockTexture(is->sub_texture, (SDL_Rect *)sub_rect, (void **)pixels, pitch)) {
sws_scale(is->sub_convert_ctx, (const uint8_t * const *)sub_rect->data, sub_rect->linesize,
0, sub_rect->h, pixels, pitch);
SDL_UnlockTexture(is->sub_texture);
}
}
sp->uploaded = 1;
}
} else
sp = NULL;
}
}
//将帧宽高按照sar最大适配到窗口，并通过rect返回视频帧在窗口的显示位置和宽高
calculate_display_rect(&rect, is->xleft, is->ytop, is->width, is->height,
vp->width, vp->height, vp->sar);
// rect.x = rect.w /2; // 测试
// rect.w = rect.w /2; // 缩放实际不是用sws， 缩放是sdl去做的
if (!vp->uploaded) {
// 把yuv数据更新到vid_texture
if (upload_texture(&is->vid_texture, vp->frame, &is->img_convert_ctx) < 0)
return;
vp->uploaded = 1;
vp->flip_v = vp->frame->linesize[0] < 0;
}
set_sdl_yuv_conversion_mode(vp->frame);
SDL_RenderCopyEx(renderer, is->vid_texture, NULL, &rect, 0, NULL, vp->flip_v ? SDL_FLIP_VERTICAL : 0);
set_sdl_yuv_conversion_mode(NULL);
if (sp) {
#if USE_ONEPASS_SUBTITLE_RENDER
SDL_RenderCopy(renderer, is->sub_texture, NULL, &rect);
#else
int i;
double xratio = (double)rect.w / (double)sp->width;
double yratio = (double)rect.h / (double)sp->height;
for (i = 0; i < sp->sub.num_rects; i++) {
SDL_Rect *sub_rect = (SDL_Rect*)sp->sub.rects[i];
SDL_Rect target = {
.x = rect.x + sub_rect->x * xratio,
.y = rect.y + sub_rect->y * yratio,
.w = sub_rect->w * xratio,
.h = sub_rect->h * yratio};
SDL_RenderCopy(renderer, is->sub_texture, sub_rect, &target);
}
#endif
}
}
video_image_display is not complex overall. Calculate_display_rect is called every time it is rendered to recalculate the display window, etc.
The most important display is to call upload_texture to pass the AVFrame image data to the sdl texture for rendering:
static int upload_texture(SDL_Texture **tex, AVFrame *frame, struct SwsContext **img_convert_ctx) {
int ret = 0;
Uint32 sdl_pix_fmt;
SDL_BlendMode sdl_blendmode;
// 根据frame中的图像格式(FFmpeg像素格式)，获取对应的SDL像素格式和blendmode
get_sdl_pix_fmt_and_blendmode(frame->format, &sdl_pix_fmt, &sdl_blendmode);
// 参数tex实际是&is->vid_texture，此处根据得到的SDL像素格式，为&is->vid_texture
if (realloc_texture(tex, sdl_pix_fmt == SDL_PIXELFORMAT_UNKNOWN ? SDL_PIXELFORMAT_ARGB8888 : sdl_pix_fmt,
frame->width, frame->height, sdl_blendmode, 0) < 0)
return -1;
//根据sdl_pix_fmt从AVFrame中取数据填充纹理
switch (sdl_pix_fmt) {
// frame格式是SDL不支持的格式，则需要进行图像格式转换，转换为目标格式AV_PIX_FMT_BGRA，
// 对应SDL_PIXELFORMAT_BGRA32
case SDL_PIXELFORMAT_UNKNOWN:
/* This should only happen if we are not using avfilter... */
*img_convert_ctx = sws_getCachedContext(*img_convert_ctx,
frame->width, frame->height, frame->format,
frame->width, frame->height, AV_PIX_FMT_BGRA,
sws_flags, NULL, NULL, NULL);
if (*img_convert_ctx != NULL) {
uint8_t *pixels[4]; // 之前取Texture的缓存
int pitch[4];
if (!SDL_LockTexture(*tex, NULL, (void **)pixels, pitch)) {
sws_scale(*img_convert_ctx, (const uint8_t * const *)frame->data, frame->linesize,
0, frame->height, pixels, pitch);
SDL_UnlockTexture(*tex);
}
} else {
av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");
ret = -1;
}
break;
// frame格式对应SDL_PIXELFORMAT_IYUV，不用进行图像格式转换，调用SDL_UpdateYUVTexture()更新SDL texture
case SDL_PIXELFORMAT_IYUV:
if (frame->linesize[0] > 0 && frame->linesize[1] > 0 && frame->linesize[2] > 0) {
ret = SDL_UpdateYUVTexture(*tex, NULL, frame->data[0], frame->linesize[0],
frame->data[1], frame->linesize[1],
frame->data[2], frame->linesize[2]);
} else if (frame->linesize[0] < 0 && frame->linesize[1] < 0 && frame->linesize[2] < 0) {
ret = SDL_UpdateYUVTexture(*tex, NULL, frame->data[0] + frame->linesize[0] * (frame->height - 1), -frame->linesize[0],
frame->data[1] + frame->linesize[1] * (AV_CEIL_RSHIFT(frame->height, 1) - 1), -frame->linesize[1],
frame->data[2] + frame->linesize[2] * (AV_CEIL_RSHIFT(frame->height, 1) - 1), -frame->linesize[2]);
} else {
av_log(NULL, AV_LOG_ERROR, "Mixed negative and positive linesizes are not supported.\n");
return -1;
}
break;
// frame格式对应其他SDL像素格式，不用进行图像格式转换，调用SDL_UpdateTexture()更新SDL texture
default:
if (frame->linesize[0] < 0) {
ret = SDL_UpdateTexture(*tex, NULL, frame->data[0] + frame->linesize[0] * (frame->height - 1), -frame->linesize[0]);
} else {
ret = SDL_UpdateTexture(*tex, NULL, frame->data[0], frame->linesize[0]);
}
break;
}
return ret;
}
The pixel format in frame is the pixel format defined in FFmpeg. Many pixel formats defined in FFmpeg are the same format as the pixel format defined in SDL, but the names are different.
According to the match between the pixel format in the frame and the pixel format of SDL, upload_texture() processes three types, corresponding to the three branches of the Switch statement:
- If the frame image format corresponds to the SDL_PIXELFORMAT_IYUV format, no image format conversion is required. Use SDL_updateYUVBTexture() to update the data to &is->vid_texture.
- If the frame image format corresponds to other formats supported by SDL, there is no need to convert the image format. Use SDL_updateTexture() to update the data to &is->vid_texture.
- If the frame image is not supported by SDL, image format conversion is required.
Get the pixel format in SDL corresponding to the frame according to the mapping table:
static void get_sdl_pix_fmt_and_blendmode(int format, Uint32 *sdl_pix_fmt, SDL_BlendMode *sdl_blendmode)
{
int i;
*sdl_blendmode = SDL_BLENDMODE_NONE;
*sdl_pix_fmt = SDL_PIXELFORMAT_UNKNOWN;
if (format == AV_PIX_FMT_RGB32 ||
format == AV_PIX_FMT_RGB32_1 ||
format == AV_PIX_FMT_BGR32 ||
format == AV_PIX_FMT_BGR32_1)
*sdl_blendmode = SDL_BLENDMODE_BLEND;
for (i = 0; i < FF_ARRAY_ELEMS(sdl_texture_format_map) - 1; i++) {
if (format == sdl_texture_format_map[i].format) {
*sdl_pix_fmt = sdl_texture_format_map[i].texture_fmt;
return;
}
}
}
Mapping table:
static const struct TextureFormatEntry {
enum AVPixelFormat format;
int texture_fmt;
}
sdl_texture_format_map[] = {
// FFmpeg PIX_FMT to SDL_PIX的映射关系
{
AV_PIX_FMT_RGB8, SDL_PIXELFORMAT_RGB332 },
{
AV_PIX_FMT_RGB444, SDL_PIXELFORMAT_RGB444 },
{
AV_PIX_FMT_RGB555, SDL_PIXELFORMAT_RGB555 },
{
AV_PIX_FMT_BGR555, SDL_PIXELFORMAT_BGR555 },
{
AV_PIX_FMT_RGB565, SDL_PIXELFORMAT_RGB565 },
{
AV_PIX_FMT_BGR565, SDL_PIXELFORMAT_BGR565 },
{
AV_PIX_FMT_RGB24, SDL_PIXELFORMAT_RGB24 },
{
AV_PIX_FMT_BGR24, SDL_PIXELFORMAT_BGR24 },
{
AV_PIX_FMT_0RGB32, SDL_PIXELFORMAT_RGB888 },
{
AV_PIX_FMT_0BGR32, SDL_PIXELFORMAT_BGR888 },
{
AV_PIX_FMT_NE(RGB0, 0BGR), SDL_PIXELFORMAT_RGBX8888 },
{
AV_PIX_FMT_NE(BGR0, 0RGB), SDL_PIXELFORMAT_BGRX8888 },
{
AV_PIX_FMT_RGB32, SDL_PIXELFORMAT_ARGB8888 },
{
AV_PIX_FMT_RGB32_1, SDL_PIXELFORMAT_RGBA8888 },
{
AV_PIX_FMT_BGR32, SDL_PIXELFORMAT_ABGR8888 },
{
AV_PIX_FMT_BGR32_1, SDL_PIXELFORMAT_BGRA8888 },
{
AV_PIX_FMT_YUV420P, SDL_PIXELFORMAT_IYUV },
{
AV_PIX_FMT_YUYV422, SDL_PIXELFORMAT_YUY2 },
{
AV_PIX_FMT_UYVY422, SDL_PIXELFORMAT_UYVY },
{
AV_PIX_FMT_NONE, SDL_PIXELFORMAT_UNKNOWN },
};
As you can see, except for the last item, images in other formats can be displayed directly when sent to SDL without image conversion.
1.2.2.4 realloc_texture() reallocates vid_texture
static int realloc_texture(SDL_Texture **texture, Uint32 new_format, int new_width, int new_height,
SDL_BlendMode blendmode, int init_texture)
{
Uint32 format;
int access, w, h;
if (!*texture || SDL_QueryTexture(*texture, &format, &access, &w, &h) < 0 || new_width != w || new_height != h || new_format != format) {
void *pixels;
int pitch;
if (*texture)
SDL_DestroyTexture(*texture);
if (!(*texture = SDL_CreateTexture(renderer, new_format, SDL_TEXTUREACCESS_STREAMING, new_width, new_height)))
return -1;
if (SDL_SetTextureBlendMode(*texture, blendmode) < 0)
return -1;
if (init_texture) {
if (SDL_LockTexture(*texture, NULL, &pixels, &pitch) < 0)
return -1;
memset(pixels, 0, pitch * new_height);
SDL_UnlockTexture(*texture);
}
av_log(NULL, AV_LOG_VERBOSE, "Created %dx%d texture with %s.\n", new_width, new_height, SDL_GetPixelFormatName(new_format));
}
return 0;
}
Under what circumstances is realloc_texture needed?
- The texture used for display is not allocated
- SDL_QueryTexture is invalid
- Currently, the width, height, and format of the texture are inconsistent with the Frame displayed by the new drug.
To sum up: the window size change is not enough for realloc_texture to re-SDL_CreateTextue
1.2.2.5 sws_getCachedContext
struct SwsContext *sws_getCachedContext(struct SwsContext *context,
int srcW, int srcH, enum AVPixelFormat srcFormat,
int dstW, int dstH, enum AVPixelFormat dstFormat,
int flags, SwsFilter *srcFilter,
SwsFilter *dstFilter, const double *param);
Create an image transformation context
The flags parameter that needs to be explained here is used to select the conversion algorithm. There are many conversion algorithms, which are in the libswscale/swscale.h file.
1.2.2.6 sws_scale image conversion
int sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[],
const int srcStride[], int srcSliceY, int srcSliceH,
uint8_t *const dst[], const int dstStride[]);
if (!SDL_LockTexture(*tex, NULL, (void **)pixels, pitch)) {
sws_scale(*img_convert_ctx, (const uint8_t * const *)frame->data, frame->linesize,
0, frame->height, pixels, pitch);
SDL_UnlockTexture(*tex);
}