前言
最近ffmpeg3.x版本命令行解码后出现drop或者dup也就是丢帧和多帧的现象，追踪了一下发现是输入视频pts和dts有问题。
有个专业属于是：插帧算法
因为实际视频中，有变帧率视频
通过追踪代码发现是ffmpeg.c中的do_video_out()函数中
static void do_video_out(OutputFile *of,
OutputStream *ost,
AVFrame *next_picture)
{
int ret, format_video_sync;
AVPacket *pkt = ost->pkt;
AVCodecContext *enc = ost->enc_ctx;
AVRational frame_rate;
int nb_frames, nb0_frames, i;
double delta, delta0;
double duration = 0;
double sync_ipts = AV_NOPTS_VALUE;
int frame_size = 0;
InputStream *ist = NULL;
AVFilterContext *filter = ost->filter->filter;
init_output_stream_wrapper(ost, next_picture, 1);
//如果输入视频pts和dts有问题，就会导致这里的sync_ipts 出现问题，正常情况下这里是正值，如果pts出现问题，就会导致这里出现负值
sync_ipts = adjust_frame_pts_to_encoder_tb(of, ost, next_picture);
if (ost->source_index >= 0)
ist = input_streams[ost->source_index];
frame_rate = av_buffersink_get_frame_rate(filter);
if (frame_rate.num > 0 && frame_rate.den > 0)
duration = 1/(av_q2d(frame_rate) * av_q2d(enc->time_base));
if(ist && ist->st->start_time != AV_NOPTS_VALUE && ist->st->first_dts != AV_NOPTS_VALUE && ost->frame_rate.num)
duration = FFMIN(duration, 1/(av_q2d(ost->frame_rate) * av_q2d(enc->time_base)));
if (!ost->filters_script &&
!ost->filters &&
(nb_filtergraphs == 0 || !filtergraphs[0]->graph_desc) &&
next_picture &&
ist &&
lrintf(next_picture->pkt_duration * av_q2d(ist->st->time_base) / av_q2d(enc->time_base)) > 0) {
duration = lrintf(next_picture->pkt_duration * av_q2d(ist->st->time_base) / av_q2d(enc->time_base));
}
if (!next_picture) {
//end, flushing
nb0_frames = nb_frames = mid_pred(ost->last_nb0_frames[0],
ost->last_nb0_frames[1],
ost->last_nb0_frames[2]);
} else {
//这里很重要 delta0 -3.0000076293945312 = -7.62939453125e-06-3
delta0 = sync_ipts - ost->sync_opts; // delta0 is the "drift" between the input frame (next_picture) and where it would fall in the output.
delta = delta0 + duration;
//丢帧要进入到这里
/* by default, we output a single frame */
nb0_frames = 0; // tracks the number of times the PREVIOUS frame should be duplicated, mostly for variable framerate (VFR)
nb_frames = 1;
format_video_sync = video_sync_method;
if (format_video_sync == VSYNC_AUTO) {
if(!strcmp(of->ctx->oformat->name, "avi")) {
format_video_sync = VSYNC_VFR;
} else
//进入到这里
format_video_sync = (of->ctx->oformat->flags & AVFMT_VARIABLE_FPS) ? ((of->ctx->oformat->flags & AVFMT_NOTIMESTAMPS) ? VSYNC_PASSTHROUGH : VSYNC_VFR) : VSYNC_CFR;
if ( ist
&& format_video_sync == VSYNC_CFR
&& input_files[ist->file_index]->ctx->nb_streams == 1
&& input_files[ist->file_index]->input_ts_offset == 0) {
format_video_sync = VSYNC_VSCFR;
}
if (format_video_sync == VSYNC_CFR && copy_ts) {
format_video_sync = VSYNC_VSCFR;
}
}
//到这里 值为1
ost->is_cfr = (format_video_sync == VSYNC_CFR || format_video_sync == VSYNC_VSCFR);
if (delta0 < 0 &&
delta > 0 &&
format_video_sync != VSYNC_PASSTHROUGH &&
format_video_sync != VSYNC_DROP) {
if (delta0 < -0.6) {
av_log(NULL, AV_LOG_VERBOSE, "Past duration %f too large\n", -delta0);
} else
av_log(NULL, AV_LOG_DEBUG, "Clipping frame in rate conversion by %f\n", -delta0);
sync_ipts = ost->sync_opts;
duration += delta0;
delta0 = 0;
}
switch (format_video_sync) {
case VSYNC_VSCFR:
if (ost->frame_number == 0 && delta0 >= 0.5) {
av_log(NULL, AV_LOG_DEBUG, "Not duplicating %d initial frames\n", (int)lrintf(delta0));
delta = duration;
delta0 = 0;
ost->sync_opts = llrint(sync_ipts);
}
case VSYNC_CFR:
// FIXME set to 0.5 after we fix some dts/pts bugs like in avidec.c
if (frame_drop_threshold && delta < frame_drop_threshold && ost->frame_number) {
nb_frames = 0;
} else if (delta < -1.1)
nb_frames = 0;//进入到这里 delta=-2.00000007685956
else if (delta > 1.1) {
nb_frames = lrintf(delta);
if (delta0 > 1.1)
nb0_frames = llrintf(delta0 - 0.6);
}
break;
case VSYNC_VFR:
if (delta <= -0.6)
nb_frames = 0;
else if (delta > 0.6)
ost->sync_opts = llrint(sync_ipts);
break;
case VSYNC_DROP:
case VSYNC_PASSTHROUGH:
ost->sync_opts = llrint(sync_ipts);
break;
default:
av_assert0(0);
}
}
nb_frames = FFMIN(nb_frames, ost->max_frames - ost->frame_number);
nb0_frames = FFMIN(nb0_frames, nb_frames);
memmove(ost->last_nb0_frames + 1,
ost->last_nb0_frames,
sizeof(ost->last_nb0_frames[0]) * (FF_ARRAY_ELEMS(ost->last_nb0_frames) - 1));
ost->last_nb0_frames[0] = nb0_frames;
if (nb0_frames == 0 && ost->last_dropped) {
//丢帧的时候进入到这里来了，ost->last_dropped=1 ost->last_frame->pts=0 ost->frame_number=3
nb_frames_drop++;
av_log(NULL, AV_LOG_VERBOSE,
"*** dropping frame %d from stream %d at ts %"PRId64"\n",
ost->frame_number, ost->st->index, ost->last_frame->pts);
}
if (nb_frames > (nb0_frames && ost->last_dropped) + (nb_frames > nb0_frames)) {
if (nb_frames > dts_error_threshold * 30) {
av_log(NULL, AV_LOG_ERROR, "%d frame duplication too large, skipping\n", nb_frames - 1);
nb_frames_drop++;
return;
}
nb_frames_dup += nb_frames - (nb0_frames && ost->last_dropped) - (nb_frames > nb0_frames);
av_log(NULL, AV_LOG_VERBOSE, "*** %d dup!\n", nb_frames - 1);
if (nb_frames_dup > dup_warning) {
av_log(NULL, AV_LOG_WARNING, "More than %d frames duplicated\n", dup_warning);
dup_warning *= 10;
}
}
// nb_frames == nb0_frames这两个都等于0，那么后面就没有编码的操作了
ost->last_dropped = nb_frames == nb0_frames && next_picture;
/* duplicates frame if needed */
for (i = 0; i < nb_frames; i++) {
AVFrame *in_picture;
int forced_keyframe = 0;
double pts_time;
if (i < nb0_frames && ost->last_frame) {
in_picture = ost->last_frame;
} else
in_picture = next_picture;
if (!in_picture)
return;
in_picture->pts = ost->sync_opts;
if (!check_recording_time(ost))
return;
in_picture->quality = enc->global_quality;
in_picture->pict_type = 0;
if (ost->forced_kf_ref_pts == AV_NOPTS_VALUE &&
in_picture->pts != AV_NOPTS_VALUE)
ost->forced_kf_ref_pts = in_picture->pts;
pts_time = in_picture->pts != AV_NOPTS_VALUE ?
(in_picture->pts - ost->forced_kf_ref_pts) * av_q2d(enc->time_base) : NAN;
if (ost->forced_kf_index < ost->forced_kf_count &&
in_picture->pts >= ost->forced_kf_pts[ost->forced_kf_index]) {
ost->forced_kf_index++;
forced_keyframe = 1;
} else if (ost->forced_keyframes_pexpr) {
double res;
ost->forced_keyframes_expr_const_values[FKF_T] = pts_time;
res = av_expr_eval(ost->forced_keyframes_pexpr,
ost->forced_keyframes_expr_const_values, NULL);
ff_dlog(NULL, "force_key_frame: n:%f n_forced:%f prev_forced_n:%f t:%f prev_forced_t:%f -> res:%f\n",
ost->forced_keyframes_expr_const_values[FKF_N],
ost->forced_keyframes_expr_const_values[FKF_N_FORCED],
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_N],
ost->forced_keyframes_expr_const_values[FKF_T],
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_T],
res);
if (res) {
forced_keyframe = 1;
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_N] =
ost->forced_keyframes_expr_const_values[FKF_N];
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_T] =
ost->forced_keyframes_expr_const_values[FKF_T];
ost->forced_keyframes_expr_const_values[FKF_N_FORCED] += 1;
}
ost->forced_keyframes_expr_const_values[FKF_N] += 1;
} else if ( ost->forced_keyframes
&& !strncmp(ost->forced_keyframes, "source", 6)
&& in_picture->key_frame==1
&& !i) {
forced_keyframe = 1;
}
if (forced_keyframe) {
in_picture->pict_type = AV_PICTURE_TYPE_I;
av_log(NULL, AV_LOG_DEBUG, "Forced keyframe at time %f\n", pts_time);
}
update_benchmark(NULL);
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "encoder <- type:video "
"frame_pts:%s frame_pts_time:%s time_base:%d/%d\n",
av_ts2str(in_picture->pts), av_ts2timestr(in_picture->pts, &enc->time_base),
enc->time_base.num, enc->time_base.den);
}
ost->frames_encoded++;
ret = avcodec_send_frame(enc, in_picture);
if (ret < 0)
goto error;
// Make sure Closed Captions will not be duplicated
av_frame_remove_side_data(in_picture, AV_FRAME_DATA_A53_CC);
while (1) {
av_packet_unref(pkt);
ret = avcodec_receive_packet(enc, pkt);
update_benchmark("encode_video %d.%d", ost->file_index, ost->index);
if (ret == AVERROR(EAGAIN))
break;
if (ret < 0)
goto error;
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "encoder -> type:video "
"pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s\n",
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, &enc->time_base),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, &enc->time_base));
}
if (pkt->pts == AV_NOPTS_VALUE && !(enc->codec->capabilities & AV_CODEC_CAP_DELAY))
pkt->pts = ost->sync_opts;
av_packet_rescale_ts(pkt, enc->time_base, ost->mux_timebase);
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "encoder -> type:video "
"pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s\n",
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, &ost->mux_timebase),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, &ost->mux_timebase));
}
frame_size = pkt->size;
output_packet(of, pkt, ost, 0);
/* if two pass, output log */
if (ost->logfile && enc->stats_out) {
fprintf(ost->logfile, "%s", enc->stats_out);
}
}
ost->sync_opts++;
/*
* For video, number of frames in == number of packets out.
* But there may be reordering, so we can't throw away frames on encoder
* flush, we need to limit them here, before they go into encoder.
*/
ost->frame_number++;
if (vstats_filename && frame_size)
do_video_stats(ost, frame_size);
}
if (!ost->last_frame)
ost->last_frame = av_frame_alloc();
av_frame_unref(ost->last_frame);
if (next_picture && ost->last_frame)
av_frame_ref(ost->last_frame, next_picture);
else
av_frame_free(&ost->last_frame);
return;
error:
av_log(NULL, AV_LOG_FATAL, "Video encoding failed\n");
exit_program(1);
}
这个关键
Thread 1 "ffmpeg_g" hit Breakpoint 2, do_video_out (of=0x556550aa0800, ost=0x556550b89880, next_picture=0x556550d54740)
at fftools/ffmpeg.c:1162
1162 sync_ipts = adjust_frame_pts_to_encoder_tb(of, ost, next_picture);
(gdb) s
adjust_frame_pts_to_encoder_tb (of=0x0, ost=0x0, frame=0x0) at fftools/ffmpeg.c:938
938 {
(gdb) n
939 double float_pts = AV_NOPTS_VALUE; // this is identical to frame.pts but with higher precision
(gdb)
940 AVCodecContext *enc = ost->enc_ctx;
(gdb)
941 if (!frame || frame->pts == AV_NOPTS_VALUE ||
(gdb)
942 !enc || !ost->filter || !ost->filter->graph->graph)
(gdb)
946 AVFilterContext *filter = ost->filter->filter;
(gdb)
948 int64_t start_time = (of->start_time == AV_NOPTS_VALUE) ? 0 : of->start_time;
(gdb)
949 AVRational filter_tb = av_buffersink_get_time_base(filter);
(gdb)
950 AVRational tb = enc->time_base;
(gdb)
951 int extra_bits = av_clip(29 - av_log2(tb.den), 0, 16);
(gdb)
953 tb.den <<= extra_bits;
(gdb)
955 av_rescale_q(frame->pts, filter_tb, tb) -
(gdb) p start_time
$53 = 0
(gdb) p filter_tb
$54 = {
num = 1, den = 1200000}
(gdb) p tb
$55 = {
num = 1, den = 1966080}
(gdb) p extra_bits
$56 = 16
(gdb) p frame->pts
$57 = 48000
(gdb) p filter_tb
$58 = {
num = 1, den = 1200000}
(gdb) n
956 av_rescale_q(start_time, AV_TIME_BASE_Q, tb);
(gdb)
955 av_rescale_q(frame->pts, filter_tb, tb) -
(gdb)
954 float_pts =
(gdb)
957 float_pts /= 1 << extra_bits;
(gdb)
959 float_pts += FFSIGN(float_pts) * 1.0 / (1<<17);
(gdb)
962 av_rescale_q(frame->pts, filter_tb, enc->time_base) -
(gdb)
963 av_rescale_q(start_time, AV_TIME_BASE_Q, enc->time_base);
(gdb)
962 av_rescale_q(frame->pts, filter_tb, enc->time_base) -
(gdb)
961 frame->pts =
(gdb)
968 if (debug_ts) {
(gdb)
977 return float_pts;
(gdb) p float_pts
$59 = 1.2000045776367188
(gdb) p frame->pts
$60 = 1
(gdb)
说明：av_rescale_q用于计算Packet的PTS。av_rescale_q的返回值是一个很大的整数，
且每次计算的结果间隔很大。
不同于avcodec_encode_video改变AVCodecContext *avctx的pts（小整数，且间隔小）。
av_rescale_q(a,b,c)是用来把时间戳从一个时基调整到另外一个时基时候用的函数。
它基本的动作是计算a*b/c，但是这个函数还是必需的，因为直接计算会有溢出的情况发生。
AV_TIME_BASE_Q是AV_TIME_BASE作为分母后的版本。
它们是很不相同的：AV_TIME_BASE * time_in_seconds = avcodec_timestamp
而AV_TIME_BASE_Q * avcodec_timestamp = time_in_seconds
（注意AV_TIME_BASE_Q实际上是一个AVRational对象，
所以你必需使用avcodec中特定的q函数来处理它）
————————————————
版权声明：本文为CSDN博主「disadministrator」的原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接及本声明。
原文链接：https://blog.csdn.net/disadministrator/article/details/46652081
static double adjust_frame_pts_to_encoder_tb(OutputFile *of, OutputStream *ost,
AVFrame *frame)
{
double float_pts = AV_NOPTS_VALUE; // this is identical to frame.pts but with higher precision
AVCodecContext *enc = ost->enc_ctx;
if (!frame || frame->pts == AV_NOPTS_VALUE ||
!enc || !ost->filter || !ost->filter->graph->graph)
goto early_exit;
{
//进入到这里来
AVFilterContext *filter = ost->filter->filter;
//start_time=0
int64_t start_time = (of->start_time == AV_NOPTS_VALUE) ? 0 : of->start_time;
AVRational filter_tb = av_buffersink_get_time_base(filter);
AVRational tb = enc->time_base;
int extra_bits = av_clip(29 - av_log2(tb.den), 0, 16);
tb.den <<= extra_bits;
float_pts =
av_rescale_q(frame->pts, filter_tb, tb) -
av_rescale_q(start_time, AV_TIME_BASE_Q, tb);
float_pts /= 1 << extra_bits;
// avoid exact midoints to reduce the chance of rounding differences, this can be removed in case the fps code is changed to work with integers
float_pts += FFSIGN(float_pts) * 1.0 / (1<<17);
frame->pts =
av_rescale_q(frame->pts, filter_tb, enc->time_base) -
av_rescale_q(start_time, AV_TIME_BASE_Q, enc->time_base);
}
early_exit:
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "filter -> pts:%s pts_time:%s exact:%f time_base:%d/%d\n",
frame ? av_ts2str(frame->pts) : "NULL",
frame ? av_ts2timestr(frame->pts, &enc->time_base) : "NULL",
float_pts,
enc ? enc->time_base.num : -1,
enc ? enc->time_base.den : -1);
}
return float_pts;
}
