这段代码费解，但是要用心读还是可以读懂的。
我们先读音频audio_thread
先上流程图
接下来我们上代码
这段代码整体上就是
af = frame_queue_peek_writable(&is->sampq)
af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
af->pos = frame->pkt_pos;
af->serial = is->auddec.pkt_serial;
af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate});
av_frame_move_ref(af->frame, frame);
frame_queue_push(&is->sampq);取到frame，将frame放入sampq队列当中
完整代码解析
static int audio_thread(void *arg)
{
FFPlayer *ffp = arg;
VideoState *is = ffp->is;
AVFrame *frame = av_frame_alloc();
Frame *af;
#if CONFIG_AVFILTER
int last_serial = -1;
int64_t dec_channel_layout;
int reconfigure;
#endif
int got_frame = 0;
AVRational tb;
int ret = 0;
int audio_accurate_seek_fail = 0;
int64_t audio_seek_pos = 0;
double frame_pts = 0;
double audio_clock = 0;
int64_t now = 0;
double samples_duration = 0;
int64_t deviation = 0;
int64_t deviation2 = 0;
int64_t deviation3 = 0;
if (!frame)
return AVERROR(ENOMEM);
do {
ffp_audio_statistic_l(ffp);
if ((got_frame = decoder_decode_frame(ffp, &is->auddec, frame, NULL)) < 0)
goto the_end;
if (got_frame) {
//播放每个采样点需要多长时间
tb = (AVRational){1, frame->sample_rate};
//是否开启了精准seek
if (ffp->enable_accurate_seek && is->audio_accurate_seek_req && !is->seek_req) {
//当前frame的开始播放时间。frame->pts当前帧是第几个采样点
frame_pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
now = av_gettime_relative() / 1000;
if (!isnan(frame_pts)) {
//当前帧播放的事件，每个帧有多个采样点
samples_duration = (double) frame->nb_samples / frame->sample_rate;
//当前帧播放完的事件点
audio_clock = frame_pts + samples_duration;
//精准audioseek的pts，用微妙作为单位
is->accurate_seek_aframe_pts = audio_clock * 1000 * 1000;
audio_seek_pos = is->seek_pos;
deviation = llabs((int64_t)(audio_clock * 1000 * 1000) - is->seek_pos);
//audioclock小于seekpos或者两者的差距超过1200ms
if ((audio_clock * 1000 * 1000 < is->seek_pos ) || deviation > MAX_DEVIATION) {
if (is->drop_aframe_count == 0) {
SDL_LockMutex(is->accurate_seek_mutex);
if (is->accurate_seek_start_time <= 0 && (is->video_stream < 0 || is->video_accurate_seek_req)) {
is->accurate_seek_start_time = now;
}
SDL_UnlockMutex(is->accurate_seek_mutex);
av_log(NULL, AV_LOG_INFO, "audio accurate_seek start, is->seek_pos=%lld, audio_clock=%lf, is->accurate_seek_start_time = %lld\n", is->seek_pos, audio_clock, is->accurate_seek_start_time);
}
is->drop_aframe_count++;
//通过sleep达到1、audio_clock-100ms<精确videopts<seekpos或者2、（audio_clock-seekpos>1200ms）&&精确videopts< seekpos
while (is->video_accurate_seek_req && !is->abort_request) {
int64_t vpts = is->accurate_seek_vframe_pts;
deviation2 = vpts - audio_clock * 1000 * 1000;
deviation3 = vpts - is->seek_pos;
if (deviation2 > -100 * 1000 && deviation3 < 0) {
break;
} else {
av_usleep(20 * 1000);
}
now = av_gettime_relative() / 1000;
if ((now - is->accurate_seek_start_time) > ffp->accurate_seek_timeout) {
break;
}
}
if(!is->video_accurate_seek_req && is->video_stream >= 0 && audio_clock * 1000 * 1000 > is->accurate_seek_vframe_pts) {
audio_accurate_seek_fail = 1;
} else {
now = av_gettime_relative() / 1000;
if ((now - is->accurate_seek_start_time) <= ffp->accurate_seek_timeout) {
av_frame_unref(frame);
continue; // drop some old frame when do accurate seek
} else {
audio_accurate_seek_fail = 1;
}
}
} else {
//等待视频发出音频调整精准seek信号
if (audio_seek_pos == is->seek_pos) {
av_log(NULL, AV_LOG_INFO, "audio accurate_seek is ok, is->drop_aframe_count=%d, audio_clock = %lf\n", is->drop_aframe_count, audio_clock);
is->drop_aframe_count = 0;
SDL_LockMutex(is->accurate_seek_mutex);
is->audio_accurate_seek_req = 0;
SDL_CondSignal(is->video_accurate_seek_cond);
if (audio_seek_pos == is->seek_pos && is->video_accurate_seek_req && !is->abort_request) {
SDL_CondWaitTimeout(is->audio_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
} else {
ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(audio_clock * 1000));
}
if (audio_seek_pos != is->seek_pos && !is->abort_request) {
is->audio_accurate_seek_req = 1;
SDL_UnlockMutex(is->accurate_seek_mutex);
av_frame_unref(frame);
continue;
}
SDL_UnlockMutex(is->accurate_seek_mutex);
}
}
} else {
audio_accurate_seek_fail = 1;
}
if (audio_accurate_seek_fail) {
//等待视频发出音频调整精准seek信号
av_log(NULL, AV_LOG_INFO, "audio accurate_seek is error, is->drop_aframe_count=%d, now = %lld, audio_clock = %lf\n", is->drop_aframe_count, now, audio_clock);
is->drop_aframe_count = 0;
SDL_LockMutex(is->accurate_seek_mutex);
is->audio_accurate_seek_req = 0;
SDL_CondSignal(is->video_accurate_seek_cond);
if (is->video_accurate_seek_req && !is->abort_request) {
SDL_CondWaitTimeout(is->audio_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
} else {
ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(audio_clock * 1000));
}
SDL_UnlockMutex(is->accurate_seek_mutex);
}
is->accurate_seek_start_time = 0;
audio_accurate_seek_fail = 0;
}
#if CONFIG_AVFILTER
dec_channel_layout = get_valid_channel_layout(frame->channel_layout, frame->channels);
reconfigure =
cmp_audio_fmts(is->audio_filter_src.fmt, is->audio_filter_src.channels,
frame->format, frame->channels) ||
is->audio_filter_src.channel_layout != dec_channel_layout ||
is->audio_filter_src.freq != frame->sample_rate ||
is->auddec.pkt_serial != last_serial ||
ffp->af_changed;
if (reconfigure) {
SDL_LockMutex(ffp->af_mutex);
ffp->af_changed = 0;
char buf1[1024], buf2[1024];
av_get_channel_layout_string(buf1, sizeof(buf1), -1, is->audio_filter_src.channel_layout);
av_get_channel_layout_string(buf2, sizeof(buf2), -1, dec_channel_layout);
av_log(NULL, AV_LOG_DEBUG,
"Audio frame changed from rate:%d ch:%d fmt:%s layout:%s serial:%d to rate:%d ch:%d fmt:%s layout:%s serial:%d\n",
is->audio_filter_src.freq, is->audio_filter_src.channels, av_get_sample_fmt_name(is->audio_filter_src.fmt), buf1, last_serial,
frame->sample_rate, frame->channels, av_get_sample_fmt_name(frame->format), buf2, is->auddec.pkt_serial);
is->audio_filter_src.fmt = frame->format;
is->audio_filter_src.channels = frame->channels;
is->audio_filter_src.channel_layout = dec_channel_layout;
is->audio_filter_src.freq = frame->sample_rate;
last_serial = is->auddec.pkt_serial;
if ((ret = configure_audio_filters(ffp, ffp->afilters, 1)) < 0) {
SDL_UnlockMutex(ffp->af_mutex);
goto the_end;
}
SDL_UnlockMutex(ffp->af_mutex);
}
if ((ret = av_buffersrc_add_frame(is->in_audio_filter, frame)) < 0)
goto the_end;
while ((ret = av_buffersink_get_frame_flags(is->out_audio_filter, frame, 0)) >= 0) {
tb = av_buffersink_get_time_base(is->out_audio_filter);
#endif
if (!(af = frame_queue_peek_writable(&is->sampq)))
goto the_end;
//视频帧放入VideoState的sampq
af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
af->pos = frame->pkt_pos;
af->serial = is->auddec.pkt_serial;
af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate});
av_frame_move_ref(af->frame, frame);
frame_queue_push(&is->sampq);
#if CONFIG_AVFILTER
if (is->audioq.serial != is->auddec.pkt_serial)
break;
}
if (ret == AVERROR_EOF)
is->auddec.finished = is->auddec.pkt_serial;
#endif
}
} while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF);
the_end:
#if CONFIG_AVFILTER
avfilter_graph_free(&is->agraph);
#endif
av_frame_free(&frame);
return ret;
}
这段代码必须结合前面的流程图看，要不会晕的。
接下来看下视频和音频代码是一模一样的。
static int queue_picture(FFPlayer *ffp, AVFrame *src_frame, double pts, double duration, int64_t pos, int serial)
{
VideoState *is = ffp->is;
Frame *vp;
int video_accurate_seek_fail = 0;
int64_t video_seek_pos = 0;
int64_t now = 0;
int64_t deviation = 0;
int64_t deviation2 = 0;
int64_t deviation3 = 0;
if (ffp->enable_accurate_seek && is->video_accurate_seek_req && !is->seek_req) {
if (!isnan(pts)) {
video_seek_pos = is->seek_pos;
is->accurate_seek_vframe_pts = pts * 1000 * 1000;
deviation = llabs((int64_t)(pts * 1000 * 1000) - is->seek_pos);
if ((pts * 1000 * 1000 < is->seek_pos) || deviation > MAX_DEVIATION) {
now = av_gettime_relative() / 1000;
if (is->drop_vframe_count == 0) {
SDL_LockMutex(is->accurate_seek_mutex);
if (is->accurate_seek_start_time <= 0 && (is->audio_stream < 0 || is->audio_accurate_seek_req)) {
is->accurate_seek_start_time = now;
}
SDL_UnlockMutex(is->accurate_seek_mutex);
av_log(NULL, AV_LOG_INFO, "video accurate_seek start, is->seek_pos=%lld, pts=%lf, is->accurate_seek_time = %lld\n", is->seek_pos, pts, is->accurate_seek_start_time);
}
is->drop_vframe_count++;
while (is->audio_accurate_seek_req && !is->abort_request) {
int64_t apts = is->accurate_seek_aframe_pts ;
deviation2 = apts - pts * 1000 * 1000;
deviation3 = apts - is->seek_pos;
if (deviation2 > -100 * 1000 && deviation3 < 0) {
break;
} else {
av_usleep(20 * 1000);
}
now = av_gettime_relative() / 1000;
if ((now - is->accurate_seek_start_time) > ffp->accurate_seek_timeout) {
break;
}
}
if ((now - is->accurate_seek_start_time) <= ffp->accurate_seek_timeout) {
return 1; // drop some old frame when do accurate seek
} else {
av_log(NULL, AV_LOG_WARNING, "video accurate_seek is error, is->drop_vframe_count=%d, now = %lld, pts = %lf\n", is->drop_vframe_count, now, pts);
video_accurate_seek_fail = 1; // if KEY_FRAME interval too big, disable accurate seek
}
} else {
av_log(NULL, AV_LOG_INFO, "video accurate_seek is ok, is->drop_vframe_count =%d, is->seek_pos=%lld, pts=%lf\n", is->drop_vframe_count, is->seek_pos, pts);
if (video_seek_pos == is->seek_pos) {
is->drop_vframe_count = 0;
SDL_LockMutex(is->accurate_seek_mutex);
is->video_accurate_seek_req = 0;
SDL_CondSignal(is->audio_accurate_seek_cond);
if (video_seek_pos == is->seek_pos && is->audio_accurate_seek_req && !is->abort_request) {
SDL_CondWaitTimeout(is->video_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
} else {
ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(pts * 1000));
}
if (video_seek_pos != is->seek_pos && !is->abort_request) {
is->video_accurate_seek_req = 1;
SDL_UnlockMutex(is->accurate_seek_mutex);
return 1;
}
SDL_UnlockMutex(is->accurate_seek_mutex);
}
}
} else {
video_accurate_seek_fail = 1;
}
if (video_accurate_seek_fail) {
is->drop_vframe_count = 0;
SDL_LockMutex(is->accurate_seek_mutex);
is->video_accurate_seek_req = 0;
SDL_CondSignal(is->audio_accurate_seek_cond);
if (is->audio_accurate_seek_req && !is->abort_request) {
SDL_CondWaitTimeout(is->video_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
} else {
if (!isnan(pts)) {
ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(pts * 1000));
} else {
ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, 0);
}
}
SDL_UnlockMutex(is->accurate_seek_mutex);
}
is->accurate_seek_start_time = 0;
video_accurate_seek_fail = 0;
is->accurate_seek_vframe_pts = 0;
}
#if defined(DEBUG_SYNC)
printf("frame_type=%c pts=%0.3f\n",
av_get_picture_type_char(src_frame->pict_type), pts);
#endif
if (!(vp = frame_queue_peek_writable(&is->pictq)))
return -1;
vp->sar = src_frame->sample_aspect_ratio;
#ifdef FFP_MERGE
vp->uploaded = 0;
#endif
/* alloc or resize hardware picture buffer */
if (!vp->bmp || !vp->allocated ||
vp->width != src_frame->width ||
vp->height != src_frame->height ||
vp->format != src_frame->format) {
if (vp->width != src_frame->width || vp->height != src_frame->height)
ffp_notify_msg3(ffp, FFP_MSG_VIDEO_SIZE_CHANGED, src_frame->width, src_frame->height);
vp->allocated = 0;
vp->width = src_frame->width;
vp->height = src_frame->height;
vp->format = src_frame->format;
/* the allocation must be done in the main thread to avoid
locking problems. */
alloc_picture(ffp, src_frame->format);
if (is->videoq.abort_request)
return -1;
}
/* if the frame is not skipped, then display it */
if (vp->bmp) {
/* get a pointer on the bitmap */
SDL_VoutLockYUVOverlay(vp->bmp);
#ifdef FFP_MERGE
#if CONFIG_AVFILTER
// FIXME use direct rendering
av_image_copy(data, linesize, (const uint8_t **)src_frame->data, src_frame->linesize,
src_frame->format, vp->width, vp->height);
#else
// sws_getCachedContext(...);
#endif
#endif
// FIXME: set swscale options
if (SDL_VoutFillFrameYUVOverlay(vp->bmp, src_frame) < 0) {
av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");
exit(1);
}
/* update the bitmap content */
SDL_VoutUnlockYUVOverlay(vp->bmp);
vp->pts = pts;
vp->duration = duration;
vp->pos = pos;
vp->serial = serial;
vp->sar = src_frame->sample_aspect_ratio;
vp->bmp->sar_num = vp->sar.num;
vp->bmp->sar_den = vp->sar.den;
#ifdef FFP_MERGE
av_frame_move_ref(vp->frame, src_frame);
#endif
frame_queue_push(&is->pictq);
if (!is->viddec.first_frame_decoded) {
ALOGD("Video: first frame decoded\n");
ffp_notify_msg1(ffp, FFP_MSG_VIDEO_DECODED_START);
is->viddec.first_frame_decoded_time = SDL_GetTickHR();
is->viddec.first_frame_decoded = 1;
}
}
return 0;
}