android的音频播放使用的是AudioTrack，如果要想看懂ijk的音频播放要对AudioTrack有个基础了解。
大家可以看我写的android的音频播放文章
AudioTrack使用_原总破局的博客-CSDN博客_audiotrack
static int stream_component_open(FFPlayer *ffp, int stream_index)
{
switch (avctx->codec_type) {
case AVMEDIA_TYPE_AUDIO:
#if CONFIG_AVFILTER
{
AVFilterContext *sink;
is->audio_filter_src.freq = avctx->sample_rate;
is->audio_filter_src.channels = avctx->channels;
is->audio_filter_src.channel_layout = get_valid_channel_layout(avctx->channel_layout, avctx->channels);
is->audio_filter_src.fmt = avctx->sample_fmt;
SDL_LockMutex(ffp->af_mutex);
if ((ret = configure_audio_filters(ffp, ffp->afilters, 0)) < 0) {
SDL_UnlockMutex(ffp->af_mutex);
goto fail;
}
ffp->af_changed = 0;
SDL_UnlockMutex(ffp->af_mutex);
sink = is->out_audio_filter;
sample_rate = av_buffersink_get_sample_rate(sink);
nb_channels = av_buffersink_get_channels(sink);
channel_layout = av_buffersink_get_channel_layout(sink);
}
#else
sample_rate = avctx->sample_rate;
nb_channels = avctx->channels;
channel_layout = avctx->channel_layout;
#endif
/* prepare audio output */
if ((ret = audio_open(ffp, channel_layout, nb_channels, sample_rate, &is->audio_tgt)) < 0)
goto fail;
ffp_set_audio_codec_info(ffp, AVCODEC_MODULE_NAME, avcodec_get_name(avctx->codec_id));
is->audio_hw_buf_size = ret;
is->audio_src = is->audio_tgt;
is->audio_buf_size = 0;
is->audio_buf_index = 0;
/* init averaging filter */
is->audio_diff_avg_coef = exp(log(0.01) / AUDIO_DIFF_AVG_NB);
is->audio_diff_avg_count = 0;
/* since we do not have a precise anough audio FIFO fullness,
we correct audio sync only if larger than this threshold */
is->audio_diff_threshold = 2.0 * is->audio_hw_buf_size / is->audio_tgt.bytes_per_sec;
is->audio_stream = stream_index;
is->audio_st = ic->streams[stream_index];
decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread);
if ((is->ic->iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH | AVFMT_NO_BYTE_SEEK)) && !is->ic->iformat->read_seek) {
is->auddec.start_pts = is->audio_st->start_time;
is->auddec.start_pts_tb = is->audio_st->time_base;
}
if ((ret = decoder_start(&is->auddec, audio_thread, ffp, "ff_audio_dec")) < 0)
goto out;
SDL_AoutPauseAudio(ffp->aout, 0);
break;
}这里有两个重要的函数audio_open和decoder_start
我们先看decoder_start
if ((ret = decoder_start(&is->auddec, audio_thread, ffp, "ff_audio_dec")) < 0)
goto out;
static int decoder_start(Decoder *d, int (*fn)(void *), void *arg, const char *name)
{
packet_queue_start(d->queue);
d->decoder_tid = SDL_CreateThreadEx(&d->_decoder_tid, fn, arg, name);
if (!d->decoder_tid) {
av_log(NULL, AV_LOG_ERROR, "SDL_CreateThread(): %s\n", SDL_GetError());
return AVERROR(ENOMEM);
}
return 0;
}
static int audio_thread(void *arg)
{
FFPlayer *ffp = arg;
VideoState *is = ffp->is;
AVFrame *frame = av_frame_alloc();
Frame *af;
#if CONFIG_AVFILTER
int last_serial = -1;
int64_t dec_channel_layout;
int reconfigure;
#endif
int got_frame = 0;
AVRational tb;
int ret = 0;
int audio_accurate_seek_fail = 0;
int64_t audio_seek_pos = 0;
double frame_pts = 0;
double audio_clock = 0;
int64_t now = 0;
double samples_duration = 0;
int64_t deviation = 0;
int64_t deviation2 = 0;
int64_t deviation3 = 0;
if (!frame)
return AVERROR(ENOMEM);
do {
ffp_audio_statistic_l(ffp);
if ((got_frame = decoder_decode_frame(ffp, &is->auddec, frame, NULL)) < 0)
goto the_end;
if (got_frame) {
tb = (AVRational){1, frame->sample_rate};
if (ffp->enable_accurate_seek && is->audio_accurate_seek_req && !is->seek_req) {
frame_pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
now = av_gettime_relative() / 1000;
if (!isnan(frame_pts)) {
samples_duration = (double) frame->nb_samples / frame->sample_rate;
audio_clock = frame_pts + samples_duration;
is->accurate_seek_aframe_pts = audio_clock * 1000 * 1000;
audio_seek_pos = is->seek_pos;
deviation = llabs((int64_t)(audio_clock * 1000 * 1000) - is->seek_pos);
if ((audio_clock * 1000 * 1000 < is->seek_pos ) || deviation > MAX_DEVIATION) {
if (is->drop_aframe_count == 0) {
SDL_LockMutex(is->accurate_seek_mutex);
if (is->accurate_seek_start_time <= 0 && (is->video_stream < 0 || is->video_accurate_seek_req)) {
is->accurate_seek_start_time = now;
}
SDL_UnlockMutex(is->accurate_seek_mutex);
av_log(NULL, AV_LOG_INFO, "audio accurate_seek start, is->seek_pos=%lld, audio_clock=%lf, is->accurate_seek_start_time = %lld\n", is->seek_pos, audio_clock, is->accurate_seek_start_time);
}
is->drop_aframe_count++;
while (is->video_accurate_seek_req && !is->abort_request) {
int64_t vpts = is->accurate_seek_vframe_pts;
deviation2 = vpts - audio_clock * 1000 * 1000;
deviation3 = vpts - is->seek_pos;
if (deviation2 > -100 * 1000 && deviation3 < 0) {
break;
} else {
av_usleep(20 * 1000);
}
now = av_gettime_relative() / 1000;
if ((now - is->accurate_seek_start_time) > ffp->accurate_seek_timeout) {
break;
}
}
if(!is->video_accurate_seek_req && is->video_stream >= 0 && audio_clock * 1000 * 1000 > is->accurate_seek_vframe_pts) {
audio_accurate_seek_fail = 1;
} else {
now = av_gettime_relative() / 1000;
if ((now - is->accurate_seek_start_time) <= ffp->accurate_seek_timeout) {
av_frame_unref(frame);
continue; // drop some old frame when do accurate seek
} else {
audio_accurate_seek_fail = 1;
}
}
} else {
if (audio_seek_pos == is->seek_pos) {
av_log(NULL, AV_LOG_INFO, "audio accurate_seek is ok, is->drop_aframe_count=%d, audio_clock = %lf\n", is->drop_aframe_count, audio_clock);
is->drop_aframe_count = 0;
SDL_LockMutex(is->accurate_seek_mutex);
is->audio_accurate_seek_req = 0;
SDL_CondSignal(is->video_accurate_seek_cond);
if (audio_seek_pos == is->seek_pos && is->video_accurate_seek_req && !is->abort_request) {
SDL_CondWaitTimeout(is->audio_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
} else {
ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(audio_clock * 1000));
}
if (audio_seek_pos != is->seek_pos && !is->abort_request) {
is->audio_accurate_seek_req = 1;
SDL_UnlockMutex(is->accurate_seek_mutex);
av_frame_unref(frame);
continue;
}
SDL_UnlockMutex(is->accurate_seek_mutex);
}
}
} else {
audio_accurate_seek_fail = 1;
}
if (audio_accurate_seek_fail) {
av_log(NULL, AV_LOG_INFO, "audio accurate_seek is error, is->drop_aframe_count=%d, now = %lld, audio_clock = %lf\n", is->drop_aframe_count, now, audio_clock);
is->drop_aframe_count = 0;
SDL_LockMutex(is->accurate_seek_mutex);
is->audio_accurate_seek_req = 0;
SDL_CondSignal(is->video_accurate_seek_cond);
if (is->video_accurate_seek_req && !is->abort_request) {
SDL_CondWaitTimeout(is->audio_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
} else {
ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(audio_clock * 1000));
}
SDL_UnlockMutex(is->accurate_seek_mutex);
}
is->accurate_seek_start_time = 0;
audio_accurate_seek_fail = 0;
}
#if CONFIG_AVFILTER
dec_channel_layout = get_valid_channel_layout(frame->channel_layout, frame->channels);
reconfigure =
cmp_audio_fmts(is->audio_filter_src.fmt, is->audio_filter_src.channels,
frame->format, frame->channels) ||
is->audio_filter_src.channel_layout != dec_channel_layout ||
is->audio_filter_src.freq != frame->sample_rate ||
is->auddec.pkt_serial != last_serial ||
ffp->af_changed;
if (reconfigure) {
SDL_LockMutex(ffp->af_mutex);
ffp->af_changed = 0;
char buf1[1024], buf2[1024];
av_get_channel_layout_string(buf1, sizeof(buf1), -1, is->audio_filter_src.channel_layout);
av_get_channel_layout_string(buf2, sizeof(buf2), -1, dec_channel_layout);
av_log(NULL, AV_LOG_DEBUG,
"Audio frame changed from rate:%d ch:%d fmt:%s layout:%s serial:%d to rate:%d ch:%d fmt:%s layout:%s serial:%d\n",
is->audio_filter_src.freq, is->audio_filter_src.channels, av_get_sample_fmt_name(is->audio_filter_src.fmt), buf1, last_serial,
frame->sample_rate, frame->channels, av_get_sample_fmt_name(frame->format), buf2, is->auddec.pkt_serial);
is->audio_filter_src.fmt = frame->format;
is->audio_filter_src.channels = frame->channels;
is->audio_filter_src.channel_layout = dec_channel_layout;
is->audio_filter_src.freq = frame->sample_rate;
last_serial = is->auddec.pkt_serial;
if ((ret = configure_audio_filters(ffp, ffp->afilters, 1)) < 0) {
SDL_UnlockMutex(ffp->af_mutex);
goto the_end;
}
SDL_UnlockMutex(ffp->af_mutex);
}
if ((ret = av_buffersrc_add_frame(is->in_audio_filter, frame)) < 0)
goto the_end;
while ((ret = av_buffersink_get_frame_flags(is->out_audio_filter, frame, 0)) >= 0) {
tb = av_buffersink_get_time_base(is->out_audio_filter);
#endif
if (!(af = frame_queue_peek_writable(&is->sampq)))
goto the_end;
af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
af->pos = frame->pkt_pos;
af->serial = is->auddec.pkt_serial;
af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate});
av_frame_move_ref(af->frame, frame);
frame_queue_push(&is->sampq);
#if CONFIG_AVFILTER
if (is->audioq.serial != is->auddec.pkt_serial)
break;
}
if (ret == AVERROR_EOF)
is->auddec.finished = is->auddec.pkt_serial;
#endif
}
} while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF);
the_end:
#if CONFIG_AVFILTER
avfilter_graph_free(&is->agraph);
#endif
av_frame_free(&frame);
return ret;
}通过上面的代码可以看到
audio_thread中decoder_decode_frame解码成了AvFrame，然后又放在了
sampq。通过前面的文章我们知道sampq有九个元素。放在sampq就是对这九个元素按顺序赋值。
我们看下decoder_decode_frame
static int decoder_decode_frame(FFPlayer *ffp, Decoder *d, AVFrame *frame, AVSubtitle *sub) {
int ret = AVERROR(EAGAIN);
for (;;) {
AVPacket pkt;
if (d->queue->serial == d->pkt_serial) {
do {
if (d->queue->abort_request)
return -1;
switch (d->avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
ffp->stat.vdps = SDL_SpeedSamplerAdd(&ffp->vdps_sampler, FFP_SHOW_VDPS_AVCODEC, "vdps[avcodec]");
if (ffp->decoder_reorder_pts == -1) {
frame->pts = frame->best_effort_timestamp;
} else if (!ffp->decoder_reorder_pts) {
frame->pts = frame->pkt_dts;
}
}
break;
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);
if (ret >= 0) {
AVRational tb = (AVRational){1, frame->sample_rate};
if (frame->pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(frame->pts, av_codec_get_pkt_timebase(d->avctx), tb);
else if (d->next_pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
if (frame->pts != AV_NOPTS_VALUE) {
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb;
}
}
break;
default:
break;
}
if (ret == AVERROR_EOF) {
d->finished = d->pkt_serial;
avcodec_flush_buffers(d->avctx);
return 0;
}
if (ret >= 0)
return 1;
} while (ret != AVERROR(EAGAIN));
}
do {
if (d->queue->nb_packets == 0)
SDL_CondSignal(d->empty_queue_cond);
if (d->packet_pending) {
av_packet_move_ref(&pkt, &d->pkt);
d->packet_pending = 0;
} else {
if (packet_queue_get_or_buffering(ffp, d->queue, &pkt, &d->pkt_serial, &d->finished) < 0)
return -1;
}
} while (d->queue->serial != d->pkt_serial);
if (pkt.data == flush_pkt.data) {
avcodec_flush_buffers(d->avctx);
d->finished = 0;
d->next_pts = d->start_pts;
d->next_pts_tb = d->start_pts_tb;
} else {
if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
int got_frame = 0;
ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt);
if (ret < 0) {
ret = AVERROR(EAGAIN);
} else {
if (got_frame && !pkt.data) {
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF);
}
} else {
if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) {
av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
}
av_packet_unref(&pkt);
}
}
}这段代码对熟悉ffmpeg的人来说很简单。
就是调用avcodec_send_packet后再调用avcodec_receive_frame来获取AVFrame，同时我们知道这个过程是异步的。可能多个send后才会有一个receive。
我们下面看下audio_open
扫描二维码关注公众号，回复：
14651806 查看本文章
static int audio_open(FFPlayer *opaque, int64_t wanted_channel_layout, int wanted_nb_channels, int wanted_sample_rate, struct AudioParams *audio_hw_params)
{
FFPlayer *ffp = opaque;
VideoState *is = ffp->is;
SDL_AudioSpec wanted_spec, spec;
const char *env;
static const int next_nb_channels[] = {0, 0, 1, 6, 2, 6, 4, 6};
#ifdef FFP_MERGE
static const int next_sample_rates[] = {0, 44100, 48000, 96000, 192000};
#endif
static const int next_sample_rates[] = {0, 44100, 48000};
int next_sample_rate_idx = FF_ARRAY_ELEMS(next_sample_rates) - 1;
av_log(NULL, AV_LOG_ERROR, "yuan audio_open next_sample_rate_idx=%d", next_sample_rate_idx);
env = SDL_getenv("SDL_AUDIO_CHANNELS");
av_log(NULL, AV_LOG_ERROR, "yuan audio_open env=%s", env);
if (env) {
wanted_nb_channels = atoi(env);
wanted_channel_layout = av_get_default_channel_layout(wanted_nb_channels);
}
if (!wanted_channel_layout || wanted_nb_channels != av_get_channel_layout_nb_channels(wanted_channel_layout)) {
wanted_channel_layout = av_get_default_channel_layout(wanted_nb_channels);
wanted_channel_layout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;
}
wanted_nb_channels = av_get_channel_layout_nb_channels(wanted_channel_layout);
wanted_spec.channels = wanted_nb_channels;
wanted_spec.freq = wanted_sample_rate;
if (wanted_spec.freq <= 0 || wanted_spec.channels <= 0) {
av_log(NULL, AV_LOG_ERROR, "Invalid sample rate or channel count!\n");
return -1;
}
av_log(NULL, AV_LOG_ERROR, "yuan audio_open wanted_sample_rate=%d", wanted_sample_rate);
while (next_sample_rate_idx && next_sample_rates[next_sample_rate_idx] >= wanted_spec.freq)
next_sample_rate_idx--;
av_log(NULL, AV_LOG_ERROR, "yuan audio_open next_sample_rate_idx=%d", next_sample_rate_idx);
wanted_spec.format = AUDIO_S16SYS;
wanted_spec.silence = 0;
wanted_spec.samples = FFMAX(SDL_AUDIO_MIN_BUFFER_SIZE, 2 << av_log2(wanted_spec.freq / SDL_AoutGetAudioPerSecondCallBacks(ffp->aout)));
av_log(NULL, AV_LOG_ERROR, "yuan audio_open wanted_spec.samples=%d", wanted_spec.samples);
wanted_spec.callback = sdl_audio_callback;
wanted_spec.userdata = opaque;
while (SDL_AoutOpenAudio(ffp->aout, &wanted_spec, &spec) < 0) {
/* avoid infinity loop on exit. --by bbcallen */
if (is->abort_request)
return -1;
av_log(NULL, AV_LOG_WARNING, "SDL_OpenAudio (%d channels, %d Hz): %s\n",
wanted_spec.channels, wanted_spec.freq, SDL_GetError());
wanted_spec.channels = next_nb_channels[FFMIN(7, wanted_spec.channels)];
if (!wanted_spec.channels) {
wanted_spec.freq = next_sample_rates[next_sample_rate_idx--];
wanted_spec.channels = wanted_nb_channels;
if (!wanted_spec.freq) {
av_log(NULL, AV_LOG_ERROR,
"No more combinations to try, audio open failed\n");
return -1;
}
}
wanted_channel_layout = av_get_default_channel_layout(wanted_spec.channels);
}
if (spec.format != AUDIO_S16SYS) {
av_log(NULL, AV_LOG_ERROR,
"SDL advised audio format %d is not supported!\n", spec.format);
return -1;
}
if (spec.channels != wanted_spec.channels) {
wanted_channel_layout = av_get_default_channel_layout(spec.channels);
if (!wanted_channel_layout) {
av_log(NULL, AV_LOG_ERROR,
"SDL advised channel count %d is not supported!\n", spec.channels);
return -1;
}
}
audio_hw_params->fmt = AV_SAMPLE_FMT_S16;
audio_hw_params->freq = spec.freq;
audio_hw_params->channel_layout = wanted_channel_layout;
audio_hw_params->channels = spec.channels;
audio_hw_params->frame_size = av_samples_get_buffer_size(NULL, audio_hw_params->channels, 1, audio_hw_params->fmt, 1);
audio_hw_params->bytes_per_sec = av_samples_get_buffer_size(NULL, audio_hw_params->channels, audio_hw_params->freq, audio_hw_params->fmt, 1);
if (audio_hw_params->bytes_per_sec <= 0 || audio_hw_params->frame_size <= 0) {
av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size failed\n");
return -1;
}
SDL_AoutSetDefaultLatencySeconds(ffp->aout, ((double)(2 * spec.size)) / audio_hw_params->bytes_per_sec);
return spec.size;
}
我们可以看到调用到了SDL_AoutOpenAudio
int SDL_AoutOpenAudio(SDL_Aout *aout, const SDL_AudioSpec *desired, SDL_AudioSpec *obtained)
{
if (aout && desired && aout->open_audio)
return aout->open_audio(aout, desired, obtained);
return -1;
}
SDL_Aout *SDL_AoutAndroid_CreateForAudioTrack()
{
SDL_Aout *aout = SDL_Aout_CreateInternal(sizeof(SDL_Aout_Opaque));
if (!aout)
return NULL;
SDL_Aout_Opaque *opaque = aout->opaque;
opaque->wakeup_cond = SDL_CreateCond();
opaque->wakeup_mutex = SDL_CreateMutex();
opaque->speed = 1.0f;
aout->opaque_class = &g_audiotrack_class;
aout->free_l = aout_free_l;
aout->open_audio = aout_open_audio;
aout->pause_audio = aout_pause_audio;
aout->flush_audio = aout_flush_audio;
aout->set_volume = aout_set_volume;
aout->close_audio = aout_close_audio;
aout->func_get_audio_session_id = aout_get_audio_session_id;
aout->func_set_playback_rate = func_set_playback_rate;
return aout;
}
static int aout_open_audio(SDL_Aout *aout, const SDL_AudioSpec *desired, SDL_AudioSpec *obtained)
{
// SDL_Aout_Opaque *opaque = aout->opaque;
JNIEnv *env = NULL;
if (JNI_OK != SDL_JNI_SetupThreadEnv(&env)) {
ALOGE("aout_open_audio: AttachCurrentThread: failed");
return -1;
}
return aout_open_audio_n(env, aout, desired, obtained);
}
static int aout_open_audio_n(JNIEnv *env, SDL_Aout *aout, const SDL_AudioSpec *desired, SDL_AudioSpec *obtained)
{
assert(desired);
SDL_Aout_Opaque *opaque = aout->opaque;
opaque->spec = *desired;
opaque->atrack = SDL_Android_AudioTrack_new_from_sdl_spec(env, desired);
if (!opaque->atrack) {
ALOGE("aout_open_audio_n: failed to new AudioTrcak()");
return -1;
}
opaque->buffer_size = SDL_Android_AudioTrack_get_min_buffer_size(opaque->atrack);
if (opaque->buffer_size <= 0) {
ALOGE("aout_open_audio_n: failed to getMinBufferSize()");
SDL_Android_AudioTrack_free(env, opaque->atrack);
opaque->atrack = NULL;
return -1;
}
opaque->buffer = malloc(opaque->buffer_size);
if (!opaque->buffer) {
ALOGE("aout_open_audio_n: failed to allocate buffer");
SDL_Android_AudioTrack_free(env, opaque->atrack);
opaque->atrack = NULL;
return -1;
}
if (obtained) {
SDL_Android_AudioTrack_get_target_spec(opaque->atrack, obtained);
SDLTRACE("audio target format fmt:0x%x, channel:0x%x", (int)obtained->format, (int)obtained->channels);
}
opaque->audio_session_id = SDL_Android_AudioTrack_getAudioSessionId(env, opaque->atrack);
ALOGI("audio_session_id = %d\n", opaque->audio_session_id);
opaque->pause_on = 1;
opaque->abort_request = 0;
opaque->audio_tid = SDL_CreateThreadEx(&opaque->_audio_tid, aout_thread, aout, "ff_aout_android");
if (!opaque->audio_tid) {
ALOGE("aout_open_audio_n: failed to create audio thread");
SDL_Android_AudioTrack_free(env, opaque->atrack);
opaque->atrack = NULL;
return -1;
}
return 0;
}我们看到启动了
SDL_Android_AudioTrack_new_from_sdl_spec
SDL_Android_AudioTrack *SDL_Android_AudioTrack_new_from_sdl_spec(JNIEnv *env, const SDL_AudioSpec *sdl_spec)
{
SDL_Android_AudioTrack_Spec atrack_spec;
SDL_Android_AudioTrack_get_default_spec(&atrack_spec);
atrack_spec.sample_rate_in_hz = sdl_spec->freq;
atrack_spec.channel_config = find_android_channel(sdl_spec->channels);
atrack_spec.audio_format = find_android_format(sdl_spec->format);
atrack_spec.buffer_size_in_bytes = sdl_spec->size;
return SDL_Android_AudioTrack_new_from_spec(env, &atrack_spec);
}
SDL_Android_AudioTrack *SDL_Android_AudioTrack_new_from_spec(JNIEnv *env, SDL_Android_AudioTrack_Spec *spec)
{
assert(spec);
switch (spec->channel_config) {
case CHANNEL_OUT_MONO:
ALOGI("SDL_Android_AudioTrack: %s", "CHANNEL_OUT_MONO");
break;
case CHANNEL_OUT_STEREO:
ALOGI("SDL_Android_AudioTrack: %s", "CHANNEL_OUT_STEREO");
break;
default:
ALOGE("%s: invalid channel %d", __func__, spec->channel_config);
return NULL;
}
switch (spec->audio_format) {
case ENCODING_PCM_16BIT:
ALOGI("SDL_Android_AudioTrack: %s", "ENCODING_PCM_16BIT");
break;
case ENCODING_PCM_8BIT:
ALOGI("SDL_Android_AudioTrack: %s", "ENCODING_PCM_8BIT");
break;
#if 0
case ENCODING_PCM_FLOAT:
ALOGI("SDL_Android_AudioTrack: %s", "ENCODING_PCM_FLOAT");
if (sdk_int < IJK_API_21_LOLLIPOP) {
ALOGI("SDL_Android_AudioTrack: %s need API 21 or above", "ENCODING_PCM_FLOAT");
return NULL;
}
break;
#endif
default:
ALOGE("%s: invalid format %d", __func__, spec->audio_format);
return NULL;
}
if (spec->sample_rate_in_hz <= 0) {
ALOGE("%s: invalid sample rate %d", __func__, spec->sample_rate_in_hz);
return NULL;
}
SDL_Android_AudioTrack *atrack = (SDL_Android_AudioTrack*) mallocz(sizeof(SDL_Android_AudioTrack));
if (!atrack) {
ALOGE("%s: mallocz faild.\n", __func__);
return NULL;
}
atrack->spec = *spec;
// libswresample is ugly, depending on native resampler
while (atrack->spec.sample_rate_in_hz < 4000) {
atrack->spec.sample_rate_in_hz *= 2;
}
while (atrack->spec.sample_rate_in_hz > 48000) {
atrack->spec.sample_rate_in_hz /= 2;
}
int min_buffer_size = J4AC_AudioTrack__getMinBufferSize(env,
atrack->spec.sample_rate_in_hz,
atrack->spec.channel_config,
atrack->spec.audio_format);
if (J4A_ExceptionCheck__catchAll(env) || min_buffer_size <= 0) {
ALOGE("%s: J4AC_AudioTrack__getMinBufferSize: return %d:", __func__, min_buffer_size);
free(atrack);
return NULL;
}
// for fast playback
min_buffer_size *= AUDIOTRACK_PLAYBACK_MAXSPEED;
atrack->thiz = J4AC_AudioTrack__AudioTrack__asGlobalRef__catchAll(env,
atrack->spec.stream_type,
atrack->spec.sample_rate_in_hz,
atrack->spec.channel_config,
atrack->spec.audio_format,
min_buffer_size,
atrack->spec.mode);
if (!atrack->thiz) {
free(atrack);
return NULL;
}
atrack->min_buffer_size = min_buffer_size;
atrack->spec.buffer_size_in_bytes = min_buffer_size;
// atrack->max_volume = J4AC_AudioTrack__getMaxVolume__catchAll(env);
// atrack->min_volume = J4AC_AudioTrack__getMinVolume__catchAll(env);
atrack->max_volume = 1.0f;
atrack->min_volume = 0.0f;
// extra init
float init_volume = 1.0f;
init_volume = IJKMIN(init_volume, atrack->max_volume);
init_volume = IJKMAX(init_volume, atrack->min_volume);
ALOGI("%s: init volume as %f/(%f,%f)", __func__, init_volume, atrack->min_volume, atrack->max_volume);
J4AC_AudioTrack__setStereoVolume__catchAll(env, atrack->thiz, init_volume, init_volume);
return atrack;
}看到了
J4AC_AudioTrack__AudioTrack__asGlobalRef__catchAll
这个就是Android的AudioTrack的构造函数
public AudioTrack (int streamType,
int sampleRateInHz,
int channelConfig,
int audioFormat,
int bufferSizeInBytes,
int mode,
int sessionId)我们知道音频最重要的就是采样率 采样大小 声道数
streamType 是android定义的音频流的类型取值有下面
sampleRateInHz 采样率
channelConfig 单或者双通道
audioFormat 采样大小 可以是8位二进制 16位二进制 或者32位二进制
bufferSizeInBytes 需要的最小缓冲区大小 这个由
这个函数获取
mode
我们看到了这个函数
opaque->audio_tid = SDL_CreateThreadEx(&opaque->_audio_tid, aout_thread, aout, "ff_aout_android");
static int aout_thread_n(JNIEnv *env, SDL_Aout *aout)
{
SDL_AudioCallback audio_cblk = opaque->spec.callback;
if (!opaque->abort_request && !opaque->pause_on)
SDL_Android_AudioTrack_play(env, atrack);
while (!opaque->abort_request) {
audio_cblk(userdata, buffer, copy_size);
int written = SDL_Android_AudioTrack_write(env, atrack, buffer,
}
SDL_Android_AudioTrack_free(env, atrack);
return 0;
}上面代码留下了主要部分可以看懂就是就是获取到数据，调用AudioTrack的write方法。调用了write的方法就可以播放。
数据又是如何获取的呢？
wanted_spec.callback = sdl_audio_callback;
static void sdl_audio_callback(void *opaque, Uint8 *stream, int len)
{
FFPlayer *ffp = opaque;
VideoState *is = ffp->is;
int audio_size, len1;
if (!ffp || !is) {
memset(stream, 0, len);
return;
}
ffp->audio_callback_time = av_gettime_relative();
if (ffp->pf_playback_rate_changed) {
ffp->pf_playback_rate_changed = 0;
#if defined(__ANDROID__)
if (!ffp->soundtouch_enable) {
SDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);
}
#else
SDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);
#endif
}
if (ffp->pf_playback_volume_changed) {
ffp->pf_playback_volume_changed = 0;
SDL_AoutSetPlaybackVolume(ffp->aout, ffp->pf_playback_volume);
}
av_log(NULL, AV_LOG_ERROR, "yuan audio_open sdl_audio_callback=%d", len);
while (len > 0) {
if (is->audio_buf_index >= is->audio_buf_size) {
audio_size = audio_decode_frame(ffp);
if (audio_size < 0) {
/* if error, just output silence */
is->audio_buf = NULL;
is->audio_buf_size = SDL_AUDIO_MIN_BUFFER_SIZE / is->audio_tgt.frame_size * is->audio_tgt.frame_size;
} else {
if (is->show_mode != SHOW_MODE_VIDEO)
update_sample_display(is, (int16_t *)is->audio_buf, audio_size);
is->audio_buf_size = audio_size;
}
is->audio_buf_index = 0;
}
if (is->auddec.pkt_serial != is->audioq.serial) {
is->audio_buf_index = is->audio_buf_size;
memset(stream, 0, len);
// stream += len;
// len = 0;
SDL_AoutFlushAudio(ffp->aout);
break;
}
len1 = is->audio_buf_size - is->audio_buf_index;
if (len1 > len)
len1 = len;
if (!is->muted && is->audio_buf && is->audio_volume == SDL_MIX_MAXVOLUME)
memcpy(stream, (uint8_t *)is->audio_buf + is->audio_buf_index, len1);
else {
memset(stream, 0, len1);
if (!is->muted && is->audio_buf)
SDL_MixAudio(stream, (uint8_t *)is->audio_buf + is->audio_buf_index, len1, is->audio_volume);
}
len -= len1;
stream += len1;
is->audio_buf_index += len1;
}
is->audio_write_buf_size = is->audio_buf_size - is->audio_buf_index;
/* Let's assume the audio driver that is used by SDL has two periods. */
if (!isnan(is->audio_clock)) {
set_clock_at(&is->audclk, is->audio_clock - (double)(is->audio_write_buf_size) / is->audio_tgt.bytes_per_sec - SDL_AoutGetLatencySeconds(ffp->aout), is->audio_clock_serial, ffp->audio_callback_time / 1000000.0);
sync_clock_to_slave(&is->extclk, &is->audclk);
}
if (!ffp->first_audio_frame_rendered) {
ffp->first_audio_frame_rendered = 1;
ffp_notify_msg1(ffp, FFP_MSG_AUDIO_RENDERING_START);
}
if (is->latest_audio_seek_load_serial == is->audio_clock_serial) {
int latest_audio_seek_load_serial = __atomic_exchange_n(&(is->latest_audio_seek_load_serial), -1, memory_order_seq_cst);
if (latest_audio_seek_load_serial == is->audio_clock_serial) {
if (ffp->av_sync_type == AV_SYNC_AUDIO_MASTER) {
ffp_notify_msg2(ffp, FFP_MSG_AUDIO_SEEK_RENDERING_START, 1);
} else {
ffp_notify_msg2(ffp, FFP_MSG_AUDIO_SEEK_RENDERING_START, 0);
}
}
}
if (ffp->render_wait_start && !ffp->start_on_prepared && is->pause_req) {
while (is->pause_req && !is->abort_request) {
SDL_Delay(20);
}
}
}
最主要的是这句
SDL_MixAudio(stream, (uint8_t *)is->audio_buf + is->audio_buf_index, len1, is->audio_volume);
这里隐藏了个算法，不停获取数据直到形参传的大小。
可以看到数据来自于audio_decode_frame
static int audio_decode_frame(FFPlayer *ffp)
{
VideoState *is = ffp->is;
int data_size, resampled_data_size;
int64_t dec_channel_layout;
av_unused double audio_clock0;
int wanted_nb_samples;
Frame *af;
#if defined(__ANDROID__)
int translate_time = 1;
#endif
if (is->paused || is->step)
return -1;
if (ffp->sync_av_start && /* sync enabled */
is->video_st && /* has video stream */
!is->viddec.first_frame_decoded && /* not hot */
is->viddec.finished != is->videoq.serial) { /* not finished */
/* waiting for first video frame */
Uint64 now = SDL_GetTickHR();
if (now < is->viddec.first_frame_decoded_time ||
now > is->viddec.first_frame_decoded_time + 2000) {
is->viddec.first_frame_decoded = 1;
} else {
/* video pipeline is not ready yet */
return -1;
}
}
reload:
do {
#if defined(_WIN32) || defined(__APPLE__)
while (frame_queue_nb_remaining(&is->sampq) == 0) {
if ((av_gettime_relative() - ffp->audio_callback_time) > 1000000LL * is->audio_hw_buf_size / is->audio_tgt.bytes_per_sec / 2)
return -1;
av_usleep (1000);
}
#endif
if (!(af = frame_queue_peek_readable(&is->sampq)))
return -1;
frame_queue_next(&is->sampq);
} while (af->serial != is->audioq.serial);
data_size = av_samples_get_buffer_size(NULL, af->frame->channels,
af->frame->nb_samples,
af->frame->format, 1);
dec_channel_layout =
(af->frame->channel_layout && af->frame->channels == av_get_channel_layout_nb_channels(af->frame->channel_layout)) ?
af->frame->channel_layout : av_get_default_channel_layout(af->frame->channels);
wanted_nb_samples = synchronize_audio(is, af->frame->nb_samples);
if (af->frame->format != is->audio_src.fmt ||
dec_channel_layout != is->audio_src.channel_layout ||
af->frame->sample_rate != is->audio_src.freq ||
(wanted_nb_samples != af->frame->nb_samples && !is->swr_ctx)) {
AVDictionary *swr_opts = NULL;
swr_free(&is->swr_ctx);
is->swr_ctx = swr_alloc_set_opts(NULL,
is->audio_tgt.channel_layout, is->audio_tgt.fmt, is->audio_tgt.freq,
dec_channel_layout, af->frame->format, af->frame->sample_rate,
0, NULL);
if (!is->swr_ctx) {
av_log(NULL, AV_LOG_ERROR,
"Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",
af->frame->sample_rate, av_get_sample_fmt_name(af->frame->format), af->frame->channels,
is->audio_tgt.freq, av_get_sample_fmt_name(is->audio_tgt.fmt), is->audio_tgt.channels);
return -1;
}
av_dict_copy(&swr_opts, ffp->swr_opts, 0);
if (af->frame->channel_layout == AV_CH_LAYOUT_5POINT1_BACK)
av_opt_set_double(is->swr_ctx, "center_mix_level", ffp->preset_5_1_center_mix_level, 0);
av_opt_set_dict(is->swr_ctx, &swr_opts);
av_dict_free(&swr_opts);
if (swr_init(is->swr_ctx) < 0) {
av_log(NULL, AV_LOG_ERROR,
"Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",
af->frame->sample_rate, av_get_sample_fmt_name(af->frame->format), af->frame->channels,
is->audio_tgt.freq, av_get_sample_fmt_name(is->audio_tgt.fmt), is->audio_tgt.channels);
swr_free(&is->swr_ctx);
return -1;
}
is->audio_src.channel_layout = dec_channel_layout;
is->audio_src.channels = af->frame->channels;
is->audio_src.freq = af->frame->sample_rate;
is->audio_src.fmt = af->frame->format;
}
if (is->swr_ctx) {
const uint8_t **in = (const uint8_t **)af->frame->extended_data;
uint8_t **out = &is->audio_buf1;
int out_count = (int)((int64_t)wanted_nb_samples * is->audio_tgt.freq / af->frame->sample_rate + 256);
int out_size = av_samples_get_buffer_size(NULL, is->audio_tgt.channels, out_count, is->audio_tgt.fmt, 0);
int len2;
if (out_size < 0) {
av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size() failed\n");
return -1;
}
if (wanted_nb_samples != af->frame->nb_samples) {
if (swr_set_compensation(is->swr_ctx, (wanted_nb_samples - af->frame->nb_samples) * is->audio_tgt.freq / af->frame->sample_rate,
wanted_nb_samples * is->audio_tgt.freq / af->frame->sample_rate) < 0) {
av_log(NULL, AV_LOG_ERROR, "swr_set_compensation() failed\n");
return -1;
}
}
av_fast_malloc(&is->audio_buf1, &is->audio_buf1_size, out_size);
if (!is->audio_buf1)
return AVERROR(ENOMEM);
len2 = swr_convert(is->swr_ctx, out, out_count, in, af->frame->nb_samples);
if (len2 < 0) {
av_log(NULL, AV_LOG_ERROR, "swr_convert() failed\n");
return -1;
}
if (len2 == out_count) {
av_log(NULL, AV_LOG_WARNING, "audio buffer is probably too small\n");
if (swr_init(is->swr_ctx) < 0)
swr_free(&is->swr_ctx);
}
is->audio_buf = is->audio_buf1;
int bytes_per_sample = av_get_bytes_per_sample(is->audio_tgt.fmt);
resampled_data_size = len2 * is->audio_tgt.channels * bytes_per_sample;
#if defined(__ANDROID__)
if (ffp->soundtouch_enable && ffp->pf_playback_rate != 1.0f && !is->abort_request) {
av_fast_malloc(&is->audio_new_buf, &is->audio_new_buf_size, out_size * translate_time);
for (int i = 0; i < (resampled_data_size / 2); i++)
{
is->audio_new_buf[i] = (is->audio_buf1[i * 2] | (is->audio_buf1[i * 2 + 1] << 8));
}
int ret_len = ijk_soundtouch_translate(is->handle, is->audio_new_buf, (float)(ffp->pf_playback_rate), (float)(1.0f/ffp->pf_playback_rate),
resampled_data_size / 2, bytes_per_sample, is->audio_tgt.channels, af->frame->sample_rate);
if (ret_len > 0) {
is->audio_buf = (uint8_t*)is->audio_new_buf;
resampled_data_size = ret_len;
} else {
translate_time++;
goto reload;
}
}
#endif
} else {
is->audio_buf = af->frame->data[0];
resampled_data_size = data_size;
}
audio_clock0 = is->audio_clock;
/* update the audio clock with the pts */
if (!isnan(af->pts))
is->audio_clock = af->pts + (double) af->frame->nb_samples / af->frame->sample_rate;
else
is->audio_clock = NAN;
is->audio_clock_serial = af->serial;
#ifdef FFP_SHOW_AUDIO_DELAY
{
static double last_clock;
printf("audio: delay=%0.3f clock=%0.3f clock0=%0.3f\n",
is->audio_clock - last_clock,
is->audio_clock, audio_clock0);
last_clock = is->audio_clock;
}
#endif
if (!is->auddec.first_frame_decoded) {
ALOGD("avcodec/Audio: first frame decoded\n");
ffp_notify_msg1(ffp, FFP_MSG_AUDIO_DECODED_START);
is->auddec.first_frame_decoded_time = SDL_GetTickHR();
is->auddec.first_frame_decoded = 1;
}
return resampled_data_size;
}这段代码的逻辑也很简单就是从sampq中获取的数据，然后进行重采样转换成我们设备能使用的数据，然后返给调用函数。
这样整个流程就明白了。