Android8.0 Media系统(二)

版权声明：未经本人同意，不得转载 https://blog.csdn.net/u013928208/article/details/81901340

上一篇从MediaPlayer的创建，设置数据源，播放准备三个流程来看Media系统，今天我们继续来看initFromDataSource()函数，将创建数据提取器，数据提取器将读取数据源文件的元数据信息，将每一路数据流的比特率进行累加，分离数据流，音频/视频/字幕分离：设置视频源mVideoTrack ；设置音频源mAudioTrack；分离字幕等。废话少说直接上代码。

1. 数据提取

status_t NuPlayer::GenericSource::initFromDataSource() {
    sp<IMediaExtractor> extractor;
    extractor = MediaExtractor::Create(mDataSource, NULL); //根据数据源创建多媒体提取器
    mFileMeta = extractor->getMetaData(); //获取元数据
    size_t numtracks = extractor->countTracks();//获取数据源的Track数
     for (size_t i = 0; i < numtracks; ++i) {
        sp<IMediaSource> track = extractor->getTrack(i); //获取MediaSource
        sp<MetaData> meta = extractor->getTrackMetaData(i);//获取Track元数据
        ......
        if (!strncasecmp(mime, "audio/", 6)) {
            if (mAudioTrack.mSource == NULL) {
                mAudioTrack.mIndex = i;
                mAudioTrack.mSource = track;
                mAudioTrack.mPackets =
                    new AnotherPacketSource(mAudioTrack.mSource->getFormat());

                if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_VORBIS)) {
                    mAudioIsVorbis = true;
                } else {
                    mAudioIsVorbis = false;
                }

                mMimes.add(String8(mime));
            }
        } else if (!strncasecmp(mime, "video/", 6)) {
            if (mVideoTrack.mSource == NULL) {
                mVideoTrack.mIndex = i;
                mVideoTrack.mSource = track;
                mVideoTrack.mPackets =
                    new AnotherPacketSource(mVideoTrack.mSource->getFormat());

                // video always at the beginning
                mMimes.insertAt(String8(mime), 0);
            }
        }
        //保存每一Track
        mSources.push(track);
        //时间同步
        int64_t durationUs;
        if (meta->findInt64(kKeyDuration, &durationUs)) {
            if (durationUs > mDurationUs) {
                mDurationUs = durationUs;
            }
        }
        //统计比特率
        int32_t bitrate;
        if (totalBitrate >= 0 && meta->findInt32(kKeyBitRate, &bitrate)) {
            totalBitrate += bitrate;
        } else {
            totalBitrate = -1;
        }
    }
    ......
    return OK;
}

frameworks\av\media\libstagefright\MediaExtractor.cpp

// static
sp<IMediaExtractor> MediaExtractor::Create(
        const sp<DataSource> &source, const char *mime) {

    if (!property_get_bool("media.stagefright.extractremote", true)) { //存在本地提取服务
        // local extractor
        return CreateFromService(source, mime);
    } else {
        // remote extractor
        sp<IBinder> binder = defaultServiceManager()->getService(String16("media.extractor"));
        if (binder != 0) {
            //获取多媒体提取器服务
            sp<IMediaExtractorService> mediaExService(interface_cast<IMediaExtractorService>(binder));
            //根据source返回具体的数据提取器
            sp<IMediaExtractor> ex = mediaExService->makeExtractor(source->asIDataSource(), mime);
            return ex;
        } else {
            return NULL;
        }
    }
    return NULL;
}

frameworks\av\services\mediaextractor\MediaExtractorService.cpp

sp<IMediaExtractor> MediaExtractorService::makeExtractor(
        const sp<IDataSource> &remoteSource, const char *mime) {
    //获取localSource 
    sp<DataSource> localSource = DataSource::CreateFromIDataSource(remoteSource);
    //调用 CreateFromService
    sp<IMediaExtractor> ret = MediaExtractor::CreateFromService(localSource, mime);

    ......

    return ret;
}

按MIME类型返回具体类型的多媒体数据提取器

sp<MediaExtractor> MediaExtractor::CreateFromService(
        const sp<DataSource> &source, const char *mime) {

    // initialize source decryption if needed
    source->DrmInitialization(nullptr /* mime */); //初始化数字版权

    sp<AMessage> meta;
    ......
    MediaExtractor *ret = NULL;
    if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG4)
            || !strcasecmp(mime, "audio/mp4")) {
        ret = new MPEG4Extractor(source); //MP4数据提取器
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_MPEG)) {
        ret = new MP3Extractor(source, meta); //MP3数据提取器
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AMR_NB)
            || !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AMR_WB)) {
        ret = new AMRExtractor(source);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_FLAC)) {
        ret = new FLACExtractor(source);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_WAV)) {
        ret = new WAVExtractor(source);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_OGG)) {
        ret = new OggExtractor(source);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MATROSKA)) {
        ret = new MatroskaExtractor(source);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG2TS)) {
        ret = new MPEG2TSExtractor(source);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC_ADTS)) {
        ret = new AACExtractor(source, meta);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG2PS)) {
        ret = new MPEG2PSExtractor(source);
    } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_MIDI)) {
        ret = new MidiExtractor(source);
    }
    return ret;
}

//获取MediaSource,成员函数parseChunk()控制从MPEG4DataSource解析元数据信息

sp<IMediaSource> MPEG4Extractor::getTrack(size_t index) {
    status_t err;
    if ((err = readMetaData()) != OK) {
        return NULL;
    }

    Track *track = mFirstTrack;
    while (index > 0) {
        track = track->next;
        --index;
    }

    Trex *trex = NULL;
    int32_t trackId;
    if (track->meta->findInt32(kKeyTrackID, &trackId)) {
        for (size_t i = 0; i < mTrex.size(); i++) {
            Trex *t = &mTrex.editItemAt(i);
            if (t->track_ID == (uint32_t) trackId) {
                trex = t;
                break;
            }
        }
    } else {
        ALOGE("b/21657957");
        return NULL;
    }
    //新建了一个多媒体源
    sp<MPEG4Source> source =  new MPEG4Source(this,
            track->meta, mDataSource, track->timescale, track->sampleTable,
            mSidxEntries, trex, mMoofOffset);
    if (source->init() != OK) {
        return NULL;
    }
    return source;
}

2. 数据解码

到目前为止MediaPlayer执行完了setDataSource(path), prepare()两个函数，为多媒体播放准备好了数据，下一步将调用start()函数，对提取出来的多媒体元数据进行解码输出。我们直接跳过Java层面繁琐的调用跳转，直接从NuPlayer的start()函数开始，Amazing !

void NuPlayer::start() {
    (new AMessage(kWhatStart, this))->post();
}

经过AHandler消息通讯机制，在如下出继续执行onStart()

    case kWhatStart:
        {
            ALOGV("kWhatStart");
            if (mStarted) {
                // do not resume yet if the source is still buffering
                if (!mPausedForBuffering) {
                    onResume();
                }
            } else {
                onStart();
            }
            mPausedByClient = false;
            break;
        }

void NuPlayer::onStart(int64_t startPositionUs, MediaPlayerSeekMode mode) {

    ......

    bool hasAudio = (mSource->getFormat(true /* audio */) != NULL);
    bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); //获取音频元数据
    //音频是否需要需要解码
    mOffloadAudio =
        canOffloadStream(audioMeta, hasVideo, mSource->isStreaming(), streamType)
                && (mPlaybackSettings.mSpeed == 1.f && mPlaybackSettings.mPitch == 1.f);
    sp<AMessage> notify = new AMessage(kWhatRendererNotify, this);
    ++mRendererGeneration;
    notify->setInt32("generation", mRendererGeneration);
    mRenderer = new Renderer(mAudioSink, notify, flags); //渲染
    mRendererLooper = new ALooper;
    mRendererLooper->setName("NuPlayerRenderer");
    mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
    mRendererLooper->registerHandler(mRenderer);
    float rate = getFrameRate(); //获取帧率
    if (rate > 0) {
        mRenderer->setVideoFrameRate(rate); //设置给渲染器
    }

    if (mVideoDecoder != NULL) {
        mVideoDecoder->setRenderer(mRenderer);
    }
    if (mAudioDecoder != NULL) {
        mAudioDecoder->setRenderer(mRenderer);
    }
    //扫描
    postScanSources();
}

void NuPlayer::postScanSources() {
    sp<AMessage> msg = new AMessage(kWhatScanSources, this);
    msg->setInt32("generation", mScanSourcesGeneration);
    msg->post();

    mScanSourcesPending = true;
}

AHandler消息循环到此处

 case kWhatScanSources:
        {
            // initialize video before audio because successful initialization of
            // video may change deep buffer mode of audio.
            if (mSurface != NULL) {
                //初始化解码器
                if (instantiateDecoder(false, &mVideoDecoder) == -EWOULDBLOCK) {
                    rescan = true;
                }
            }

            // Don't try to re-open audio sink if there's an existing decoder.
            if (mAudioSink != NULL && mAudioDecoder == NULL) {
                if (instantiateDecoder(true, &mAudioDecoder) == -EWOULDBLOCK) {
                    rescan = true;
                }
            }

            ......

            if (rescan) { //延时重扫
                msg->post(100000ll);
                mScanSourcesPending = true;
            }
            break;
        }

初始化解码器

status_t NuPlayer::instantiateDecoder(
        bool audio, sp<DecoderBase> *decoder, bool checkAudioModeChange) 

    ......

    if (audio) { //音频
        sp<AMessage> notify = new AMessage(kWhatAudioNotify, this);
        ++mAudioDecoderGeneration;
        notify->setInt32("generation", mAudioDecoderGeneration);

        if (checkAudioModeChange) {
            determineAudioModeChange(format);
        }
        if (mOffloadAudio) {

            mSource->setOffloadAudio(true /* offload */); //设置离载音频

            const bool hasVideo = (mSource->getFormat(false /*audio */) != NULL);
            format->setInt32("has-video", hasVideo);
            *decoder = new DecoderPassThrough(notify, mSource, mRenderer); //创建解码器
            ALOGV("instantiateDecoder audio DecoderPassThrough  hasVideo: %d", hasVideo);
        } else {
            mSource->setOffloadAudio(false /* offload */);
            *decoder = new Decoder(notify, mSource, mPID, mUID, mRenderer); //创建解码器
            ALOGV("instantiateDecoder audio Decoder");
        }
        mAudioDecoderError = false;
    } else { //视频
        sp<AMessage> notify = new AMessage(kWhatVideoNotify, this);
        ++mVideoDecoderGeneration;
        notify->setInt32("generation", mVideoDecoderGeneration);

        *decoder = new Decoder( //解码器
                notify, mSource, mPID, mUID, mRenderer, mSurface, mCCDecoder);
        mVideoDecoderError = false;

        // enable FRC if high-quality AV sync is requested, even if not
        // directly queuing to display, as this will even improve textureview
        // playback.
        {
            if (property_get_bool("persist.sys.media.avsync", false)) {
                format->setInt32("auto-frc", 1);
            }
        }
    }
    (*decoder)->init(); //解码器初始化

    // Modular DRM
    if (mIsDrmProtected) { //是否受数字版权保护
        format->setPointer("crypto", mCrypto.get());
    }

    (*decoder)->configure(format); //配置解码器

    .....

    return OK;
}

frameworks\av\media\libmediaplayerservice\nuplayer\NuPlayerDecoder.cpp
配置NuPlayer解码器，将调用StageFright的MediaCodec进行解码

void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {

    ......
    mComponentName = mime;
    mComponentName.append(" decoder");
   //创建解码器
    mCodec = MediaCodec::CreateByType(
            mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid);
    int32_t secure = 0;
    if (format->findInt32("secure", &secure) && secure != 0) {
        if (mCodec != NULL) {
            mCodec->getName(&mComponentName); //设置组件
            mComponentName.append(".secure");
            mCodec->release();
            //按组件名构建
            mCodec = MediaCodec::CreateByComponentName(
                    mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid);
        }
    }

    mIsSecure = secure;

    mCodec->getName(&mComponentName);

    // Modular DRM
    void *pCrypto;
    if (!format->findPointer("crypto", &pCrypto)) {
        pCrypto = NULL;
    }
    sp<ICrypto> crypto = (ICrypto*)pCrypto;
    // non-encrypted source won't have a crypto
    mIsEncrypted = (crypto != NULL);
    // configure is called once; still using OR in case the behavior changes.
    mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted;
    ALOGV("onConfigure mCrypto: %p (%d)  mIsSecure: %d",
            crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure);
    //配置解码器
    err = mCodec->configure(
            format, mSurface, crypto, 0 /* flags */);

    ......

    if (!mIsAudio) { //视频
        int32_t width, height;
        if (mOutputFormat->findInt32("width", &width)
                && mOutputFormat->findInt32("height", &height)) {
            mStats->setInt32("width", width);
            mStats->setInt32("height", height);
        }
    }

    sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
    mCodec->setCallback(reply); //处理mCodec返回的消息，数据的输入输出等

    err = mCodec->start(); //启动解码
    ......
}

frameworks\av\media\libmediaplayerservice\nuplayer\NuPlayerDecoder.cpp
NuPlayerDecoder接收来自MediaCodec的解码消息

 case MediaCodec::CB_OUTPUT_AVAILABLE:
                {
                    ......
                    handleAnOutputBuffer(index, offset, size, timeUs, flags);
                    break;
                }

处理来自多媒体解码器的渲染输出

bool NuPlayer::Decoder::handleAnOutputBuffer(......) {
//    CHECK_LT(bufferIx, mOutputBuffers.size());
    sp<MediaCodecBuffer> buffer;
    mCodec->getOutputBuffer(index, &buffer);

    ......

    mOutputBuffers.editItemAt(index) = buffer;

    buffer->setRange(offset, size);
    buffer->meta()->clear();
    buffer->meta()->setInt64("timeUs", timeUs);

    bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
    // we do not expect CODECCONFIG or SYNCFRAME for decoder
   //发送kWhatRenderBuffer消息
    sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
    reply->setSize("buffer-ix", index);
    reply->setInt32("generation", mBufferGeneration);

    ......

    mNumFramesTotal += !mIsAudio;

    // wait until 1st frame comes out to signal resume complete
    notifyResumeCompleteIfNecessary();

    if (mRenderer != NULL) {
         //把Buffer送到Renderer
        mRenderer->queueBuffer(mIsAudio, buffer, reply);
        if (eos && !isDiscontinuityPending()) {
            mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
        }
    }

    return true;
}

收到渲染输出消息

case kWhatRenderBuffer:
        {
            if (!isStaleReply(msg)) {
                onRenderBuffer(msg);
            }
            break;
        }

渲染输出

void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {

    if (!mIsAudio) { //视频
        int64_t timeUs;
        sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx];
        buffer->meta()->findInt64("timeUs", &timeUs);
        //字幕显示
        if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
            mCCDecoder->display(timeUs);
        }
    }

    if (msg->findInt32("render", &render) && render) {
        int64_t timestampNs;
        CHECK(msg->findInt64("timestampNs", &timestampNs));
        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); //渲染输出
    } else {
        mNumOutputFramesDropped += !mIsAudio;
        err = mCodec->releaseOutputBuffer(bufferIx);
    }
    ......
}

收到来自解码器的多媒体数据输入请求

 case MediaCodec::CB_INPUT_AVAILABLE:
                {
                    int32_t index;
                    CHECK(msg->findInt32("index", &index));

                    handleAnInputBuffer(index);
                    break;
                }

处理多媒体数据输入请求

bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
    ......   
    onRequestInputBuffers();
    return true;
}

抓取多媒体数据

bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
        .....
     // attempt to queue EOS
        status_t err = mCodec->queueInputBuffer(
                bufferIx,
                0,
                0,
                0,
                MediaCodec::BUFFER_FLAG_EOS);
        ......
    return true;
}

循环的发消息请求数据

void NuPlayer::DecoderBase::onRequestInputBuffers() {
    if (mRequestInputBuffersPending) {
        return;
    }

    // doRequestBuffers() return true if we should request more data
    if (doRequestBuffers()) {
        mRequestInputBuffersPending = true;
　　　//注意这里，会自己循环调用，不停地request
        sp<AMessage> msg = new AMessage(kWhatRequestInputBuffers, this);
        msg->post(10 * 1000ll);
    }
}

请求缓冲数据

bool NuPlayer::Decoder::doRequestBuffers() {
    ......
    while (err == OK && !mDequeuedInputBuffers.empty()) {
        size_t bufferIx = *mDequeuedInputBuffers.begin();
        sp<AMessage> msg = new AMessage();
        msg->setSize("buffer-ix", bufferIx);
        err = fetchInputData(msg);
        ......
    }

    return err == -EWOULDBLOCK
            && mSource->feedMoreTSData() == OK;
}

从DataSource取数据

status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
    sp<ABuffer> accessUnit;
    bool dropAccessUnit;
    do {
        //在这里调用source的方法，建立起了联系
        status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
    ......
｝while(...)
}

回到启动解码函数 err = mCodec->start();

  case kWhatStart:
        {
            sp<AReplyToken> replyID;
            CHECK(msg->senderAwaitsResponse(&replyID));

            if (mState == FLUSHED) {
                setState(STARTED);
                if (mHavePendingInputBuffers) {
                    onInputBufferAvailable();
                    mHavePendingInputBuffers = false;
                }
                mCodec->signalResume();
                PostReplyWithError(replyID, OK);
                break;
            } else if (mState != CONFIGURED) {
                PostReplyWithError(replyID, INVALID_OPERATION);
                break;
            }

            mReplyID = replyID;
            setState(STARTING);

            mCodec->initiateStart();
            break;
        }

frameworks\av\media\libstagefright\ACodec.cpp

void ACodec::initiateStart() {
    (new AMessage(kWhatStart, this))->post();
}

调用openmax解码

void ACodec::LoadedState::onStart() {
    ALOGV("onStart");

    status_t err = mCodec->mOMXNode->sendCommand(OMX_CommandStateSet, OMX_StateIdle);
    if (err != OK) {
        mCodec->signalError(OMX_ErrorUndefined, makeNoSideEffectStatus(err));
    } else {
        mCodec->changeState(mCodec->mLoadedToIdleState);
    }
}

关于openmax解码部分暂且不做分析，我们从一个Demo开始分析了MediaPlayer的整个流程，对多媒体的分析暂且就到这里，以后涉及到Surface系统，Camera系统再来结合具体情境分析。下一个主题我们去跟踪Camera子系统。从近两年Google在相机方面的改进来看，Camera子系统势必会复杂一点，还是那句话：show me the code !