若有需要请查看前面章节分析:
【一】Android MediaRecorder整体架构源码浅析
【二】Android MediaRecorder C++底层架构音视频处理过程和音视频同步源码分析
【三】Android MediaRecorder C++底层架构音视频处理过程和音视频同步源码分析
分析此前未分析完的queue->pushBuffer(mbuf);:
首先分析queue变量的来历:Mutexed::Locked queue(mQueue);
因此其实就是加锁后的mQueue变量,而该变量就是Queue类型的数据:
如下结构:
struct Queue {
Queue()
: mReadPendingSince(0),
mPaused(false),
mPulling(false) {
}
int64_t mReadPendingSince;
bool mPaused;
bool mPulling;
// 非常重要的读取缓冲区数据集合,用来存储此前分析的read之后的音视频原始数据
Vector<MediaBuffer *> mReadBuffers;
void flush();
// if queue is empty, return false and set *|buffer| to NULL . Otherwise, pop
// buffer from front of the queue, place it into *|buffer| and return true.
bool readBuffer(MediaBuffer **buffer);
// add a buffer to the back of the queue
void pushBuffer(MediaBuffer *mbuf);
};
其实现为:
void MediaCodecSource::Puller::Queue::pushBuffer(MediaBuffer *mbuf) {
// 即就是将mbuf放入到mReadBuffers缓冲区集合中
mReadBuffers.push_back(mbuf);
}
那么接下来就要分析Puller的【kWhatPull】事件中最后一个重要的处理,如下:
/** 然后通知mNotify即之前的携带有【kWhatPullerNotify】事件通知发送给MediaCodecSource的onMessageReceived()方法去接收已经获取到的原始音视频数据,进行编码。下面会继续分析该方法的处理流程的
**/
mNotify->post();
因此查看MediaCodecSource::onMessageReceived()该方法的【kWhatPullerNotify】事件处理如下:
case kWhatPullerNotify:
{
int32_t eos = 0;
if (msg->findInt32("eos", &eos) && eos) {
// 如果是已经没有数据了,那么发出End Of Stream事件回调给上层一直到java层,
// 并做各种的资源释放,停止录制等操作
ALOGV("puller (%s) reached EOS", mIsVideo ? "video" : "audio");
signalEOS();
break;
}
if (mEncoder == NULL) {
ALOGV("got msg '%s' after encoder shutdown.", msg->debugString().c_str());
break;
}
// 下面分析该方法
feedEncoderInputBuffers();
break;
}
分析:feedEncoderInputBuffers();如下:
status_t MediaCodecSource::feedEncoderInputBuffers() {
// 从Puller读到的buffer数据缓存
MediaBuffer* mbuf = NULL;
/**
当可用的编码输入缓冲区索引可用时,并且能够读取到音视频原始数据时则进行编码等处理。下面会进行分析readBuffer方法处理流程
**/
while (!mAvailEncoderInputIndices.empty() && mPuller->readBuffer(&mbuf)) {
// 注意此处是可用编码缓冲区索引值,而不是缓冲区内存地址
size_t bufferIndex = *mAvailEncoderInputIndices.begin();
mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin());
int64_t timeUs = 0ll;
uint32_t flags = 0;
size_t size = 0;
if (mbuf != NULL) {
CHECK(mbuf->meta_data()->findInt64(kKeyTime, &timeUs));
if (mFirstSampleSystemTimeUs < 0ll) {
mFirstSampleSystemTimeUs = systemTime() / 1000;
if (mPausePending) {
mPausePending = false;
onPause(mFirstSampleSystemTimeUs);
mbuf->release();
mAvailEncoderInputIndices.push_back(bufferIndex);
return OK;
}
}
mInputBufferTimeOffsetUs = AVUtils::get()->overwriteTimeOffset(mIsHFR,
mInputBufferTimeOffsetUs, &mPrevBufferTimestampUs, timeUs, mBatchSize);
timeUs += mInputBufferTimeOffsetUs;
// push decoding time for video, or drift time for audio
if (mIsVideo) {
// 如果是视频将解码时间戳放入缓冲区集合中,此处应该用来设置手机预览时用的
mDecodingTimeQueue.push_back(timeUs);
if (!(mFlags & FLAG_USE_SURFACE_INPUT)) {
mbuf->meta_data()->setInt64(kKeyTime, timeUs);
AVUtils::get()->addDecodingTimesFromBatch(mbuf, mDecodingTimeQueue);
}
} else {
// 省略部分代码
}
/**
此处从MediaCodec类的编码器encoder实例中获取一个可用的输入给编码器进行编码的输入缓冲区,根据缓冲区的索引进行获取。
**/
sp<MediaCodecBuffer> inbuf;
status_t err = mEncoder->getInputBuffer(bufferIndex, &inbuf);
if (err != OK || inbuf == NULL || inbuf->data() == NULL
|| mbuf->data() == NULL || mbuf->size() == 0) {
mbuf->release();
signalEOS(err);
break;
}
size = mbuf->size();
// 数据正常时将Puller读取到的mbuf中的数据拷贝一份给MediaCodec中的inbuf中
memcpy(inbuf->data(), mbuf->data(), size);
if (mIsVideo) {
// 如果是视频,当拷贝完底层数据时视频编码器将释放MediaBuffer。此处理不过多分析。
// video encoder will release MediaBuffer when done
// with underlying data.
inbuf->setMediaBufferBase(mbuf);
} else {
mbuf->release();
}
} else {
flags = MediaCodec::BUFFER_FLAG_EOS;
}
/** 将根据buffer索引通知MediaCodec把刚拷贝的未编码数据进行入队,进入输入缓冲区队列进行编码
**/
status_t err = mEncoder->queueInputBuffer(
bufferIndex, 0, size, timeUs, flags);
if (err != OK) {
return err;
}
}
return OK;
}
分析上面的Puller的readBuffer方法:
bool MediaCodecSource::Puller::readBuffer(MediaBuffer **mbuf) {
Mutexed<Queue>::Locked queue(mQueue);
return queue->readBuffer(mbuf);
}
然后调用
bool MediaCodecSource::Puller::Queue::readBuffer(MediaBuffer **mbuf) {
if (mReadBuffers.empty()) {
*mbuf = NULL;
return false;
}
*mbuf = *mReadBuffers.begin();
mReadBuffers.erase(mReadBuffers.begin());
return true;
}
如此也就有此前的pushBuffer联系起来了,一个push数据,一个read数据。
即就是从前面音视频放入到队列集合数据中的buffer数据取出来第一个buffer数据进行送入编码器进行编码等操作。
继续分析如下:
/** 【MediaCodec : public AHandler】
此处从MediaCodec类的编码器encoder实例中获取一个可用的输入给编码器进行编码的输入缓冲区,根据缓冲区的索引进行获取。
**/
mEncoder->getInputBuffer(bufferIndex, &inbuf);如下:
status_t MediaCodec::getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
sp<AMessage> format;
// 根据输入缓冲区的端口索引获取对应的buffer缓冲区
return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
}
接着:
status_t MediaCodec::getBufferAndFormat(
size_t portIndex, size_t index,
sp<MediaCodecBuffer> *buffer, sp<AMessage> *format) {
// 使用互斥体而不是上下文切换
// use mutex instead of a context switch
// 省略部分代码
buffer->clear();
format->clear();
// 是否正在执行:mState == STARTED || mState == FLUSHED;
if (!isExecuting()) {
ALOGE("getBufferAndFormat - not executing");
return INVALID_OPERATION;
}
// we do not want mPortBuffers to change during this section
// we also don't want mOwnedByClient to change during this
Mutex::Autolock al(mBufferLock);
// 根据缓冲区端口索引获取到一个缓冲区信息列表
std::vector<BufferInfo> &buffers = mPortBuffers[portIndex];
if (index >= buffers.size()) {
ALOGE("getBufferAndFormat - trying to get buffer with "
"bad index (index=%zu buffer_size=%zu)", index, buffers.size());
return INVALID_OPERATION;
}
// 然后根据缓冲区索引在输入缓冲区列表中找到一个缓冲区buffer
const BufferInfo &info = buffers[index];
if (!info.mOwnedByClient) {
ALOGE("getBufferAndFormat - invalid operation "
"(the index %zu is not owned by client)", index);
return INVALID_OPERATION;
}
// 然后将其赋值返回上层可以使用的一个空的输入缓冲区buffer
*buffer = info.mData;
*format = info.mData->format();
return OK;
}
接下来分析将原始音视频数据入队列进行编码操作:
/** 将根据buffer索引通知MediaCodec把刚拷贝的未编码数据进行入队,进入输入缓冲区队列进行编码
**/
status_t err = mEncoder->queueInputBuffer(
bufferIndex, 0, size, timeUs, flags);
status_t MediaCodec::queueInputBuffer(
size_t index,
size_t offset,
size_t size,
int64_t presentationTimeUs,
uint32_t flags,
AString *errorDetailMsg) {
// 发送一个【kWhatQueueInputBuffer】开始解码事件
sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
msg->setSize("index", index);
msg->setSize("offset", offset);
msg->setSize("size", size);
msg->setInt64("timeUs", presentationTimeUs);
msg->setInt32("flags", flags);
msg->setPointer("errorDetailMsg", errorDetailMsg);
sp<AMessage> response;
return PostAndAwaitResponse(msg, &response);
}
分析:【kWhatQueueInputBuffer】开始解码事件:
在MediaCodec::onMessageReceived()方法中处理如下:
case kWhatQueueInputBuffer:
{
sp<AReplyToken> replyID;
CHECK(msg->senderAwaitsResponse(&replyID));
if (!isExecuting()) {
PostReplyWithError(replyID, INVALID_OPERATION);
break;
} else if (mFlags & kFlagStickyError) {
PostReplyWithError(replyID, getStickyError());
break;
}
// 执行此方法
status_t err = onQueueInputBuffer(msg);
PostReplyWithError(replyID, err);
break;
}
分析:onQueueInputBuffer(msg);如下:
status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
size_t index;
size_t offset;
size_t size;
int64_t timeUs;
uint32_t flags;
CHECK(msg->findSize("index", &index));
CHECK(msg->findSize("offset", &offset));
CHECK(msg->findInt64("timeUs", &timeUs));
CHECK(msg->findInt32("flags", (int32_t *)&flags));
const CryptoPlugin::SubSample *subSamples;
size_t numSubSamples;
const uint8_t *key;
const uint8_t *iv;
// 默认不加密编码
CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
// We allow the simpler queueInputBuffer API to be used even in
// secure mode, by fabricating a single unencrypted subSample.
CryptoPlugin::SubSample ss;
CryptoPlugin::Pattern pattern;
// 省略部分代码
/** 根据当前传入需要被编码的缓冲区索引在端口缓冲槽中查找到该buffer数据,默认是两个缓冲槽如定义: std::vector<BufferInfo> mPortBuffers[2];
**/
BufferInfo *info = &mPortBuffers[kPortIndexInput][index];
if (info->mData == nullptr || !info->mOwnedByClient) {
return -EACCES;
}
if (offset + size > info->mData->capacity()) {
if ( ((int)size < 0) && !(flags & BUFFER_FLAG_EOS)) {
size = 0;
ALOGD("EOS, reset size to zero");
} else {
return -EINVAL;
}
}
info->mData->setRange(offset, size);
info->mData->meta()->setInt64("timeUs", timeUs);
if (flags & BUFFER_FLAG_EOS) {
info->mData->meta()->setInt32("eos", true);
}
if (flags & BUFFER_FLAG_CODECCONFIG) {
info->mData->meta()->setInt32("csd", true);
}
// 得到需要编码的原始buffer数据结构
sp<MediaCodecBuffer> buffer = info->mData;
status_t err = OK;
if (hasCryptoOrDescrambler()) {
// 省略部分代码
} else {
// 默认不加密编码执行此处,将需要编码的数据送入
err = mBufferChannel->queueInputBuffer(buffer);
}
// 省略部分代码
return err;
}
分析:
// 默认不加密编码执行此处,将需要编码的数据送入
err = mBufferChannel->queueInputBuffer(buffer);
首先得找到mBufferChannel如何来的,如下:
mBufferChannel = mCodec->getBufferChannel();
而mCodec根据代码追踪是ACodec的实例。
std::shared_ptr<BufferChannelBase> ACodec::getBufferChannel() {
if (!mBufferChannel) {
mBufferChannel = std::make_shared<ACodecBufferChannel>(
new AMessage(kWhatInputBufferFilled, this),
new AMessage(kWhatOutputBufferDrained, this));
}
return mBufferChannel;
}
该方法返回了【ACodecBufferChannel】对象实例,并且它内部持有【kWhatInputBufferFilled】和【kWhatOutputBufferDrained】事件通知ACodec。
即分析ACodecBufferChannel的queueInputBuffer(buffer);方法即可:如下
status_t ACodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
if (mDealer != nullptr) {
return -ENOSYS;
}
std::shared_ptr<const std::vector<const BufferInfo>> array(
std::atomic_load(&mInputBuffers));
BufferInfoIterator it = findClientBuffer(array, buffer);
if (it == array->end()) {
return -ENOENT;
}
ALOGV("queueInputBuffer #%d", it->mBufferId);
/**有两个对应的变量
const sp<AMessage> mInputBufferFilled;
const sp<AMessage> mOutputBufferDrained;
即对应【kWhatInputBufferFilled】和【kWhatOutputBufferDrained】事件给ACodec进行处理
**/
sp<AMessage> msg = mInputBufferFilled->dup();
msg->setObject("buffer", it->mCodecBuffer);
msg->setInt32("buffer-id", it->mBufferId);
// 即发送【kWhatInputBufferFilled】事件给ACodec进行处理
msg->post();
return OK;
}
分析:发送【kWhatInputBufferFilled】事件给ACodec进行处理
仔细看头文件中源码如下:
// AHierarchicalStateMachine implements the message handling
virtual void onMessageReceived(const sp<AMessage> &msg) {
handleMessage(msg);
}
然后调用父类中的:
void AHierarchicalStateMachine::handleMessage(const sp<AMessage> &msg) {
sp<AState> save = mState;
sp<AState> cur = mState;
// 此处的onMessageReceived(msg)则是处理的方法,
while (cur != NULL && !cur->onMessageReceived(msg)) {
// If you claim not to have handled the message you shouldn't
// have called setState...
CHECK(save == mState);
cur = cur->parentState();
}
if (cur != NULL) {
return;
}
ALOGW("Warning message %s unhandled in root state.",
msg->debugString().c_str());
}
上面代码最终会调用:
ACodec::BaseState::onMessageReceived(const sp<AMessage> &msg)方法
case kWhatInputBufferFilled:
{
onInputBufferFilled(msg);
break;
}
case kWhatOutputBufferDrained:
{
onOutputBufferDrained(msg);
break;
}
因此调用了onInputBufferFilled(msg);:
关键代码:
sp<MediaCodecBuffer> buffer;
BufferInfo *info = mCodec->findBufferByID(kPortIndexInput, bufferID);
BufferInfo::Status status = BufferInfo::getSafeStatus(info);
info->mStatus = BufferInfo::OWNED_BY_US;
info->mData = buffer;
case IOMX::kPortModePresetByteBuffer:
case IOMX::kPortModePresetANWBuffer:
case IOMX::kPortModePresetSecureBuffer:
{
// 此处已经与OpenMAX取得联系了,即通过OpenMAX来进行真正的编解码处理
err2 = mCodec->mOMXNode->emptyBuffer(
bufferID, info->mCodecData, flags, timeUs, info->mFenceFd);
}
break;
#ifndef OMX_ANDROID_COMPILE_AS_32BIT_ON_64BIT_PLATFORMS
case IOMX::kPortModeDynamicNativeHandle:
if (info->mCodecData->size() >= sizeof(VideoNativeHandleMetadata)) {
VideoNativeHandleMetadata *vnhmd =
(VideoNativeHandleMetadata*)info->mCodecData->base();
sp<NativeHandle> handle = NativeHandle::create(
vnhmd->pHandle, false /* ownsHandle */);
// 最终调用了OpenMAX的emptyBuffer让其进行读取输入缓冲区的数据进行编码操作
err2 = mCodec->mOMXNode->emptyBuffer(
bufferID, handle, flags, timeUs, info->mFenceFd);
}
break;
case IOMX::kPortModeDynamicANWBuffer:
if (info->mCodecData->size() >= sizeof(VideoNativeMetadata)) {
VideoNativeMetadata *vnmd = (VideoNativeMetadata*)info->mCodecData->base();
sp<GraphicBuffer> graphicBuffer = GraphicBuffer::from(vnmd->pBuffer);
err2 = mCodec->mOMXNode->emptyBuffer(
bufferID, graphicBuffer, flags, timeUs, info->mFenceFd);
}
break;
#endif
因此,目前先了解到编解码都是给到OpenMAX框架层进行真正的编解码处理,不深入分析。
前面Track的start()方法的第二处开启新线程ThreadWrapper的执行未分析:如下:
调用了
// static
void *MPEG4Writer::Track::ThreadWrapper(void *me) {
Track *track = static_cast<Track *>(me);
status_t err = track->threadEntry();
return (void *)(uintptr_t)err;
}
即又调用了:status_t MPEG4Writer::Track::threadEntry()方法,该方法处理非常多,以下列出关键代码分析:
sp<MetaData> meta_data;
status_t err = OK;
MediaBuffer *buffer;
// 开启一个循环读取数据loop处理,该read其实就是MediaCodecSource的read()方法,下面有分析。
while (!mDone && (err = mSource->read(&buffer)) == OK) {
}
接下来分析:MediaCodecSource的read()方法:
status_t MediaCodecSource::read(
MediaBuffer** buffer, const ReadOptions* /* options */) {
Mutexed<Output>::Locked output(mOutput);
*buffer = NULL;
while (output->mBufferQueue.size() == 0 && !output->mEncoderReachedEOS) {
// 此时没有缓冲区编码后的数据就wait等待被唤醒
output.waitForCondition(output->mCond);
}
// 有数据时
if (!output->mEncoderReachedEOS) {
// 将队列中第一个编码后的缓冲区数据提取出来,给Track写入文件使用
*buffer = *output->mBufferQueue.begin();
output->mBufferQueue.erase(output->mBufferQueue.begin());
int64_t timeUs = 0;
(*buffer)->meta_data()->findInt64(kKeyTime, &timeUs);
VTRACE_ASYNC_BEGIN(mIsVideo?"write-video":"write-audio", (int)timeUs);
return OK;
}
return output->mErrorCode;
}
此处的output变量结构为:
Mutexed<Output> mOutput;
struct Output {
Output();
// 存入了MediaBuffer结构数据
List<MediaBuffer*> mBufferQueue;
bool mEncoderReachedEOS;
status_t mErrorCode;
Condition mCond;
};
而该队列mBufferQueue数据是在MediaCodecSource::onMessageReceived(const sp &msg)方法中【kWhatEncoderActivity】事件中写入到缓冲区队列数据中的并且唤起read()处理流程,而该【kWhatEncoderActivity】事件是在MediaCodecSource::initEncoder()方法中处理的:
mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, mReflector);
mEncoder->setCallback(mEncoderActivityNotify);
而mEncoder = MediaCodec::CreateByComponentName(
mCodecLooper, matchingCodecs[ix]);
将该事件交给了MediaCodec类的实例,
status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
sp<AMessage> msg = new AMessage(kWhatSetCallback, this);
msg->setMessage("callback", callback);
sp<AMessage> response;
return PostAndAwaitResponse(msg, &response);
}
然后在MediaCodec的onMessageReceived()方法中找到该事件:
case kWhatSetCallback:
{
sp<AReplyToken> replyID;
CHECK(msg->senderAwaitsResponse(&replyID));
if (mState == UNINITIALIZED
|| mState == INITIALIZING
|| isExecuting()) {
// callback can't be set after codec is executing,
// or before it's initialized (as the callback
// will be cleared when it goes to INITIALIZED)
PostReplyWithError(replyID, INVALID_OPERATION);
break;
}
sp<AMessage> callback;
CHECK(msg->findMessage("callback", &callback));
// 最终非常关键的一句是把该事件作为了一个回调事件缓存起来了
mCallback = callback;
if (mCallback != NULL) {
ALOGI("MediaCodec will operate in async mode");
mFlags |= kFlagIsAsync;
} else {
mFlags &= ~kFlagIsAsync;
}
sp<AMessage> response = new AMessage;
response->postReply(replyID);
break;
}
最终该【kWhatEncoderActivity】事件会在
MediaCodec::onInputBufferAvailable()、
MediaCodec::onOutputBufferAvailable()、
MediaCodec::onOutputFormatChanged()
这是三个事件方法中被触发,这三个事件被触发时机就是底层OpenMAX编解码完成之后调用,然后一层层的返回上来的。
再次分析:
该队列mBufferQueue数据是在MediaCodecSource::onMessageReceived(const sp &msg)方法中【kWhatEncoderActivity】事件中写入到缓冲区队列数据中的并且唤起read()处理流程:
case kWhatEncoderActivity:
关键代码如下:
// 读取到可用有效编码之后的输出缓冲区
sp<MediaCodecBuffer> outbuf;
status_t err = mEncoder->getOutputBuffer(index, &outbuf);
if (err != OK || outbuf == NULL || outbuf->data() == NULL
|| outbuf->size() == 0) {
signalEOS(err);
break;
}
// 然后将其封装成上层需要的MediaBuffer类型结构的数据,但此时还有把【outbuf】中的数据拷贝过来。
MediaBuffer *mbuf = new MediaBuffer(outbuf->size());
sp<MetaData> meta = mbuf->meta_data();
AVUtils::get()->setDeferRelease(meta);
mbuf->setObserver(this);
mbuf->add_ref();
// 拷贝【outbuf】中数据给mbuf
memcpy(mbuf->data(), outbuf->data(), outbuf->size());
{
// 将其拷贝好的编码后的数据放入缓冲区队列中,并将之前的read()处理流程唤醒
Mutexed<Output>::Locked output(mOutput);
output->mBufferQueue.push_back(mbuf);
// 将之前的read()处理流程唤醒,通过信号量机制
output->mCond.signal();
}
// 然后让编码器释放掉输出缓冲区数据的内存,将其交还给编码器再次使用
mEncoder->releaseOutputBuffer(index);
接下来的流程分析请查看:
【五】Android MediaRecorder C++底层架构音视频处理过程和音视频同步源码分析
MediaRecorder本系列章节内容分析已在一两年前分析完成的,当初只是用于分析的笔记记录下来,如今已走上了我向往的音视频开发领域,也调研和参与过音视频相关技术,目前喜爱C++底层音视频技术的开发,偏向于底层的音视频复用/解复用、解码/编码、推流拉流技术等,而当初分析本章内容时音视频技术积累较少,因此若本文中分析有误还请多多指教,谢谢。
过了这么久时间才分享出来的原因是,以往技术知识也向前辈们分享文章有所收获,现如今我也可以有所技术分享给需要的人,也希望可以帮忙到需要的人,技术分享帮助他人的同时也能作为自身技术掌握的总结记录。
以往忙于技术深入和研究未有所分享的技术,往后坚持有质量的技术分享。