MediaCodec解码得到Image
- 对于仅仅需要将视频切分为一帧一帧并保存为图片的用户来说，使用这种方法比bigflake的方法会快10倍左右，因为没有OpenGL渲染，以及转换为Bitmap的开销。而对于需要获得视频帧YUV格式数据的用户来说，这种方法能够直接得到YUV格式数据，中间没有数学运算，不会出现不必要的精度损失，而且，也是效率最高的。
package com.vision.opengl.decoder;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.ImageFormat;
import android.graphics.Rect;
import android.graphics.YuvImage;
import android.media.Image;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Log;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.LinkedBlockingQueue;
public class VideoToFrames {
private static final String TAG = "VideoToFrames";
private static final boolean VERBOSE = false;
private static final long DEFAULT_TIMEOUT_US = 10000;
private boolean stopDecode = false;
private final LinkedBlockingQueue<byte[]> queue = new LinkedBlockingQueue<>(50);
public Callback2 callback;
public interface Callback2 {
void onFinishDecode();
void onDecodeFrame(int index, Bitmap bitmap);
}
public void setCallback(Callback2 callback) {
this.callback = callback;
}
public void stopDecode() {
stopDecode = true;
}
public LinkedBlockingQueue<byte[]> decode(String videoFilePath) {
new Thread(() -> videoDecode(videoFilePath)).start();
return queue;
}
private void videoDecode(String videoFilePath) {
MediaExtractor extractor = null;
MediaCodec decoder = null;
try {
File videoFile = new File(videoFilePath);
extractor = new MediaExtractor();
extractor.setDataSource(videoFile.toString());
int trackIndex = selectTrack(extractor);
if (trackIndex < 0) {
throw new RuntimeException("No video track found in " + videoFilePath);
}
extractor.selectTrack(trackIndex);
MediaFormat mediaFormat = extractor.getTrackFormat(trackIndex);
String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
decoder = MediaCodec.createDecoderByType(mime);
showSupportedColorFormat(decoder.getCodecInfo().getCapabilitiesForType(mime));
int decodeColorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible;
if (isColorFormatSupported(decodeColorFormat, decoder.getCodecInfo().getCapabilitiesForType(mime))) {
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, decodeColorFormat);
Log.i(TAG, "set decode color format to type " + decodeColorFormat);
} else {
Log.i(TAG, "unable to set decode color format, color format type " + decodeColorFormat + " not supported");
}
decodeFramesToImage(decoder, extractor, mediaFormat);
decoder.stop();
} catch (IOException | InterruptedException e) {
e.printStackTrace();
} finally {
if (decoder != null) {
decoder.stop();
decoder.release();
}
if (extractor != null) {
extractor.release();
}
}
}
private void showSupportedColorFormat(MediaCodecInfo.CodecCapabilities caps) {
System.out.print("supported color format: ");
for (int c : caps.colorFormats) {
System.out.print(c + "\t");
}
System.out.println();
}
private boolean isColorFormatSupported(int colorFormat, MediaCodecInfo.CodecCapabilities caps) {
for (int c : caps.colorFormats) {
if (c == colorFormat) {
return true;
}
}
return false;
}
private void decodeFramesToImage(MediaCodec decoder, MediaExtractor extractor, MediaFormat mediaFormat) throws InterruptedException {
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
decoder.configure(mediaFormat, null, null, 0);
decoder.start();
int outputFrameCount = 0;
while (!sawOutputEOS && !stopDecode) {
if (!sawInputEOS) {
int inputBufferId = decoder.dequeueInputBuffer(DEFAULT_TIMEOUT_US);
if (inputBufferId >= 0) {
ByteBuffer inputBuffer = decoder.getInputBuffer(inputBufferId);
int sampleSize = extractor.readSampleData(inputBuffer, 0);
if (sampleSize < 0) {
decoder.queueInputBuffer(inputBufferId, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
sawInputEOS = true;
} else {
long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufferId, 0, sampleSize, presentationTimeUs, 0);
extractor.advance();
}
}
}
int outputBufferId = decoder.dequeueOutputBuffer(info, DEFAULT_TIMEOUT_US);
if (outputBufferId >= 0) {
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
boolean doRender = (info.size != 0);
if (doRender) {
outputFrameCount++;
Image image = decoder.getOutputImage(outputBufferId);
compressToJpeg2(image);
image.close();
decoder.releaseOutputBuffer(outputBufferId, true);
}
}
}
if (callback != null) {
callback.onFinishDecode();
}
}
private static int selectTrack(MediaExtractor extractor) {
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
if (VERBOSE) {
Log.d(TAG, "Extractor selected track " + i + " (" + mime + "): " + format);
}
return i;
}
}
return -1;
}
private static boolean isImageFormatSupported(Image image) {
int format = image.getFormat();
switch (format) {
case ImageFormat.YUV_420_888:
case ImageFormat.NV21:
case ImageFormat.YV12:
return true;
}
return false;
}
private static byte[] getDataFromImage(Image image) {
if (!isImageFormatSupported(image)) {
throw new RuntimeException("can't convert Image to byte array, format " + image.getFormat());
}
Rect crop = image.getCropRect();
int format = image.getFormat();
int width = crop.width();
int height = crop.height();
Image.Plane[] planes = image.getPlanes();
byte[] data = new byte[width * height * ImageFormat.getBitsPerPixel(format) / 8];
byte[] rowData = new byte[planes[0].getRowStride()];
if (VERBOSE) Log.v(TAG, "get data from " + planes.length + " planes");
int channelOffset = 0;
int outputStride = 1;
for (int i = 0; i < planes.length; i++) {
switch (i) {
case 0:
channelOffset = 0;
outputStride = 1;
break;
case 1:
channelOffset = width * height + 1;
outputStride = 2;
break;
case 2:
channelOffset = width * height;
outputStride = 2;
break;
}
ByteBuffer buffer = planes[i].getBuffer();
int rowStride = planes[i].getRowStride();
int pixelStride = planes[i].getPixelStride();
if (VERBOSE) {
Log.v(TAG, "pixelStride " + pixelStride);
Log.v(TAG, "rowStride " + rowStride);
Log.v(TAG, "width " + width);
Log.v(TAG, "height " + height);
Log.v(TAG, "buffer size " + buffer.remaining());
}
int shift = (i == 0) ? 0 : 1;
int w = width >> shift;
int h = height >> shift;
buffer.position(rowStride * (crop.top >> shift) + pixelStride * (crop.left >> shift));
for (int row = 0; row < h; row++) {
int length;
if (pixelStride == 1 && outputStride == 1) {
length = w;
buffer.get(data, channelOffset, length);
channelOffset += length;
} else {
length = (w - 1) * pixelStride + 1;
buffer.get(rowData, 0, length);
for (int col = 0; col < w; col++) {
data[channelOffset] = rowData[col * pixelStride];
channelOffset += outputStride;
}
}
if (row < h - 1) {
buffer.position(buffer.position() + rowStride - length);
}
}
if (VERBOSE) Log.v(TAG, "Finished reading data from plane " + i);
}
return data;
}
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
private Bitmap compressToJpeg2(Image image) throws InterruptedException {
outStream.reset();
Rect rect = image.getCropRect();
YuvImage yuvImage = new YuvImage(getDataFromImage(image), ImageFormat.NV21, rect.width(), rect.height(), null);
yuvImage.compressToJpeg(rect, 100, outStream);
byte[] bytes = outStream.toByteArray();
queue.put(bytes);
return null;
}
}
MediaCodec通过Surface播放数据
package com.vision.editor.utils;
import java.io.IOException;
import java.nio.ByteBuffer;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Log;
import android.view.Surface;
public class VideoDecoderThread extends Thread {
private static final String VIDEO = "video/";
private static final String TAG = "VideoDecoder";
private MediaExtractor mExtractor;
private MediaCodec mDecoder;
private boolean eosReceived;
public boolean init(Surface surface, String filePath) {
eosReceived = false;
try {
mExtractor = new MediaExtractor();
mExtractor.setDataSource(filePath);
Log.d(TAG, "getTrackCount: " + mExtractor.getTrackCount() );
for (int i = 0; i < mExtractor.getTrackCount(); i++) {
MediaFormat format = mExtractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith(VIDEO)) {
mExtractor.selectTrack(i);
mDecoder = MediaCodec.createDecoderByType(mime);
try {
Log.d(TAG, "format : " + format);
mDecoder.configure(format, surface, null, 0 );
} catch (IllegalStateException e) {
Log.e(TAG, "codec '" + mime + "' failed configuration. " + e);
return false;
}
mDecoder.start();
break;
}
}
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
@Override
public void run() {
BufferInfo info = new BufferInfo();
ByteBuffer[] inputBuffers = mDecoder.getInputBuffers();
mDecoder.getOutputBuffers();
boolean isInput = true;
boolean first = false;
long startWhen = 0;
while (!eosReceived) {
if (isInput) {
int inputIndex = mDecoder.dequeueInputBuffer(10000);
if (inputIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputIndex];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (mExtractor.advance() && sampleSize > 0) {
mDecoder.queueInputBuffer(inputIndex, 0, sampleSize, mExtractor.getSampleTime(), 0);
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mDecoder.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isInput = false;
}
}
}
int outIndex = mDecoder.dequeueOutputBuffer(info, 10000);
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
mDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " + mDecoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
break;
default:
if (!first) {
startWhen = System.currentTimeMillis();
first = true;
}
try {
long sleepTime = (info.presentationTimeUs / 1000) - (System.currentTimeMillis() - startWhen);
if (sleepTime > 0)
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
e.printStackTrace();
}
mDecoder.releaseOutputBuffer(outIndex, true );
break;
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
}
mDecoder.stop();
mDecoder.release();
mExtractor.release();
}
public void close() {
eosReceived = true;
}
}
参考资料