基于FFmpeg的Unity视频播放器
之前使用VLC做视频播放器，功能基本都能实现，但是性能还是很低，花时间试了一下直接用FFmpeg，目前还没有完成，只是简单的把视频帧和音频帧解出来做了显示和播放。
下面把主要代码贴一下
using FFmpeg.AutoGen;
using FFmpeg.AutoGen.Example;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using UnityEngine;
namespace UnityFFmpeg
{
/// <summary>
/// FFPlayer
/// </summary>
public sealed unsafe class FFPlayer : IDisposable
{
private string _url;
int error;
AVHWDeviceType deviceType;
private Size _frameSize;
private AVPixelFormat _pixelFormat;
private AVSampleFormat _sample_fmt;
private readonly AVCodecContext* _pVideoContext;
private readonly AVCodecContext* _pAudioContext;
private readonly AVFormatContext* _pFormatContext;
private readonly int _videoStreamIndex;
private readonly int _audioStreamIndex;
private readonly AVFrame* _audioFrame;
private readonly AVFrame* _videoFrame;
private AVFrame* _g2cFrame;
private AVFrame _tempFrame;
private readonly AVPacket* _packet;
SwrContext* _audioSwrContext;
int outChannelCount;
AVSampleFormat outFormat;
AVPixelFormat destinationPixelFormat = AVPixelFormat.AV_PIX_FMT_RGB24;
private Action<byte[]> _onVideoData;
private Action<byte[]> _onAudioData;
private Action<int, int> _onVideoSize;
Thread thread = null;
public FFPlayer(string url, Action<int, int> onVideoSize, Action<byte[]> onVideoData, Action<byte[]> onAudioData)
{
_url = url;
_onVideoSize += onVideoSize;
_onVideoData += onVideoData;
_onAudioData += onAudioData;
Init();
_pFormatContext = ffmpeg.avformat_alloc_context();
var pFormatContext = _pFormatContext;
ffmpeg.avformat_open_input(&pFormatContext, url, null, null).ThrowExceptionIfError();
ffmpeg.avformat_find_stream_info(_pFormatContext, null).ThrowExceptionIfError();
AVCodec* videoCodec = null;
_videoStreamIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0).ThrowExceptionIfError();
AVCodec* audioCodec = null;
_audioStreamIndex = ffmpeg.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &audioCodec, 0).ThrowExceptionIfError();
Debug.LogWarning("_videoStreamIndex:" + _videoStreamIndex);
Debug.LogWarning("_audioStreamIndex:" + _audioStreamIndex);
_pVideoContext = ffmpeg.avcodec_alloc_context3(videoCodec);
Debug.LogWarning("deviceType:" + deviceType);
if (deviceType != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE)
{
ffmpeg.av_hwdevice_ctx_create(&_pVideoContext->hw_device_ctx, deviceType, null, null, 0).ThrowExceptionIfError();
}
AVCodecParameters* vavcp = _pFormatContext->streams[_videoStreamIndex]->codecpar;
ffmpeg.avcodec_parameters_to_context(_pVideoContext, vavcp).ThrowExceptionIfError();
_pAudioContext = ffmpeg.avcodec_alloc_context3(audioCodec);
AVCodecParameters* aavcp = _pFormatContext->streams[_audioStreamIndex]->codecpar;
ffmpeg.avcodec_parameters_to_context(_pAudioContext, aavcp).ThrowExceptionIfError();
ffmpeg.avcodec_open2(_pVideoContext, videoCodec, null).ThrowExceptionIfError();
ffmpeg.avcodec_open2(_pAudioContext, audioCodec, null).ThrowExceptionIfError();
_frameSize = new Size(_pVideoContext->width, _pVideoContext->height);
if (_onVideoSize != null)
{
_onVideoSize(_frameSize.Width, _frameSize.Height);
}
_pixelFormat = _pVideoContext->pix_fmt;
_sample_fmt = _pVideoContext->sample_fmt;
//frame->16bit 44100 PCM 统一音频采样格式与采样率
//创建swrcontext上下文件
_audioSwrContext = ffmpeg.swr_alloc();
//音频格式 输入的采样设置参数
AVSampleFormat inFormat = _pAudioContext->sample_fmt;
// 出入的采样格式
outFormat = AVSampleFormat.AV_SAMPLE_FMT_S16;
// 输入采样率
int inSampleRate = _pAudioContext->sample_rate;
// 输出采样率
int outSampleRate = 44100;
// 输入声道布局
ulong in_ch_layout = _pAudioContext->channel_layout;
//输出声道布局
int out_ch_layout = ffmpeg.AV_CH_LAYOUT_STEREO;
//给Swrcontext 分配空间，设置公共参数
ffmpeg.swr_alloc_set_opts(_audioSwrContext, out_ch_layout, outFormat, outSampleRate,
(long)in_ch_layout, inFormat, inSampleRate, 0, null
);
// 初始化
ffmpeg.swr_init(_audioSwrContext);
// 获取声道数量
outChannelCount = ffmpeg.av_get_channel_layout_nb_channels((ulong)out_ch_layout);
_packet = ffmpeg.av_packet_alloc();
_audioFrame = ffmpeg.av_frame_alloc();
_videoFrame = ffmpeg.av_frame_alloc();
_g2cFrame = ffmpeg.av_frame_alloc();
thread = new Thread(new ThreadStart(DecodeTest));
thread.IsBackground = true;
thread.Start();
}
private void Init()
{
ffmpeg.RootPath = Application.streamingAssetsPath + "/FFmpeg/x86_64";
Debug.LogWarning($"FFmpeg version info: {
ffmpeg.av_version_info()}");
SetupLogging();
ConfigureHWDecoder();
}
bool isVideo;
private void DecodeTest()
{
while (true)
{
ffmpeg.av_frame_unref(_audioFrame);
ffmpeg.av_frame_unref(_videoFrame);
do
{
//开始读取源文件，进行解码
error = ffmpeg.av_read_frame(_pFormatContext, _packet);
if (error == ffmpeg.AVERROR_EOF)
{
Debug.LogWarning("over");
break;
}
if (_packet->stream_index == _audioStreamIndex)
{
error = ffmpeg.avcodec_send_packet(_pAudioContext, _packet);
//解码
error = ffmpeg.avcodec_receive_frame(_pAudioContext, _audioFrame);
isVideo = false;
}
else if (_packet->stream_index == _videoStreamIndex)
{
error = ffmpeg.avcodec_send_packet(_pVideoContext, _packet);
//解码
error = ffmpeg.avcodec_receive_frame(_pVideoContext, _videoFrame);
isVideo = true;
}
}
while (error == ffmpeg.AVERROR(ffmpeg.EAGAIN));
if (!isVideo)
{
byte* out_buffer = (byte*)Marshal.AllocHGlobal(2 * 44100);
//将每一帧数据转换成pcm
ffmpeg.swr_convert(_audioSwrContext, &out_buffer, 2 * 44100, (byte**)&_audioFrame->data, _audioFrame->nb_samples);
//获取实际的缓存大小
int out_buffer_size = ffmpeg.av_samples_get_buffer_size(null, outChannelCount, _audioFrame->nb_samples, outFormat, 1);
if (out_buffer_size > 0)
{
byte[] data = new byte[out_buffer_size];
Marshal.Copy((IntPtr)out_buffer, data, 0, out_buffer_size);
if (_onAudioData != null)
{
_onAudioData(data);
}
}
else
{
Debug.LogWarning("out_buffer_size:" + out_buffer_size);
}
Marshal.FreeCoTaskMem((IntPtr)out_buffer);
}
if (isVideo)
{
if (_pVideoContext->hw_device_ctx != null)
{
ffmpeg.av_hwframe_transfer_data(_g2cFrame, _videoFrame, 0).ThrowExceptionIfError();
_tempFrame = *_g2cFrame;
}
else
{
_tempFrame = *_videoFrame;
}
var sourcePixelFormat = GetHWPixelFormat(deviceType);
using (var vfc = new VideoFrameConverter(_frameSize, sourcePixelFormat, _frameSize, destinationPixelFormat))
{
AVFrame convertedFrame = vfc.Convert(_tempFrame);
IntPtr imgPtr = (IntPtr)convertedFrame.data[0];
int dataLen = _frameSize.Width * _frameSize.Height * 3;
byte[] data = new byte[dataLen];
Marshal.Copy((IntPtr)convertedFrame.data[0], data, 0, data.Length);
if (_onVideoData != null)
{
_onVideoData(data);
}
Marshal.FreeCoTaskMem(imgPtr);
}
}
Thread.Sleep(12);
}
}
public void Play()
{
}
public void Pause()
{
}
public void Stop()
{
}
private unsafe void SetupLogging()
{
ffmpeg.av_log_set_level(ffmpeg.AV_LOG_VERBOSE);
// do not convert to local function
av_log_set_callback_callback logCallback = (p0, level, format, vl) =>
{
if (level > ffmpeg.av_log_get_level())
{
return;
}
var lineSize = 1024;
var lineBuffer = stackalloc byte[lineSize];
var printPrefix = 1;
ffmpeg.av_log_format_line(p0, level, format, vl, lineBuffer, lineSize, &printPrefix);
var line = Marshal.PtrToStringAnsi((IntPtr)lineBuffer);
Debug.LogWarning(line);
};
ffmpeg.av_log_set_callback(logCallback);
}
private void ConfigureHWDecoder()
{
deviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;
var availableHWDecoders = new Dictionary<int, AVHWDeviceType>();
var type = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;
var number = 0;
while ((type = ffmpeg.av_hwdevice_iterate_types(type)) != AVHWDeviceType.AV_HWDEVICE_TYPE_NONE)
{
Debug.Log($"{
++number}. {
type}");
availableHWDecoders.Add(number, type);
}
if (availableHWDecoders.Count == 0)
{
Debug.Log("Your system have no hardware decoders.");
deviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;
return;
}
int decoderNumber = availableHWDecoders.SingleOrDefault(t => t.Value == AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2).Key;
if (decoderNumber == 0)
{
decoderNumber = availableHWDecoders.First().Key;
}
Debug.LogWarning($"Selected [{
decoderNumber}]");
int.TryParse(Console.ReadLine(), out var inputDecoderNumber);
availableHWDecoders.TryGetValue(inputDecoderNumber == 0 ? decoderNumber : inputDecoderNumber, out deviceType);
}
private AVPixelFormat GetHWPixelFormat(AVHWDeviceType hWDevice)
{
switch (hWDevice)
{
case AVHWDeviceType.AV_HWDEVICE_TYPE_NONE:
return AVPixelFormat.AV_PIX_FMT_NONE;
case AVHWDeviceType.AV_HWDEVICE_TYPE_VDPAU:
return AVPixelFormat.AV_PIX_FMT_VDPAU;
case AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA:
return AVPixelFormat.AV_PIX_FMT_CUDA;
case AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI:
return AVPixelFormat.AV_PIX_FMT_VAAPI;
case AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2:
return AVPixelFormat.AV_PIX_FMT_NV12;
case AVHWDeviceType.AV_HWDEVICE_TYPE_QSV:
return AVPixelFormat.AV_PIX_FMT_QSV;
case AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
return AVPixelFormat.AV_PIX_FMT_VIDEOTOOLBOX;
case AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA:
return AVPixelFormat.AV_PIX_FMT_NV12;
case AVHWDeviceType.AV_HWDEVICE_TYPE_DRM:
return AVPixelFormat.AV_PIX_FMT_DRM_PRIME;
case AVHWDeviceType.AV_HWDEVICE_TYPE_OPENCL:
return AVPixelFormat.AV_PIX_FMT_OPENCL;
case AVHWDeviceType.AV_HWDEVICE_TYPE_MEDIACODEC:
return AVPixelFormat.AV_PIX_FMT_MEDIACODEC;
default:
return AVPixelFormat.AV_PIX_FMT_NONE;
}
}
public void Dispose()
{
if (thread != null)
{
if (thread.IsAlive)
{
thread.Abort();
}
}
ffmpeg.av_frame_unref(_audioFrame);
ffmpeg.av_free(_audioFrame);
ffmpeg.av_frame_unref(_videoFrame);
ffmpeg.av_free(_videoFrame);
ffmpeg.av_frame_unref(_g2cFrame);
ffmpeg.av_free(_g2cFrame);
ffmpeg.av_packet_unref(_packet);
ffmpeg.av_free(_packet);
ffmpeg.avcodec_close(_pVideoContext);
ffmpeg.avcodec_close(_pAudioContext);
var pFormatContext = _pFormatContext;
ffmpeg.avformat_close_input(&pFormatContext);
}
}
}
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using UnityFFmpeg;
public class TestPlayer : MonoBehaviour
{
Texture2D texture2D;
Queue<byte[]> imgQueue = new Queue<byte[]>();
Queue<byte[]> audioQueue = new Queue<byte[]>();
public RawImage rawImage;
FFPlayer ffPlayer;
// Start is called before the first frame update
void Start()
{
//var url = "rtmp://58.200.131.2:1935/livetv/hunantv";
var url = Application.streamingAssetsPath + "/test.mp4";
ffPlayer = new FFPlayer(url, OnVideoSize, OnVideoData, OAudioData);
}
byte[] data;
// Update is called once per frame
void Update()
{
if (imgQueue.Count > 0)
{
data = imgQueue.Dequeue();
if (texture2D != null)
{
texture2D.LoadRawTextureData(data);
texture2D.Apply();
}
}
}
private void OnVideoSize(int width, int height)
{
texture2D = new Texture2D(width, height, TextureFormat.RGB24, false);
texture2D.Apply();
rawImage.texture = texture2D;
}
private void OnVideoData(byte[] data)
{
imgQueue.Enqueue(data);
}
private void OAudioData(byte[] data)
{
audioQueue.Enqueue(data);
}
byte[] bsBuffer;
private void OnAudioFilterRead(float[] data, int channels)
{
if (audioQueue.Count > 0)
{
bsBuffer = audioQueue.Dequeue();
float[] _buffer = ByteArrayToFloatArray(bsBuffer, bsBuffer.Length);
for (int i = 0; i < data.Length; i++)
{
data[i] = _buffer[i];
}
}
}
public static float[] ByteArrayToFloatArray(byte[] byteArray, int length)
{
float[] resultFloatArray = new float[length / 2];
if (resultFloatArray == null || resultFloatArray.Length != (length / 2))
{
resultFloatArray = new float[length / 2];
}
int arrIdx = 0;
for (int i = 0; i < length; i += 2)
{
resultFloatArray[arrIdx++] = BytesToFloat(byteArray[i], byteArray[i + 1]);
}
return resultFloatArray;
}
static float BytesToFloat(byte firstByte, byte secondByte)
{
return (float)((short)((int)secondByte << 8 | (int)firstByte)) / 32768f;
}
private void OnApplicationQuit()
{
ffPlayer.Dispose();
}
}
播放4K视频
最后是工程地址 https://gitee.com/awnuxcvbn/UnityFFmpeg