硬件视频加速技术
硬件视频加速(Hardware video acceleration) 通过让显卡编解码视频，从而减轻cpu负担并且还能节省电源。
在linux上的几种实现:
Video Acceleration API (VAAPI)英特尔开发的规范和开源库，提供硬件加速视频的编/解码。
Video Decode and Presentation API for Unix (VDPAU)是一套NVIDIA开发的，用于将部分视频解码和渲染分担给GPU来处理的开源库和API。
NVDECODE/NVENCODENVIDIA Fermi, Kepler, Maxwell and Pascal系列显卡专有的硬件加速API
intel 硬件加速技术
intel 的开源媒体栈
MSDK : https://github.com/Intel-Media-SDK/MediaSDK
Libva : https://github.com/intel/libva
i965 driver : https://github.com/intel/intel-vaapi-driver
iHD driver : https://github.com/intel/media-driver
VA-API
开源驱动:
AMD: 位于官方软件仓库的libva-vdpau-driver和mesa两者为 Radeon 9500 或更新的GPU提供支持。
Intel: 位于官方软件仓库的libva-intel-driver软件包为 GMA 4500 系列或者更新的GPU提供支持。
NVIDIA: 位于官方软件仓库的libva-vdpau-driver和mesa两者为 GeForce 8 系列和更新的GPU提供支持。位于AUR的 nouveau-fwAUR 软件包包含从NVIDIA闭源驱动中提取出的必要的固件文件。
闭源驱动:
AMD: 位于AUR的libva-xvba-driverAUR[broken link: archived in aur-mirror]软件包为 Radeon HD 4000 系列或更新的GPU提供支持。在 Radeon HD 5000 系列或者更新的GPU上请使用 catalyst-utilsAUR 驱动；在 Radeon HD 4000 系列上使用 catalyst-total-hd234kAUR 作为驱动程序。
NVIDIA: 位于官方软件仓库的libva-vdpau-driver软件包和nvidia-utils驱动为 GeForce 8 系列或更新的GPU提供支持。
intel 依赖libva-intel-driver
AMD 依赖libva-mesa-driver)
安装
以前整理过ubuntu下的简洁安装记录：
linux 下 intel vaapi安装
主要是安装media-driver及其依赖库。
检验 VA-API
运行 libva-utils 包提供的 vainfo工具 来检查 VA-API 的设置：
$ vainfo
libva info: VA-API version 0.39.4
libva info: va_getDriverName() returns 0
libva info: Trying to open /usr/lib/dri/i965_drv_video.so
libva info: Found init function __vaDriverInit_0_39
libva info: va_openDriver() returns 0
vainfo: VA-API version: 0.39 (libva 1.7.3)
vainfo: Driver version: Intel i965 driver for Intel(R) Skylake - 1.7.3
vainfo: Supported profile and entrypoints
VAProfileMPEG2Simple : VAEntrypointVLD
VAProfileMPEG2Simple : VAEntrypointEncSlice
VAProfileMPEG2Main : VAEntrypointVLD
VAProfileMPEG2Main : VAEntrypointEncSlice
VAProfileH264ConstrainedBaseline: VAEntrypointVLD
VAProfileH264ConstrainedBaseline: VAEntrypointEncSlice
VAProfileH264ConstrainedBaseline: VAEntrypointEncSliceLP
VAProfileH264Main : VAEntrypointVLD
VAProfileH264Main : VAEntrypointEncSlice
VAProfileH264Main : VAEntrypointEncSliceLP
VAProfileH264High : VAEntrypointVLD
VAProfileH264High : VAEntrypointEncSlice
VAProfileH264High : VAEntrypointEncSliceLP
VAProfileH264MultiviewHigh : VAEntrypointVLD
VAProfileH264MultiviewHigh : VAEntrypointEncSlice
VAProfileH264StereoHigh : VAEntrypointVLD
VAProfileH264StereoHigh : VAEntrypointEncSlice
VAProfileVC1Simple : VAEntrypointVLD
VAProfileVC1Main : VAEntrypointVLD
VAProfileVC1Advanced : VAEntrypointVLD
VAProfileNone : VAEntrypointVideoProc
VAProfileJPEGBaseline : VAEntrypointVLD
VAProfileJPEGBaseline : VAEntrypointEncPicture
VAProfileVP8Version0_3 : VAEntrypointVLD
VAProfileVP8Version0_3 : VAEntrypointEncSlice
VAProfileHEVCMain : VAEntrypointVLD
VAProfileHEVCMain : VAEntrypointEncSlice样例
ffmpeg vaapi h264加速样例，其他ffmpeg wiki还有很多。
$ ffmpeg -threads 1 -i file.ext -vaapi_device /dev/dri/renderD128 -vcodec h264_vaapi -vf format='nv12|vaapi,hwupload' output.mp4代码例
曾经写过的vaapi样例: 解码->overlay->编码
package main
import(
"github.com/ailumiyana/goav-incr/goav/avcodec"
"github.com/ailumiyana/goav-incr/goav/avutil"
"github.com/ailumiyana/goav-incr/goav/avfilter"
"github.com/ailumiyana/tools/latency"
log "github.com/astaxie/beego/logs"
"fmt"
//"time"
"io/ioutil"
"os"
"strconv"
"unsafe"
)
var input_h264 string = "1080p.h264"
var output_h264 string = "out.h264"
type VaapiHWDeviceCtx struct {
hw_device_ctx *avutil.BufferRef
}
// input : "/dev/dri/card0" or "/dev/dri/renderD128"
func Create(device string) *VaapiHWDeviceCtx {
var hw_device_ctx *avutil.BufferRef
err := avutil.AVHwdeviceCtxCreate(&hw_device_ctx, avutil.AV_HWDEVICE_TYPE_VAAPI,
device, nil, 0)
if err < 0 {
fmt.Println("AVHwdeviceCtxCreate err : ", avutil.ErrorFromCode(err))
return nil
}
return &VaapiHWDeviceCtx{
hw_device_ctx,
}
}
func (v *VaapiHWDeviceCtx)Context() *avutil.BufferRef{
return v.hw_device_ctx
}
//warning need unref by avutil.AVBufferUnref()
func (v *VaapiHWDeviceCtx)GetAnAllocHwframeCtxRef(format, sw_format avutil.PixelFormat, w, h int) *avutil.BufferRef{
hw_frames_ref := avutil.AVHwframeCtxAlloc(v.hw_device_ctx)
if hw_frames_ref == nil {
fmt.Println("AVHwframeCtxAlloc err")
return nil
}
frames_ctx := hw_frames_ref.HWFramesContext()
//frames_ctx.SetHWFramesContextPrarms(avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),
// avutil.PixelFormat(avcodec.AV_PIX_FMT_YUV420P),
// 1280, 720)
frames_ctx.SetHWFramesContextPrarms(format, sw_format, w, h)
err := avutil.AVHwFrameCtxInit(hw_frames_ref)
if err < 0 {
fmt.Println("AVHwFrameCtxInit err : ", avutil.ErrorFromCode(err))
avutil.AVBufferUnref(&hw_frames_ref)
return nil
}
return hw_frames_ref
}
//var hw_device_ctx *avutil.BufferRef// = avutil.AVHwdeviceCtxAlloc(avutil.AV_HWDEVICE_TYPE_VAAPI)
func SetHwframesContext(ctxt *avcodec.Context) {
/*var hw_device_ctx *avutil.BufferRef
err := avutil.AVHwdeviceCtxCreate(&hw_device_ctx, avutil.AV_HWDEVICE_TYPE_VAAPI,
"/dev/dri/card0", nil, 0)
if err < 0 {
fmt.Println("AVHwdeviceCtxCreate err : ", avutil.ErrorFromCode(err))
panic("AVHwdeviceCtxCreate faild")
}*/
vaapi_device := Create("/dev/dri/card0")
//hw_device_ctx := vaapi_device.Context()
/*hw_frames_ref := avutil.AVHwframeCtxAlloc(hw_device_ctx)
if hw_frames_ref == nil {
fmt.Println("AVHwframeCtxAlloc err")
panic("AVHwframeCtxAlloc faild")
}
frames_ctx := hw_frames_ref.HWFramesContext()
frames_ctx.SetHWFramesContextPrarms(avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),
avutil.PixelFormat(avcodec.AV_PIX_FMT_YUV420P),
1280, 720)
err := avutil.AVHwFrameCtxInit(hw_frames_ref)
if err < 0 {
fmt.Println("AVHwFrameCtxInit err : ", avutil.ErrorFromCode(err))
avutil.AVBufferUnref(&hw_frames_ref)
panic("AVHwFrameCtxInit faild")
}*/
hw_frames_ref := vaapi_device.GetAnAllocHwframeCtxRef(
avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),
avutil.PixelFormat(avcodec.AV_PIX_FMT_YUV420P), 1280, 720)
//enc.Context().HWFramesCtx() = avutil.AVBufferRef(hw_frames_ref)
ctxt.SetHWFramesCtx(avutil.AVBufferRef(hw_frames_ref))
if ctxt.HWFramesCtx() == nil {
fmt.Println("SetHWFrameCtx err ")
avutil.AVBufferUnref(&hw_frames_ref)
panic("SetHWFrameCtx faild")
}
//fmt.Println(frames_ctx)
//fmt.Println(hw_frames_ref)
//fmt.Println(ctxt.HWFramesCtx())
avutil.AVBufferUnref(&hw_frames_ref)
}
func decode_encode() {
//decoder
vaapi_device := Create("/dev/dri/renderD128")
pkt := avcodec.AvPacketAlloc()
if pkt == nil {
log.Critical("AvPacketAlloc failed.")
return
}
codec := avcodec.AvcodecFindDecoder(avcodec.CodecId(avcodec.AV_CODEC_ID_H264))
if codec == nil {
log.Critical("AvcodecFindDecoder failed.")
}
context := codec.AvcodecAllocContext3()
if context == nil {
log.Critical("AvcodecAllocContext3 failed.")
return
}
parserContext := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))
if parserContext == nil {
log.Critical("AvParserInit failed.")
return
}
frame := avutil.AvFrameAlloc()
if frame == nil {
log.Critical("AvFrameAlloc failed.")
return
}
context.SetHWDeviceCtx(avutil.AVBufferRef(vaapi_device.Context()))
err := context.AvcodecOpen2(codec, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
// encoder
codec_enc := avcodec.AvcodecFindEncoderByName("h264_vaapi")
if codec_enc == nil {
log.Critical("AvcodecFindEncoderByName failed.")
}
pkt_enc := avcodec.AvPacketAlloc()
if pkt_enc == nil {
log.Critical("AvPacketAlloc failed.")
}
context_enc := codec_enc.AvcodecAllocContext3()
if context_enc == nil {
log.Critical("AvcodecAllocContext3 failed.")
}
context_enc.SetVideoEncodeParams(2000000, 1920, 1080,
avcodec.AV_PIX_FMT_VAAPI,
false, 10)
context_enc.SetTimebase(1, 30)
hw_frames_ref := vaapi_device.GetAnAllocHwframeCtxRef(
avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),
avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080)
context_enc.SetHWFramesCtx(avutil.AVBufferRef(hw_frames_ref))
err = context_enc.AvcodecOpen2(codec_enc, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
decLatency := latency.New("vaapi", "decode")
encLatency := latency.New("vaapi", "encode")
/*scaleLatency := latency.New("vaapi", "scale")
overlayLatency := latency.New("vaapi", "overlay")*/
data, erro := ioutil.ReadFile(input_h264)
if erro != nil {
log.Debug("File reading error", erro)
return
}
log.Debug("Open Success.")
l := len(data)
log.Debug("size of file:", l)
b := make([]byte, 4096 + 64)
file, erro := os.Create(output_h264)
if erro != nil {
log.Critical("Error Reading")
}
defer file.Close()
//var pts int64 = 0
//var opts int64 = 0
//var frames [25]*avutil.Frame
started := false
sum := 0
for sum < l {
remain := 4096
for remain > 0 {
copy(b, data[sum:sum + 4096])
if !started {
decLatency.Start()
started = true
}
n := context.AvParserParse2(parserContext, pkt, b,
remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)
log.Debug("parser ", n, "bytes")
sum = sum + n
remain = remain - n;
//log.Trace("--------", dec.Packet().GetPacketSize())
if pkt.GetPacketSize() > 0 {
//fmt.Println(decLatency.End())
ret := context.AvcodecSendPacket(pkt)
if ret < 0 {
log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))
continue
}
ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
started = false
decLatency.End()
fmt.Println(decLatency.End())
log.Debug("dec-frame:",frame)
encLatency.Start()
ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))
continue
}
ret = context_enc.AvcodecReceivePacket(pkt_enc)
if ret < 0 {
log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
data0 := pkt_enc.Data()
buf := make([]byte, pkt_enc.GetPacketSize())
start := uintptr(unsafe.Pointer(data0))
for i := 0; i < pkt_enc.GetPacketSize(); i++ {
elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))
buf[i] = elem
}
file.Write(buf)
//encLatency.End()
fmt.Println(encLatency.End())
}
avutil.AvFrameUnref(frame)
//avutil.AvFrameUnref(oframe)
}
}
}
}
}
func decode_overlay_encode() {
//decoder
vaapi_device := Create("/dev/dri/renderD128")
pkt := avcodec.AvPacketAlloc()
if pkt == nil {
log.Critical("AvPacketAlloc failed.")
return
}
codec := avcodec.AvcodecFindDecoder(avcodec.CodecId(avcodec.AV_CODEC_ID_H264))
if codec == nil {
log.Critical("AvcodecFindDecoder failed.")
}
context := codec.AvcodecAllocContext3()
if context == nil {
log.Critical("AvcodecAllocContext3 failed.")
return
}
parserContext := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))
if parserContext == nil {
log.Critical("AvParserInit failed.")
return
}
frame := avutil.AvFrameAlloc()
if frame == nil {
log.Critical("AvFrameAlloc failed.")
return
}
context.SetHWDeviceCtx(avutil.AVBufferRef(vaapi_device.Context()))
err := context.AvcodecOpen2(codec, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
// encoder
codec_enc := avcodec.AvcodecFindEncoderByName("h264_vaapi")
if codec_enc == nil {
log.Critical("AvcodecFindEncoderByName failed.")
}
pkt_enc := avcodec.AvPacketAlloc()
if pkt_enc == nil {
log.Critical("AvPacketAlloc failed.")
}
context_enc := codec_enc.AvcodecAllocContext3()
if context_enc == nil {
log.Critical("AvcodecAllocContext3 failed.")
}
context_enc.SetVideoEncodeParams(2000000, 1920, 1080,
avcodec.AV_PIX_FMT_VAAPI,
false, 10)
context_enc.SetTimebase(1, 30)
hw_frames_ref := vaapi_device.GetAnAllocHwframeCtxRef(
avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),
avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080)
/*context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))
err = context_enc.AvcodecOpen2(codec_enc, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}*/
decLatency := latency.New("vaapi", "decode")
encLatency := latency.New("vaapi", "encode")
// filter
args := "video_size=1920x1080:pix_fmt=46:time_base=1/30:pixel_aspect=1/1"
des := "[in0] fps=30,scale_vaapi=w=960:h=540:format=nv12, hwdownload,format=nv12 [s0];color=black:r=30:size=1920x1080:sar=1/1 [b0];[b0][s0] overlay=x=0:y=0, format=nv12, hwupload"
graph := avfilter.AvfilterGraphAlloc()
if graph == nil {
log.Critical("AvfilterGraphAlloc Failed.")
return
}
//graph.SetNbThreads(8)
inputs := avfilter.AvfilterInoutAlloc()
outputs := avfilter.AvfilterInoutAlloc()
if inputs == nil || outputs == nil {
log.Critical("AvfilterInoutAlloc Failed.")
return
}
defer avfilter.AvfilterInoutFree(inputs)
defer avfilter.AvfilterInoutFree(outputs)
var buffersrc *avfilter.Filter
var buffersink *avfilter.Filter
if false {
buffersrc = avfilter.AvfilterGetByName("abuffer")
buffersink = avfilter.AvfilterGetByName("abuffersink")
} else {
buffersrc = avfilter.AvfilterGetByName("buffer")
buffersink = avfilter.AvfilterGetByName("buffersink")
}
if buffersink == nil || buffersrc == nil {
log.Critical("AvfilterGetByName Failed.")
return
}
ret := graph.AvfilterGraphParse2(des, &inputs, &outputs)
if ret < 0 {
log.Critical("AvfilterInoutAlloc Failed des : ", avutil.ErrorFromCode(ret))
return
}
if vaapi_device.Context() != nil {
for _, v := range graph.Filters() {
v.SetHWDeviceCtx(avutil.AVBufferRef(vaapi_device.Context()))
}
}
var ins []*avfilter.Context
var outs []*avfilter.Context
var frames []*avutil.Frame
// inputs
index := 0
for cur := inputs; cur != nil; cur = cur.Next() {
//log.Debug("index :", index)
var in *avfilter.Context
//var args = "video_size=1280x720:pix_fmt=0:time_base=1/30:pixel_aspect=1/1"
inName := "in" + strconv.Itoa(index)
ret = avfilter.AvfilterGraphCreateFilter(&in, buffersrc, inName, args, 0, graph)
if ret < 0 {
log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))
return
}
par := avfilter.AvBuffersrcParametersAlloc()
if par == nil {
log.Critical("AvBuffersrcParametersAlloc Failed.")
return
}
par.SetHwFramesCtx(avutil.AVBufferRef(hw_frames_ref))
ret = in.AvBuffersrcParametersSet(par)
if ret < 0 {
log.Critical("AvBuffersrcParametersSet Failed.")
return
}
avutil.AvFreep(unsafe.Pointer(&par))
ins = append(ins, in)
ret = avfilter.AvfilterLink(ins[index], 0, cur.FilterContext(), cur.PadIdx())
if ret < 0 {
log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))
return
}
index++
}
// outputs
index = 0
for cur := outputs; cur != nil; cur = cur.Next() {
var out *avfilter.Context
outName := "out" + strconv.Itoa(index)
ret = avfilter.AvfilterGraphCreateFilter(&out, buffersink, outName, "", 0, graph)
if ret < 0 {
log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))
return
}
outs = append(outs, out)
ret = avfilter.AvfilterLink(cur.FilterContext(), cur.PadIdx(), outs[index], 0)
if ret < 0 {
log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))
return
}
index++
f := avutil.AvFrameAlloc()
if f == nil {
log.Critical("AvFrameAlloc failed.")
return
}
frames = append(frames, f)
}
ret = graph.AvfilterGraphConfig(0)
if ret < 0 {
log.Critical("AvfilterGraphConfig Failed des : ", avutil.ErrorFromCode(ret))
return
}
overlayLatency := latency.New("vaapi", "overlay")
context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))
err = context_enc.AvcodecOpen2(codec_enc, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
data, erro := ioutil.ReadFile(input_h264)
if erro != nil {
log.Debug("File reading error", erro)
return
}
log.Debug("Open Success.")
l := len(data)
log.Debug("size of file:", l)
b := make([]byte, 4096 + 64)
file, erro := os.Create(output_h264)
if erro != nil {
log.Critical("Error Reading")
}
defer file.Close()
var pts int64 = 0
//var opts int64 = 0
//var frames [25]*avutil.Frame
started := false
sum := 0
for sum < l {
remain := 4096
for remain > 0 {
copy(b, data[sum:sum + 4096])
if !started {
decLatency.Start()
started = true
}
n := context.AvParserParse2(parserContext, pkt, b,
remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)
log.Debug("parser ", n, "bytes")
sum = sum + n
remain = remain - n;
//log.Trace("--------", dec.Packet().GetPacketSize())
if pkt.GetPacketSize() > 0 { // decode
//fmt.Println(decLatency.End())
ret := context.AvcodecSendPacket(pkt)
if ret < 0 {
log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))
continue
}
ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
decLatency.End()
fmt.Println(decLatency.End())
} else {
continue
}
overlayLatency.Start()
pts++
frame.SetPts(pts)
ret = avfilter.AvBuffersrcAddFrame(ins[0], (*avfilter.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))
continue
}
ret = avfilter.AvBufferSinkGetFrame(outs[0], (*avfilter.Frame)(unsafe.Pointer(frames[0])))
if ret == avutil.AvErrorEOF || ret == avutil.AvErrorEAGAIN {
log.Error(avutil.ErrorFromCode(ret))
continue
}
if ret < 0 {
log.Error("AvBufferSinkGetFrame Failed des : ", ret, avutil.ErrorFromCode(ret))
continue
}
frame = frames[0]
fmt.Println(overlayLatency.End())
if ret == 0 { // encode
started = false
//log.Debug("dec-frame:",frame)
encLatency.Start()
ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))
continue
}
ret = context_enc.AvcodecReceivePacket(pkt_enc)
if ret < 0 {
log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
data0 := pkt_enc.Data()
buf := make([]byte, pkt_enc.GetPacketSize())
start := uintptr(unsafe.Pointer(data0))
for i := 0; i < pkt_enc.GetPacketSize(); i++ {
elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))
buf[i] = elem
}
file.Write(buf)
//encLatency.End()
fmt.Println(encLatency.End())
}
avutil.AvFrameUnref(frame)
//avutil.AvFrameUnref(oframe)
}
}
}
}
}
func main() {
//decode_encode()
decode_overlay_encode()
}Intel Quick Sync(QSV)
除了VAAPI之外，intel还有一项自己平台专属的硬件加速技术QSV(quick-sync-video)。
Quick Sync Video 使用Intel图形技术专用媒体处理能力来快速编解码。
ffmpeg wiki有一页qsv的介绍，但是推荐使用VA-API(iHD / i965) 驱动代替libmfx.
API支持情况
QSV支持访问多种不同的媒体库API:
DXVA2 / D3D11VA : 标准Windows api，由Intel图形驱动程序实现，以支持视频解码。
linux libmfx:这是intel的一个库，可以作为Intel Media SDK的一部分安装，并且支持一些编码和解码类型。
Windows libmfx: 英特尔的图形驱动程序提供，它支持所有的编码和解码的类型。
Media Foundation:通过英特尔图形驱动程序，支持一部分编解码类型，没有继承进ffmpeg。
VAAPI i965驱动:这是一个几乎免费的libva/VAAPI驱动程序。大多数Linux发行版都打包了它。
VAAPI iHD驱动:Linux上的libmfx后期使用了的一个修改过的libva和VAAPI驱动程序;这也可以由用户直接使用。
vaapi/mfx比较
vaapi/i965
1)在大多数Linux发行版中都是作为标准打包的。
2)可运行在所有可用的硬件上，包括一些较老和较便宜的设备。
3)支持大多数编解码器。
4)通用的API，也可以使用基于Mesa的AMD / Nvidia硬件。
5)可与标准api (EGL/OpenGL, OpenCL)交互。
libmfx /iHD
1)在某些情况下，可能会提供更好的编码质量(如look_ahead)。
2)在某些情况下(如MFE，特别是在Iris图形上)可能会提供更高的编码吞吐量。
3)linux和windows皆通用的API。
4)可与Intel实现的OpenCL实现交互。
安装
linux下
可以安装基于vaapi开源的MediaSDK
windows下
官方有驱动下载:https://software.intel.com/media-sdk
样例
h264解码
ffmpeg -hwaccel qsv -c:v h264_qsv -i input.mp4 -f null -代码例
qsv 解码->overlay->编码
package main
import (
log "github.com/astaxie/beego/logs"
"github.com/ailumiyana/goav-incr/goav/avutil"
"github.com/ailumiyana/goav-incr/goav/avcodec"
"github.com/ailumiyana/goav-incr/goav/avfilter"
"github.com/ailumiyana/tools/latency"
"fmt"
"time"
"io/ioutil"
"os"
"strconv"
"unsafe"
)
var input_h264 string = "1080p.h264"
var input_h264_720 string = "recv0.h264"
var output_h264 string = "out.h264"
var overlay16 string = "color=white:r=30:size=1920x1080:sar=1/1,hwupload=extra_hw_frames=300,format=qsv [background];" +
"[in0] scale_qsv=479:269 [in_0];" +
"[in1] scale_qsv=479:269 [in_1];" +
"[in2] scale_qsv=479:269 [in_2];" +
"[in3] scale_qsv=479:269 [in_3];" +
"[in4] scale_qsv=479:269 [in_4];" +
"[in5] scale_qsv=479:269 [in_5];" +
"[in6] scale_qsv=479:269 [in_6];" +
"[in7] scale_qsv=479:269 [in_7];" +
"[in8] scale_qsv=479:269 [in_8];" +
"[in9] scale_qsv=479:269 [in_9];" +
"[in10] scale_qsv=479:269 [in_10];" +
"[in11] scale_qsv=479:269 [in_11];" +
"[in12] scale_qsv=479:269 [in_12];" +
"[in13] scale_qsv=479:269 [in_13];" +
"[in14] scale_qsv=479:269 [in_14];" +
"[in15] scale_qsv=479:269 [in_15];" +
"[background][in_0] overlay_qsv=x=0:y=0 [background+in0_scale];" +
"[background+in0_scale][in_1] overlay_qsv=x=481:y=0 [background+in0_scale+in1_scale];" +
"[background+in0_scale+in1_scale][in_2] overlay_qsv=x=961:y=0 [background+in0_scale+in1_scale+in2_scale];" +
"[background+in0_scale+in1_scale+in2_scale][in_3] overlay_qsv=x=1441:y=0 [background+in0_scale+in1_scale+in2_scale+in3_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale][in_4] overlay_qsv=x=0:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale][in_5] overlay_qsv=x=481:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale][in_6] overlay_qsv=x=961:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale][in_7] overlay_qsv=x=1441:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale][in_8] overlay_qsv=x=0:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale][in_9] overlay_qsv=x=481:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale][in_10] overlay_qsv=x=961:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale][in_11] overlay_qsv=x=1441:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale][in_12] overlay_qsv=x=0:y=811 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale][in_13] overlay_qsv=x=481:y=811 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale][in_14] overlay_qsv=x=961:y=811 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale+in14_scale];" +
"[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale+in14_scale][in_15] overlay_qsv=x=1441:y=811"
type QsvHWDeviceCtx struct {
hw_device_ctx *avutil.BufferRef
}
// input : "/dev/dri/card0" or "/dev/dri/renderD128"
func Create(device string) *QsvHWDeviceCtx {
var hw_device_ctx *avutil.BufferRef
err := avutil.AVHwdeviceCtxCreate(&hw_device_ctx, avutil.AV_HWDEVICE_TYPE_QSV,
device, nil, 0)
if err < 0 {
log.Error("AVHwdeviceCtxCreate err : ", avutil.ErrorFromCode(err))
return nil
}
return &QsvHWDeviceCtx{
hw_device_ctx,
}
}
func (v *QsvHWDeviceCtx) Context() *avutil.BufferRef{
return v.hw_device_ctx
}
//warning need unref by avutil.AVBufferUnref()
func (v *QsvHWDeviceCtx)GetAnAllocHwframeCtxRef(format, sw_format avutil.PixelFormat, w, h, s int) *avutil.BufferRef{
hw_frames_ref := avutil.AVHwframeCtxAlloc(v.hw_device_ctx)
if hw_frames_ref == nil {
log.Error("AVHwframeCtxAlloc err")
return nil
}
frames_ctx := hw_frames_ref.HWFramesContext()
frames_ctx.SetQsvHWFramesContextPrarms(format, sw_format, w, h, s)
err := avutil.AVHwFrameCtxInit(hw_frames_ref)
if err < 0 {
log.Error("AVHwFrameCtxInit err : ", avutil.ErrorFromCode(err))
avutil.AVBufferUnref(&hw_frames_ref)
return nil
}
return hw_frames_ref
}
func decode_overlay_encode() {
//decoder
qsv_device := Create("/dev/dri/renderD128")
pkt := avcodec.AvPacketAlloc()
if pkt == nil {
log.Critical("AvPacketAlloc failed.")
return
}
codec := avcodec.AvcodecFindDecoderByName("h264_qsv")
if codec == nil {
log.Critical("AvcodecFindDecoderByName failed.")
return
}
context := codec.AvcodecAllocContext3()
if context == nil {
log.Critical("AvcodecAllocContext3 failed.")
return
}
parserContext := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))
if parserContext == nil {
log.Critical("AvParserInit failed.")
return
}
frame := avutil.AvFrameAlloc()
if frame == nil {
log.Critical("AvFrameAlloc failed.")
return
}
context.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))
err := context.AvcodecOpen2(codec, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
context.SetDefaultQsvGetFormat()
// encoder
codec_enc := avcodec.AvcodecFindEncoderByName("h264_qsv")
if codec_enc == nil {
log.Critical("AvcodecFindEncoderByName failed.")
}
pkt_enc := avcodec.AvPacketAlloc()
if pkt_enc == nil {
log.Critical("AvPacketAlloc failed.")
}
context_enc := codec_enc.AvcodecAllocContext3()
if context_enc == nil {
log.Critical("AvcodecAllocContext3 failed.")
}
context_enc.SetVideoEncodeParams(2000000, 1920, 1080,
avcodec.AV_PIX_FMT_QSV,
false, 10)
context_enc.SetTimebase(1, 30)
hw_frames_ref := qsv_device.GetAnAllocHwframeCtxRef(
avutil.PixelFormat(avcodec.AV_PIX_FMT_QSV),
avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080, 64)
/*context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))
err = context_enc.AvcodecOpen2(codec_enc, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}*/
decLatency := latency.New("vaapi", "decode")
encLatency := latency.New("vaapi", "encode")
// filter
args := "video_size=1920x1080:pix_fmt=qsv:time_base=1/30:pixel_aspect=1/1"
//des := "[in0] scale_qsv=w=960:h=540 [s0];color=black:r=30:size=1920x1080:sar=1/1, hwupload=extra_hw_frames=64,format=qsv [b0];[b0][s0] overlay_qsv=x=0:y=0"
des := overlay16
graph := avfilter.AvfilterGraphAlloc()
if graph == nil {
log.Critical("AvfilterGraphAlloc Failed.")
return
}
//graph.SetNbThreads(8)
inputs := avfilter.AvfilterInoutAlloc()
outputs := avfilter.AvfilterInoutAlloc()
if inputs == nil || outputs == nil {
log.Critical("AvfilterInoutAlloc Failed.")
return
}
defer avfilter.AvfilterInoutFree(inputs)
defer avfilter.AvfilterInoutFree(outputs)
var buffersrc *avfilter.Filter
var buffersink *avfilter.Filter
if false {
buffersrc = avfilter.AvfilterGetByName("abuffer")
buffersink = avfilter.AvfilterGetByName("abuffersink")
} else {
buffersrc = avfilter.AvfilterGetByName("buffer")
buffersink = avfilter.AvfilterGetByName("buffersink")
}
if buffersink == nil || buffersrc == nil {
log.Critical("AvfilterGetByName Failed.")
return
}
ret := graph.AvfilterGraphParse2(des, &inputs, &outputs)
if ret < 0 {
log.Critical("AvfilterInoutAlloc Failed des : ", avutil.ErrorFromCode(ret))
return
}
if qsv_device.Context() != nil {
for _, v := range graph.Filters() {
v.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))
}
}
var ins []*avfilter.Context
var outs []*avfilter.Context
var frames []*avutil.Frame
// inputs
index := 0
for cur := inputs; cur != nil; cur = cur.Next() {
//log.Debug("index :", index)
var in *avfilter.Context
//var args = "video_size=1280x720:pix_fmt=0:time_base=1/30:pixel_aspect=1/1"
inName := "in" + strconv.Itoa(index)
ret = avfilter.AvfilterGraphCreateFilter(&in, buffersrc, inName, args, 0, graph)
if ret < 0 {
log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))
return
}
par := avfilter.AvBuffersrcParametersAlloc()
if par == nil {
log.Critical("AvBuffersrcParametersAlloc Failed.")
return
}
par.SetHwFramesCtx(avutil.AVBufferRef(hw_frames_ref))
ret = in.AvBuffersrcParametersSet(par)
if ret < 0 {
log.Critical("AvBuffersrcParametersSet Failed.")
return
}
avutil.AvFreep(unsafe.Pointer(&par))
ins = append(ins, in)
ret = avfilter.AvfilterLink(ins[index], 0, cur.FilterContext(), cur.PadIdx())
if ret < 0 {
log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))
return
}
index++
}
// outputs
index = 0
for cur := outputs; cur != nil; cur = cur.Next() {
var out *avfilter.Context
outName := "out" + strconv.Itoa(index)
ret = avfilter.AvfilterGraphCreateFilter(&out, buffersink, outName, "", 0, graph)
if ret < 0 {
log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))
return
}
outs = append(outs, out)
ret = avfilter.AvfilterLink(cur.FilterContext(), cur.PadIdx(), outs[index], 0)
if ret < 0 {
log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))
return
}
index++
f := avutil.AvFrameAlloc()
if f == nil {
log.Critical("AvFrameAlloc failed.")
return
}
frames = append(frames, f)
}
ret = graph.AvfilterGraphConfig(0)
if ret < 0 {
log.Critical("AvfilterGraphConfig Failed des : ", avutil.ErrorFromCode(ret))
return
}
overlayLatency := latency.New("qsv", "overlay")
context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))
var dict *avutil.Dictionary = avutil.AvDictAlloc()
er := dict.AvDictSet("profile", "high", 0)
if er < 0 {
log.Critical("AvDictSet failed")
return
}
er = dict.AvDictSet("level", "52", 0)
if er < 0 {
log.Critical("AvDictSet failed")
return
}
fmt.Println(dict.AvDictCount())
err = context_enc.AvcodecOpen2(codec_enc, (**avcodec.Dictionary)(unsafe.Pointer(&dict)))
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
fmt.Println(dict.AvDictCount())
data, erro := ioutil.ReadFile(input_h264)
if erro != nil {
log.Debug("File reading error", erro)
return
}
log.Debug("Open Success.")
l := len(data)
log.Debug("size of file:", l)
b := make([]byte, 4096 + 64)
file, erro := os.Create(output_h264)
if erro != nil {
log.Critical("Error Reading")
}
defer file.Close()
var pts int64 = 0
//var opts int64 = 0
//var frames [25]*avutil.Frame
started := false
sum := 0
for sum < l {
remain := 4096
for remain > 0 {
copy(b, data[sum:sum + 4096])
if !started {
decLatency.Start()
started = true
}
n := context.AvParserParse2(parserContext, pkt, b,
remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)
log.Debug("parser ", n, "bytes")
sum = sum + n
remain = remain - n;
//log.Trace("--------", dec.Packet().GetPacketSize())
if pkt.GetPacketSize() > 0 { // decode
//fmt.Println(decLatency.End())
ret := context.AvcodecSendPacket(pkt)
if ret < 0 {
log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))
continue
}
ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
decLatency.End()
fmt.Println(decLatency.End())
} else {
continue
}
overlayLatency.Start()
pts++
frame.SetPts(pts)
var fs []*avutil.Frame
for i := 0 ; i < 16 ; i++ {
fs = append(fs, avutil.AvFrameClone(frame))
ret = avfilter.AvBuffersrcAddFrame(ins[i], (*avfilter.Frame)(unsafe.Pointer(fs[i])))
if ret < 0 {
for i := 0 ; i < len(fs) ; i++ {
avutil.AvFrameFree(fs[i])
}
fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))
continue
}
}
avutil.AvFrameUnref(frame)
/*ret = avfilter.AvBuffersrcAddFrame(ins[0], (*avfilter.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))
continue
}*/
ret = avfilter.AvBufferSinkGetFrame(outs[0], (*avfilter.Frame)(unsafe.Pointer(frames[0])))
for i := 0 ; i < len(fs) ; i++ {
avutil.AvFrameFree(fs[i])
}
if ret == avutil.AvErrorEOF || ret == avutil.AvErrorEAGAIN {
log.Error(avutil.ErrorFromCode(ret))
continue
}
if ret < 0 {
log.Error("AvBufferSinkGetFrame Failed des : ", ret, avutil.ErrorFromCode(ret))
continue
}
frame = frames[0]
fmt.Println(overlayLatency.End())
if ret == 0 { // encode
started = false
//log.Debug("dec-frame:",frame)
encLatency.Start()
ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))
continue
}
ret = context_enc.AvcodecReceivePacket(pkt_enc)
if ret < 0 {
log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
data0 := pkt_enc.Data()
buf := make([]byte, pkt_enc.GetPacketSize())
start := uintptr(unsafe.Pointer(data0))
for i := 0; i < pkt_enc.GetPacketSize(); i++ {
elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))
buf[i] = elem
}
file.Write(buf)
//encLatency.End()
fmt.Println(encLatency.End())
}
avutil.AvFrameUnref(frame)
//avutil.AvFrameUnref(oframe)
}
}
}
}
}
func filter_inpute_args() {
//decoder
qsv_device := Create("/dev/dri/renderD128")
pkt := avcodec.AvPacketAlloc()
if pkt == nil {
log.Critical("AvPacketAlloc failed.")
return
}
codec := avcodec.AvcodecFindDecoderByName("h264_qsv")
if codec == nil {
log.Critical("AvcodecFindDecoderByName failed.")
return
}
context := codec.AvcodecAllocContext3()
if context == nil {
log.Critical("AvcodecAllocContext3 failed.")
return
}
parserContext := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))
if parserContext == nil {
log.Critical("AvParserInit failed.")
return
}
frame := avutil.AvFrameAlloc()
if frame == nil {
log.Critical("AvFrameAlloc failed.")
return
}
context.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))
err := context.AvcodecOpen2(codec, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
context.SetDefaultQsvGetFormat()
// encoder
codec_enc := avcodec.AvcodecFindEncoderByName("h264_qsv")
if codec_enc == nil {
log.Critical("AvcodecFindEncoderByName failed.")
}
pkt_enc := avcodec.AvPacketAlloc()
if pkt_enc == nil {
log.Critical("AvPacketAlloc failed.")
}
context_enc := codec_enc.AvcodecAllocContext3()
if context_enc == nil {
log.Critical("AvcodecAllocContext3 failed.")
}
context_enc.SetVideoEncodeParams(2000000, 1920, 1080,
avcodec.AV_PIX_FMT_QSV,
false, 10)
context_enc.SetTimebase(1, 30)
hw_frames_ref := qsv_device.GetAnAllocHwframeCtxRef(
avutil.PixelFormat(avcodec.AV_PIX_FMT_QSV),
avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080, 64)
pkt1 := avcodec.AvPacketAlloc()
if pkt == nil {
log.Critical("AvPacketAlloc failed.")
return
}
codec1 := avcodec.AvcodecFindDecoderByName("h264_qsv")
if codec == nil {
log.Critical("AvcodecFindDecoderByName failed.")
return
}
context1 := codec.AvcodecAllocContext3()
if context1 == nil {
log.Critical("AvcodecAllocContext3 failed.")
return
}
parserContext1 := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))
if parserContext1 == nil {
log.Critical("AvParserInit failed.")
return
}
frame1 := avutil.AvFrameAlloc()
if frame1 == nil {
log.Critical("AvFrameAlloc failed.")
return
}
context1.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))
err = context1.AvcodecOpen2(codec1, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
context1.SetDefaultQsvGetFormat()
hw_frames_ref1 := qsv_device.GetAnAllocHwframeCtxRef(
avutil.PixelFormat(avcodec.AV_PIX_FMT_QSV),
avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1280, 720, 64)
/*context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))
err = context_enc.AvcodecOpen2(codec_enc, nil)
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}*/
decLatency := latency.New("vaapi", "decode")
encLatency := latency.New("vaapi", "encode")
// filter
args := "video_size=1920x1080:pix_fmt=qsv:time_base=1/30:pixel_aspect=1/1"
//des := "[in0] scale_qsv=w=960:h=540 [s0];color=black:r=30:size=1920x1080:sar=1/1, hwupload=extra_hw_frames=64,format=qsv [b0];[b0][s0] overlay_qsv=x=0:y=0"
des := overlay16
graph := avfilter.AvfilterGraphAlloc()
if graph == nil {
log.Critical("AvfilterGraphAlloc Failed.")
return
}
//graph.SetNbThreads(8)
inputs := avfilter.AvfilterInoutAlloc()
outputs := avfilter.AvfilterInoutAlloc()
if inputs == nil || outputs == nil {
log.Critical("AvfilterInoutAlloc Failed.")
return
}
defer avfilter.AvfilterInoutFree(inputs)
defer avfilter.AvfilterInoutFree(outputs)
var buffersrc *avfilter.Filter
var buffersink *avfilter.Filter
if false {
buffersrc = avfilter.AvfilterGetByName("abuffer")
buffersink = avfilter.AvfilterGetByName("abuffersink")
} else {
buffersrc = avfilter.AvfilterGetByName("buffer")
buffersink = avfilter.AvfilterGetByName("buffersink")
}
if buffersink == nil || buffersrc == nil {
log.Critical("AvfilterGetByName Failed.")
return
}
ret := graph.AvfilterGraphParse2(des, &inputs, &outputs)
if ret < 0 {
log.Critical("AvfilterInoutAlloc Failed des : ", avutil.ErrorFromCode(ret))
return
}
if qsv_device.Context() != nil {
for _, v := range graph.Filters() {
v.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))
}
}
var ins []*avfilter.Context
var outs []*avfilter.Context
var frames []*avutil.Frame
// inputs
index := 0
for cur := inputs; cur != nil; cur = cur.Next() {
//log.Debug("index :", index)
var in *avfilter.Context
//var args = "video_size=1280x720:pix_fmt=0:time_base=1/30:pixel_aspect=1/1"
inName := "in" + strconv.Itoa(index)
ret = avfilter.AvfilterGraphCreateFilter(&in, buffersrc, inName, args, 0, graph)
if ret < 0 {
log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))
return
}
par := avfilter.AvBuffersrcParametersAlloc()
if par == nil {
log.Critical("AvBuffersrcParametersAlloc Failed.")
return
}
par.SetHwFramesCtx(avutil.AVBufferRef(hw_frames_ref))
ret = in.AvBuffersrcParametersSet(par)
if ret < 0 {
log.Critical("AvBuffersrcParametersSet Failed.")
return
}
avutil.AvFreep(unsafe.Pointer(&par))
ins = append(ins, in)
ret = avfilter.AvfilterLink(ins[index], 0, cur.FilterContext(), cur.PadIdx())
if ret < 0 {
log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))
return
}
index++
}
// outputs
index = 0
for cur := outputs; cur != nil; cur = cur.Next() {
var out *avfilter.Context
outName := "out" + strconv.Itoa(index)
ret = avfilter.AvfilterGraphCreateFilter(&out, buffersink, outName, "", 0, graph)
if ret < 0 {
log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))
return
}
outs = append(outs, out)
ret = avfilter.AvfilterLink(cur.FilterContext(), cur.PadIdx(), outs[index], 0)
if ret < 0 {
log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))
return
}
index++
f := avutil.AvFrameAlloc()
if f == nil {
log.Critical("AvFrameAlloc failed.")
return
}
frames = append(frames, f)
}
ret = graph.AvfilterGraphConfig(0)
if ret < 0 {
log.Critical("AvfilterGraphConfig Failed des : ", avutil.ErrorFromCode(ret))
return
}
overlayLatency := latency.New("qsv", "overlay")
context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))
var dict *avutil.Dictionary = avutil.AvDictAlloc()
er := dict.AvDictSet("profile", "high", 0)
if er < 0 {
log.Critical("AvDictSet failed")
return
}
er = dict.AvDictSet("level", "52", 0)
if er < 0 {
log.Critical("AvDictSet failed")
return
}
fmt.Println(dict.AvDictCount())
err = context_enc.AvcodecOpen2(codec_enc, (**avcodec.Dictionary)(unsafe.Pointer(&dict)))
if err < 0 {
log.Critical("AvcodecOpen2 failed.")
return
}
fmt.Println(dict.AvDictCount())
data, erro := ioutil.ReadFile(input_h264)
if erro != nil {
log.Debug("File reading error", erro)
return
}
log.Debug("Open Success.")
l := len(data)
log.Debug("size of file:", l)
b := make([]byte, 4096 + 64)
file, erro := os.Create(output_h264)
if erro != nil {
log.Critical("Error Reading")
}
defer file.Close()
var pts int64 = 0
//var opts int64 = 0
//var frames [25]*avutil.Frame
started := false
sum := 0
for sum < l {
remain := 4096
for remain > 0 {
copy(b, data[sum:sum + 4096])
if !started {
decLatency.Start()
started = true
}
n := context.AvParserParse2(parserContext, pkt, b,
remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)
log.Debug("parser ", n, "bytes")
sum = sum + n
remain = remain - n;
//log.Trace("--------", dec.Packet().GetPacketSize())
if pkt.GetPacketSize() > 0 { // decode
//fmt.Println(decLatency.End())
ret := context.AvcodecSendPacket(pkt)
if ret < 0 {
log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))
continue
}
ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
decLatency.End()
fmt.Println(decLatency.End())
} else {
continue
}
overlayLatency.Start()
pts++
frame.SetPts(pts)
var fs []*avutil.Frame
for i := 0 ; i < 16 ; i++ {
fs = append(fs, avutil.AvFrameClone(frame))
ret = avfilter.AvBuffersrcAddFrame(ins[i], (*avfilter.Frame)(unsafe.Pointer(fs[i])))
if ret < 0 {
for i := 0 ; i < len(fs) ; i++ {
avutil.AvFrameFree(fs[i])
}
fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))
continue
}
}
avutil.AvFrameUnref(frame)
/*ret = avfilter.AvBuffersrcAddFrame(ins[0], (*avfilter.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))
continue
}*/
ret = avfilter.AvBufferSinkGetFrame(outs[0], (*avfilter.Frame)(unsafe.Pointer(frames[0])))
for i := 0 ; i < len(fs) ; i++ {
avutil.AvFrameFree(fs[i])
}
if ret == avutil.AvErrorEOF || ret == avutil.AvErrorEAGAIN {
log.Error(avutil.ErrorFromCode(ret))
continue
}
if ret < 0 {
log.Error("AvBufferSinkGetFrame Failed des : ", ret, avutil.ErrorFromCode(ret))
continue
}
frame = frames[0]
fmt.Println(overlayLatency.End())
if ret == 0 { // encode
started = false
//log.Debug("dec-frame:",frame)
encLatency.Start()
ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))
if ret < 0 {
log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))
continue
}
ret = context_enc.AvcodecReceivePacket(pkt_enc)
if ret < 0 {
log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))
continue
}
if ret == 0 {
data0 := pkt_enc.Data()
buf := make([]byte, pkt_enc.GetPacketSize())
start := uintptr(unsafe.Pointer(data0))
for i := 0; i < pkt_enc.GetPacketSize(); i++ {
elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))
buf[i] = elem
}
file.Write(buf)
//encLatency.End()
fmt.Println(encLatency.End())
}
avutil.AvFrameUnref(frame)
//avutil.AvFrameUnref(oframe)
}
}
}
}
}
func main() {
avutil.AvLogSetLevel(avutil.AV_LOG_TRACE)
//fmt.Println(filter.P1080InTestOverlay16)
//fmt.Println(avcodec.AV_PIX_FMT_QSV)
//fmt.Println(avcodec.AV_PIX_FMT_NV12)
//fmt.Println(avutil.ErrorFromCode(-15))
decode_overlay_encode()
time.Sleep(time.Hour)
//oclOpen()
}参考 : Wiki/QuickSync