Use of ffmpeg hardware decoder

What is hardware decoding?

Ordinary decoding is to use cpu to decode, that is, software decoding, hardware decoding is to use gpu to decode  

Why use hardware decoding? 

First of all, the biggest advantage is that the video played by the fast hard solution is relatively smooth, and it can prolong the playing time of the mobile device; while the soft solution increases the CPU workload due to the soft solution, which will occupy too much mobile CPU resources. If the CPU capacity is insufficient, Then the software will also be affected. The most important thing  is a quick word

How to use hardware decoding? 

ffmpeg internally provides us with a friendly interface to implement hardware decoding

Precautions

There are many codecs in ffmpeg, not all codecs support hardware decoding, and even codecs that support hardware decoding may not be able to support your graphics card. That is to say, when using hardware decoding, we must first judge the decoding. Whether the device supports hardware codec for this graphics card on this platform, otherwise it cannot be used  

Encapsulate and integrate the graphics card manufacturer's SDK to realize part of the hardware codec 

 

Secondly, the software codec in ffmpeg can achieve related hardware solution acceleration. For example, cuda acceleration, qsv acceleration, dxva2 acceleration, d3d11va acceleration, opencl acceleration, etc. can be used in the h264 decoder. cuda qsv, etc. are toolkits for gpu programming launched by different companies 

 

AV_CODEC_ID_H264; represents the h264 codec. And name represents a certain encoder or decoder. Usually we use avcodec_find_decoder(ID) and avcodec_find_encoder(ID) for decoder and encoder. The software codec used by default. If we need to use hardware codec, use avcodec_find_encoder_by_name(name) and avcodec_find_decoder_by_name(name) to specify the encoder. Other code flow is consistent with software codec.

	//codec = avcodec_find_decoder(AV_CODEC_ID_H264);
	codec = avcodec_find_decoder_by_name("h264_cuvid");
	if (!codec) {
		fprintf(stderr, "Codec not found\n");
		exit(1);
	}

The codec found by id may not be what you expected. The codec found by name must be what you want

The following is the official hardware decoding example of ffmpeg. I added Chinese notes for easy understanding.

 

#include <stdio.h>

#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/pixdesc.h>
#include <libavutil/hwcontext.h>
#include <libavutil/opt.h>
#include <libavutil/avassert.h>
#include <libavutil/imgutils.h>

static AVBufferRef *hw_device_ctx = NULL;
static enum AVPixelFormat hw_pix_fmt;
static FILE *output_file = NULL;

static int hw_decoder_init(AVCodecContext *ctx, const enum AVHWDeviceType type)
{
	int err = 0;
	//创建硬件设备信息上下文 
	if ((err = av_hwdevice_ctx_create(&hw_device_ctx, type,
		NULL, NULL, 0)) < 0) {
		fprintf(stderr, "Failed to create specified HW device.\n");
		return err;
	}
	//绑定编解码器上下文和硬件设备信息上下文
	ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);

	return err;
}

static enum AVPixelFormat get_hw_format(AVCodecContext *ctx,
	const enum AVPixelFormat *pix_fmts)
{
	const enum AVPixelFormat *p;

	for (p = pix_fmts; *p != -1; p++) {
		if (*p == hw_pix_fmt)
			return *p;
	}

	fprintf(stderr, "Failed to get HW surface format.\n");
	return AV_PIX_FMT_NONE;
}

static int decode_write(AVCodecContext *avctx, AVPacket *packet)
{
	AVFrame *frame = NULL, *sw_frame = NULL;
	AVFrame *tmp_frame = NULL;
	uint8_t *buffer = NULL;
	int size;
	int ret = 0;

	ret = avcodec_send_packet(avctx, packet);
	if (ret < 0) {
		fprintf(stderr, "Error during decoding\n");
		return ret;
	}

	while (1) {
		if (!(frame = av_frame_alloc()) || !(sw_frame = av_frame_alloc())) {
			fprintf(stderr, "Can not alloc frame\n");
			ret = AVERROR(ENOMEM);
			goto fail;
		}

		ret = avcodec_receive_frame(avctx, frame);
		if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
			av_frame_free(&frame);
			av_frame_free(&sw_frame);
			return 0;
		}
		else if (ret < 0) {
			fprintf(stderr, "Error while decoding\n");
			goto fail;
		}

		if (frame->format == hw_pix_fmt) {
			/* retrieve data from GPU to CPU */
			if ((ret = av_hwframe_transfer_data(sw_frame, frame, 0)) < 0) {
				fprintf(stderr, "Error transferring the data to system memory\n");
				goto fail;
			}
			tmp_frame = sw_frame;
		}
		else
			tmp_frame = frame;

		size = av_image_get_buffer_size(tmp_frame->format, tmp_frame->width,
			tmp_frame->height, 1);
		buffer = av_malloc(size);
		if (!buffer) {
			fprintf(stderr, "Can not alloc buffer\n");
			ret = AVERROR(ENOMEM);
			goto fail;
		}
		ret = av_image_copy_to_buffer(buffer, size,
			(const uint8_t * const *)tmp_frame->data,
			(const int *)tmp_frame->linesize, tmp_frame->format,
			tmp_frame->width, tmp_frame->height, 1);
		if (ret < 0) {
			fprintf(stderr, "Can not copy image to buffer\n");
			goto fail;
		}

		if ((ret = fwrite(buffer, 1, size, output_file)) < 0) {
			fprintf(stderr, "Failed to dump raw data.\n");
			goto fail;
		}

	fail:
		av_frame_free(&frame);
		av_frame_free(&sw_frame);
		av_freep(&buffer);
		if (ret < 0)
			return ret;
	}
}

int main(int argc, char *argv[])
{
	AVFormatContext *input_ctx = NULL;
	int video_stream, ret;
	AVStream *video = NULL;
	AVCodecContext *decoder_ctx = NULL;
	AVCodec *decoder = NULL;
	AVPacket packet;
	enum AVHWDeviceType type;
	int i;

	if (argc < 4) {
		fprintf(stderr, "Usage: %s <device type> <input file> <output file>\n", argv[0]);
		return -1;
	}
	//通过你传入的名字来找到对应的硬件解码类型
	type = av_hwdevice_find_type_by_name(argv[1]);
	if (type == AV_HWDEVICE_TYPE_NONE) {
		fprintf(stderr, "Device type %s is not supported.\n", argv[1]);
		fprintf(stderr, "Available device types:");
		while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE)
			fprintf(stderr, " %s", av_hwdevice_get_type_name(type));
		fprintf(stderr, "\n");
		return -1;
	}

	/* open the input file */
	if (avformat_open_input(&input_ctx, argv[2], NULL, NULL) != 0) {
		fprintf(stderr, "Cannot open input file '%s'\n", argv[2]);
		return -1;
	}

	if (avformat_find_stream_info(input_ctx, NULL) < 0) {
		fprintf(stderr, "Cannot find input stream information.\n");
		return -1;
	}

	/* find the video stream information */
	ret = av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &decoder, 0);
	if (ret < 0) {
		fprintf(stderr, "Cannot find a video stream in the input file\n");
		return -1;
	}
	video_stream = ret;
	//去遍历所有编解码器支持的硬件解码配置 如果和之前你指定的是一样的 那么就可以继续执行了 不然就找不到
	for (i = 0;; i++) {
		const AVCodecHWConfig *config = avcodec_get_hw_config(decoder, i);
		if (!config) {
			fprintf(stderr, "Decoder %s does not support device type %s.\n",
				decoder->name, av_hwdevice_get_type_name(type));
			return -1;
		}
		if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
			config->device_type == type) {
			//把硬件支持的像素格式设置进去
			hw_pix_fmt = config->pix_fmt;
			break;
		}
	}

	if (!(decoder_ctx = avcodec_alloc_context3(decoder)))
		return AVERROR(ENOMEM);

	video = input_ctx->streams[video_stream];
	if (avcodec_parameters_to_context(decoder_ctx, video->codecpar) < 0)
		return -1;
	//填入回调函数 通过这个函数 编解码器能够知道显卡支持的像素格式
	decoder_ctx->get_format = get_hw_format;

	if (hw_decoder_init(decoder_ctx, type) < 0)
		return -1;
   //绑定完成后 打开编解码器
	if ((ret = avcodec_open2(decoder_ctx, decoder, NULL)) < 0) {
		fprintf(stderr, "Failed to open codec for stream #%u\n", video_stream);
		return -1;
	}

	/* open the file to dump raw data */
	output_file = fopen(argv[3], "w+");

	/* actual decoding and dump the raw data */
	while (ret >= 0) {
		if ((ret = av_read_frame(input_ctx, &packet)) < 0)
			break;

		if (video_stream == packet.stream_index)
			ret = decode_write(decoder_ctx, &packet);


		av_packet_unref(&packet);
	}

	/* flush the decoder */
	packet.data = NULL;
	packet.size = 0;
	ret = decode_write(decoder_ctx, &packet);
	av_packet_unref(&packet);

	if (output_file)
		fclose(output_file);
	avcodec_free_context(&decoder_ctx);
	avformat_close_input(&input_ctx);
	av_buffer_unref(&hw_device_ctx);

	return 0;
}

 Analysis of key functions 

enum AVHWDeviceType av_hwdevice_find_type_by_name(const char *name);

 Find the corresponding hardware type through the parameters passed in. The AVHWDeviceType value is as follows

 

 const AVCodecHWConfig *avcodec_get_hw_config(const AVCodec *codec, int index);

Get the hardware configuration supported by the codec, such as the pixel format supported by the hardware, etc.

 

 

static enum AVPixelFormat get_hw_format(AVCodecContext *ctx,
	const enum AVPixelFormat *pix_fmts)
{
	const enum AVPixelFormat *p;

	for (p = pix_fmts; *p != -1; p++) {
		if (*p == hw_pix_fmt)
			return *p;
	}

	fprintf(stderr, "Failed to get HW surface format.\n");
	return AV_PIX_FMT_NONE;
}

 

This is a callback function, and its function is to tell the decoder codec what its target pixel format is. After obtaining the target format that the hard decoder codec can support in the previous step, inform the codec through this callback function

1. fmt is the pixel format supported by this decoder codec, and sorted according to the quality;
2. If there is no special need, this step can be omitted. The "native" format is used internally by default.

int av_hwdevice_ctx_create(AVBufferRef **pdevice_ref, enum AVHWDeviceType type, const char *device, AVDictionary *opts, int flags) 

The function of this function is to create the context information AVHWDeviceContext related to the hardware device, including allocating memory resources and initializing the hardware device.

After preparing the hardware device context AVHWDeviceContext, you need to bind this information to the AVCodecContext, and then you can perform the decoding operation in the same process as the soft solution. The binding operation is as follows

 Note that the hardware device information context here is a reference stored through AVBuffer, not an entity

 

int av_hwframe_transfer_data(AVFrame *dst, const AVFrame *src, int flags) 

This function is responsible for data exchange between cpu memory and hardware memory (the original text is hw surface). In other words, it can not only move data from the hardware surface back to the system memory, but also supports reverse operations; it can even directly interact with data between hardware memories.