文章目录
1.ffmpeg介绍
项目地址：https://github.com/FFmpeg/FFmpeg
音视频处理工具，分为库和命令行工具两部分。
Libraries:
- libavcodec provides implementation of a wider range of codecs.
- libavformat implements streaming protocols, container formats and basic I/O access.
- libavutil includes hashers, decompressors and miscellaneous utility functions.
- libavfilter provides means to alter decoded audio and video through a directed graph of connected filters.
- libavdevice provides an abstraction to access capture and playback devices.
- libswresample implements audio mixing and resampling routines.
- libswscale implements color conversion and scaling routines.
Tools:
- ffmpeg is a command line toolbox to manipulate, convert and stream multimedia content.
- ffplay is a minimalistic multimedia player.
- ffprobe is a simple analysis tool to inspect multimedia content.
- Additional small tools such as aviocat, ismindex and qt-faststart.
2. Hello World及输出版本号
2.1代码
#include <stdio.h>
extern "C" {
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libavutil/ffversion.h"
#include "libswresample/swresample.h"
#include "libswscale/swscale.h"
#include "libpostproc/postprocess.h"
}
int main(int argc,char* argv[])
{
av_log_set_level(AV_LOG_DEBUG);
av_log(NULL,AV_LOG_INFO,"Hello ffmpeg:%s %s\n",argv[0],argv[1]);
unsigned codecVer = avcodec_version();
av_log(NULL,AV_LOG_INFO,"FFmpeg version is: %s, avcodec version is: %d\n",FFMPEG_VERSION,codecVer);
return 0;
}
2.2 编译运行
% gcc main.cpp -lavutil -lavcodec -o a.out
% ./a.out
Hello ffmpeg:./a.out (null)
Current ffmpeg version is: 4.2.2 ,avcodec version is: 3815012=58.54.100
% ffmpeg
ffmpeg version 4.2.2 Copyright (c) 2000-2019 the FFmpeg developers
built with gcc 9 (Ubuntu 9.4.0-1ubuntu1~20.04.1)
configuration:
libavutil 56. 31.100 / 56. 31.100
libavcodec 58. 54.100 / 58. 54.100
libavformat 58. 29.100 / 58. 29.100
libavdevice 58. 8.100 / 58. 8.100
libavfilter 7. 57.100 / 7. 57.100
libswscale 5. 5.100 / 5. 5.100
libswresample 3. 5.100 / 3. 5.100
Hyper fast Audio and Video encoder
usage: ffmpeg [options] [[infile options] -i infile]... {
[outfile options] outfile}...
3.截取视频帧保存图片ppm
把视频中的某些帧保存为ppm格式的图片，其实就是直接保存rgb的字节内容。ffmpeg解码后得到的帧类型是yuv，所以需要帧类型转换。
3.1代码
#include <stdio.h>
#include <stdlib.h>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libavutil/ffversion.h"
#include "libswresample/swresample.h"
#include "libswscale/swscale.h"
#include "libpostproc/postprocess.h"
#include "libavutil/imgutils.h"
}
//将FFmpeg解码后的数据保存到本地文件
void saveFrame(AVFrame* pFrame, int width, int height, int iFrame)
{
FILE* pFile;
char szFilename[32];
int y;
// 打开文件
sprintf(szFilename, "/home/chao/testffmpeg/frame%d.ppm", iFrame);
pFile = fopen(szFilename, "wb");
if (pFile == NULL)
return;
// 写入文件头
fprintf(pFile, "P6\n%d %d\n255\n", width, height);
// 写入像素数据
for (y = 0; y < height; y++)
fwrite(pFrame->data[0] + y * pFrame->linesize[0], 1, width * 3, pFile);
// 关闭文件
fclose(pFile);
}
int main() {
char filePath[] = "/home/chao/testffmpeg/19.mp4";//文件地址
int videoStreamIndex = -1;//视频流所在流序列中的索引
int ret = 0;//默认返回值
//需要的变量名并初始化
AVFormatContext* fmtCtx = NULL;
AVPacket* pkt = NULL;
AVCodecContext* codecCtx = NULL;
AVCodecParameters* avCodecPara = NULL;
AVCodec* codec = NULL;
AVFrame* yuvFrame = av_frame_alloc();
AVFrame* rgbFrame = av_frame_alloc();
do {
//=========================== 创建AVFormatContext结构体 ===============================//
//分配一个AVFormatContext，FFMPEG所有的操作都要通过这个AVFormatContext来进行
fmtCtx = avformat_alloc_context();
//==================================== 打开文件 ======================================//
if ((ret = avformat_open_input(&fmtCtx, filePath, NULL, NULL)) != 0) {
printf("cannot open video file\n");
break;
}
//=================================== 获取视频流信息 ===================================//
if ((ret = avformat_find_stream_info(fmtCtx, NULL)) < 0) {
printf("cannot retrive video info\n");
break;
}
//循环查找视频中包含的流信息，直到找到视频类型的流
//便将其记录下来 保存到videoStreamIndex变量中
for (unsigned int i = 0; i < fmtCtx->nb_streams; i++) {
if (fmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
videoStreamIndex = i;
break;//找到视频流就退出
}
}
//如果videoStream为-1 说明没有找到视频流
if (videoStreamIndex == -1) {
printf("cannot find video stream\n");
break;
}
//打印输入和输出信息：长度 比特率 流格式等
av_dump_format(fmtCtx, 0, filePath, 0);
//================================= 查找解码器 ===================================//
avCodecPara = fmtCtx->streams[videoStreamIndex]->codecpar;
codec = avcodec_find_decoder(avCodecPara->codec_id);
if (codec == NULL) {
printf("cannot find decoder\n");
break;
}
//根据解码器参数来创建解码器内容
codecCtx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(codecCtx, avCodecPara);
if (codecCtx == NULL) {
printf("Cannot alloc context.");
break;
}
//================================ 打开解码器 ===================================//
if ((ret = avcodec_open2(codecCtx, codec, NULL)) < 0) {
// 具体采用什么解码器ffmpeg经过封装 我们无须知道
printf("cannot open decoder\n");
break;
}
//================================ 设置数据转换参数 ================================//
struct SwsContext* img_ctx = sws_getContext(
codecCtx->width, codecCtx->height, codecCtx->pix_fmt, //源地址长宽以及数据格式
codecCtx->width, codecCtx->height, AV_PIX_FMT_RGB32, //目的地址长宽以及数据格式
SWS_BICUBIC, NULL, NULL, NULL); //算法类型 AV_PIX_FMT_YUVJ420P AV_PIX_FMT_BGR24
//==================================== 分配空间 ==================================//
//一帧图像数据大小
int numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB32, codecCtx->width, codecCtx->height, 1);
unsigned char* out_buffer = (unsigned char*)av_malloc(numBytes * sizeof(unsigned char));
//=========================== 分配AVPacket结构体 ===============================//
int i = 0;//用于帧计数
pkt = av_packet_alloc(); //分配一个packet
av_new_packet(pkt, codecCtx->width * codecCtx->height); //调整packet的数据
//会将pFrameRGB的数据按RGB格式自动"关联"到buffer 即pFrameRGB中的数据改变了
//out_buffer中的数据也会相应的改变
av_image_fill_arrays(rgbFrame->data, rgbFrame->linesize, out_buffer, AV_PIX_FMT_RGB32,
codecCtx->width, codecCtx->height, 1);
//=========================== 读取视频信息 ===============================//
while (av_read_frame(fmtCtx, pkt) >= 0) {
//读取的是一帧视频 数据存入一个AVPacket的结构中
//printf("ptk->pts:%ld\t", pkt->pts);
if (pkt->stream_index == videoStreamIndex) {
printf("video ptk->pts:%ld\t", pkt->pts);
if (avcodec_send_packet(codecCtx, pkt) == 0) {
while (avcodec_receive_frame(codecCtx, yuvFrame) == 0) {
if (++i <= 500 && i >= 200) {
sws_scale(img_ctx,
(const uint8_t* const*)yuvFrame->data,
yuvFrame->linesize,
0,
codecCtx->height,
rgbFrame->data,
rgbFrame->linesize);
saveFrame(rgbFrame, codecCtx->width, codecCtx->height, i);
}
}
}
}
av_packet_unref(pkt);//重置pkt的内容
}
printf("There are %d frames int total.\n", i);
} while (0);
//===========================释放所有指针===============================//
av_packet_free(&pkt);
avcodec_close(codecCtx);
//avcodec_parameters_free(&avCodecPara);
avformat_close_input(&fmtCtx);
avformat_free_context(fmtCtx);
av_frame_free(&yuvFrame);
av_frame_free(&rgbFrame);
return ret;
}
3.2 说明
- 一次avcodec_send_packet后会调用多次avcodec_receive_frame，这是由于视频的编码造成的，比如B帧的存在，ffmpeg的解码器需要积攒后边的多帧后才能解码出B帧。
- avcodec_receive_frame的第二个参数为返回的解码帧，一般类型为YUV420P, 因为要保存PPM文件，而PPM文件中的内容为RBG的像素值，因此需要调用sws_scale进行帧格式的转换。
4.解码视频文件(mp4)为yuv文件
yuv文件就是完全未经压缩的视频文件，文件大小会比较大。ffmpeg解码后的视频帧是yuv420格式，保存为yuv视频无需格式转换，但是保存的时候需要分别保存y，u，v三路数据。需要理解yuv420的格式定义。
4.1 代码
#include <iostream>
#include <stdio.h>
#include <stdlib.h>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libavutil/ffversion.h"
#include "libswresample/swresample.h"
#include "libswscale/swscale.h"
#include "libpostproc/postprocess.h"
#include "libavutil/imgutils.h"
}
int main() {
FILE* fp = fopen("/mnt/d/result.yuv", "w+b");
if (fp == NULL) {
printf("Cannot open file.\n");
return -1;
}
char filePath[] = "/home/chao/testffmpeg/19.mp4";//文件地址
int videoStreamIndex = -1;//视频流所在流序列中的索引
int ret = 0;//默认返回值
//需要的变量名并初始化
AVFormatContext* fmtCtx = NULL;
AVPacket* pkt = NULL;
AVCodecContext* codecCtx = NULL;
AVCodecParameters* avCodecPara = NULL;
AVCodec* codec = NULL;
AVFrame* yuvFrame = av_frame_alloc();
do {
//=========================== 创建AVFormatContext结构体 ===============================//
//分配一个AVFormatContext，FFMPEG所有的操作都要通过这个AVFormatContext来进行
fmtCtx = avformat_alloc_context();
//==================================== 打开文件 ======================================//
if ((ret = avformat_open_input(&fmtCtx, filePath, NULL, NULL)) != 0) {
printf("cannot open video file\n");
break;
}
//=================================== 获取视频流信息 ===================================//
if ((ret = avformat_find_stream_info(fmtCtx, NULL)) < 0) {
printf("cannot retrive video info\n");
break;
}
//循环查找视频中包含的流信息，直到找到视频类型的流
//便将其记录下来 保存到videoStreamIndex变量中
for (unsigned int i = 0; i < fmtCtx->nb_streams; i++) {
if (fmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
videoStreamIndex = i;
break;//找到视频流就退出
}
}
//如果videoStream为-1 说明没有找到视频流
if (videoStreamIndex == -1) {
printf("cannot find video stream\n");
break;
}
//打印输入和输出信息：长度 比特率 流格式等
av_dump_format(fmtCtx, 0, filePath, 0);
//================================= 查找解码器 ===================================//
avCodecPara = fmtCtx->streams[videoStreamIndex]->codecpar;
codec = avcodec_find_decoder(avCodecPara->codec_id);
if (codec == NULL) {
printf("cannot find decoder\n");
break;
}
//根据解码器参数来创建解码器内容
codecCtx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(codecCtx, avCodecPara);
if (codecCtx == NULL) {
printf("Cannot alloc context.");
break;
}
//================================ 打开解码器 ===================================//
if ((ret = avcodec_open2(codecCtx, codec, NULL)) < 0) {
// 具体采用什么解码器ffmpeg经过封装 我们无须知道
printf("cannot open decoder\n");
break;
}
int w = codecCtx->width;//视频宽度
int h = codecCtx->height;//视频高度
//=========================== 分配AVPacket结构体 ===============================//
pkt = av_packet_alloc(); //分配一个packet
av_new_packet(pkt, codecCtx->width * codecCtx->height); //调整packet的数据
//=========================== 读取视频信息 ===============================//
while (av_read_frame(fmtCtx, pkt) >= 0) {
//读取的是一帧视频 数据存入一个AVPacket的结构中
if (pkt->stream_index == videoStreamIndex) {
if (avcodec_send_packet(codecCtx, pkt) == 0) {
while (avcodec_receive_frame(codecCtx, yuvFrame) == 0) {
fwrite(yuvFrame->data[0], 1, w * h, fp);//y
fwrite(yuvFrame->data[1], 1, w * h / 4, fp);//u
fwrite(yuvFrame->data[2], 1, w * h / 4, fp);//v
/*for (int i = 0; i < yuvFrame->height; i++) {
//out.write((char*)(yuvFrame->data[0] + i * yuvFrame->linesize[0]), yuvFrame->width);
fwrite(yuvFrame->data[0] + i * yuvFrame->linesize[0], 1, yuvFrame->width, fp);
}
int loop = yuvFrame->height / 2;
int len_uv = yuvFrame->width / 2;
for (int i = 0; i < loop; i++) {
//out.write((char*)(yuvFrame->data[1] + i * yuvFrame->linesize[1]), len_uv);
fwrite(yuvFrame->data[1] + i * yuvFrame->linesize[1], 1, len_uv, fp);
}
for (int i = 0; i < loop; i++) {
//out.write((char*)(yuvFrame->data[2] + i * yuvFrame->linesize[2]), len_uv);
fwrite(yuvFrame->data[2] + i * yuvFrame->linesize[2], 1, len_uv, fp);
}
*/
}
}
}
av_packet_unref(pkt);//重置pkt的内容
}
} while (0);
//===========================释放所有指针===============================//
av_packet_free(&pkt);
avcodec_close(codecCtx);
avcodec_parameters_free(&avCodecPara);
//avformat_close_input(&fmtCtx);
//avformat_free_context(fmtCtx);
av_frame_free(&yuvFrame);
return ret;
}
4.2 说明
- YUV420格式的说明见《图像格式RGB/HSV/YUV》，《YUV格式简介、YUV444、YUV422、YUV420》，领会了YUV420的格式，才能明白代码中
fwrite(yuvFrame->data[1], 1, w * h / 4, fp)的含义。 - 如何播放YUV视频，可以使用ffplay,也可以使用vlc，如果使用vlc播放，需要点击媒体 -> 高级打开 -> 显示更多选项，在编辑选项中加入
demux=rawvideo :rawvid-width=1920 :rawvid-height=1080 :rawvid-chroma=I420 :rawvid-fps=30，以此来指定视频的画面大小，帧格式，帧率信息。 其中的rawvid-chroma可选项见Chroma
5.视频解码硬件加速
使用cuda进行视频解码的加速。需要有nvida的显卡，还需安装开发包，还需ffmpeg编译为支持cuda的版本。cuda对速度的提升比较明显。
5.1代码
#include <stdio.h>
#include <stdlib.h>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libavutil/ffversion.h"
#include "libswresample/swresample.h"
#include "libswscale/swscale.h"
#include "libpostproc/postprocess.h"
#include "libavutil/imgutils.h"
#include "libavutil/hwcontext.h"
#include "libavutil/pixdesc.h"
#include "libavutil/imgutils.h"
#include "libavutil/opt.h"
}
static enum AVPixelFormat hw_pix_fmt;
static AVBufferRef* hw_device_ctx = NULL;
AVPixelFormat get_hw_format(AVCodecContext* ctx, const AVPixelFormat* pix_fmts)
{
const enum AVPixelFormat* p;
for (p = pix_fmts; *p != -1; p++) {
if (*p == hw_pix_fmt)
return *p;
}
fprintf(stderr, "Failed to get HW surface format.\n");
return AV_PIX_FMT_NONE;
}
static int hw_decoder_init(AVCodecContext* ctx, const enum AVHWDeviceType type)
{
int err = 0;
if ((err = av_hwdevice_ctx_create(&hw_device_ctx, type,
NULL, NULL, 0)) < 0) {
fprintf(stderr, "Failed to create specified HW device.\n");
return err;
}
ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
return err;
}
int main()
{
//=========================== 查找所有硬件=================================================//
AVHWDeviceType type_ = AV_HWDEVICE_TYPE_NONE;
while ((type_ = av_hwdevice_iterate_types(type_)) != AV_HWDEVICE_TYPE_NONE)
fprintf(stderr, "found: %s\n", av_hwdevice_get_type_name(type_));
//=========================== 通过名称查找硬解码类型是否存在 ===============================//
AVHWDeviceType type = av_hwdevice_find_type_by_name("cuda");
if (type == AV_HWDEVICE_TYPE_NONE) {
fprintf(stderr, "Device type %s is not supported.\n", "h264_cuvid");
fprintf(stderr, "Available device types:");
while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE)
fprintf(stderr, " %s", av_hwdevice_get_type_name(type));
fprintf(stderr, "\n");
return -1;
}
printf("found cuda!!\n");
FILE* fp = fopen("/mnt/d/result.yuv", "w+b");
if (fp == NULL) {
printf("Cannot open file.\n");
return -1;
}
char filePath[] = "/home/chao/testffmpeg/19.mp4";//文件地址
int videoStreamIndex = -1;//视频流所在流序列中的索引
int ret = 0;//默认返回值
//需要的变量名并初始化
AVFormatContext* fmtCtx = NULL;
AVCodec* videoCodec = NULL;
AVCodecContext* videoCodecCtx = NULL;
AVPacket* pkt = NULL;
AVFrame* yuvFrame = NULL;
AVFrame* rgbFrame = NULL;
AVFrame* nv12Frame = NULL;
AVStream* videoStream = NULL;
AVBufferRef* hw_device_ctx = NULL;
unsigned char* out_buffer;
struct SwsContext* img_ctx = NULL;
fmtCtx = avformat_alloc_context();
pkt = av_packet_alloc();
yuvFrame = av_frame_alloc();
rgbFrame = av_frame_alloc();
nv12Frame = av_frame_alloc();
/* open the input file */
if (avformat_open_input(&fmtCtx, filePath, NULL, NULL) != 0) {
printf("open input mp4 file failed\n");
return -1;
}
if (avformat_find_stream_info(fmtCtx, NULL) < 0) {
fprintf(stderr, "Cannot find input stream information.\n");
return -1;
}
/* find the video stream information */
ret = av_find_best_stream(fmtCtx, AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0);
if (ret < 0) {
fprintf(stderr, "Cannot find a video stream in the input file\n");
return -1;
}
videoStreamIndex = ret;
//获取支持该decoder的hw配置型
for (int i = 0;; i++) {
const AVCodecHWConfig* config = avcodec_get_hw_config(videoCodec, i);
if (!config) {
fprintf(stderr, "Decoder %s does not support device type %s.\n",
videoCodec->name, av_hwdevice_get_type_name(type));
return -1;
}
if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
config->device_type == type) {
hw_pix_fmt = config->pix_fmt;
break;
}
}
if (!(videoCodecCtx = avcodec_alloc_context3(videoCodec)))
return AVERROR(ENOMEM);
videoStream = fmtCtx->streams[videoStreamIndex];
if (avcodec_parameters_to_context(videoCodecCtx, videoStream->codecpar) < 0)
return -1;
videoCodecCtx->get_format = get_hw_format;
if (hw_decoder_init(videoCodecCtx, type) < 0)
return -1;
if ((ret = avcodec_open2(videoCodecCtx, videoCodec, NULL)) < 0) {
fprintf(stderr, "Failed to open codec for stream #%u\n", videoStreamIndex);
return -1;
}
if ((ret = av_hwdevice_ctx_create(&hw_device_ctx, type,
NULL, NULL, 0)) < 0) {
fprintf(stderr, "Failed to create specified HW device.\n");
return ret;
}
videoCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
//if (hw_decoder_init(videoCodecCtx, type) < 0)
// return -1;
if ((ret = avcodec_open2(videoCodecCtx, videoCodec, NULL)) < 0) {
fprintf(stderr, "Failed to open codec for stream #%u\n", videoStreamIndex);
return -1;
}
img_ctx = sws_getContext(videoCodecCtx->width,
videoCodecCtx->height,
AV_PIX_FMT_NV12,
videoCodecCtx->width,
videoCodecCtx->height,
AV_PIX_FMT_RGB32,
SWS_BICUBIC, NULL, NULL, NULL);
unsigned numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB32, videoCodecCtx->width, videoCodecCtx->height, 1);
out_buffer = (unsigned char*)av_malloc(numBytes * sizeof(unsigned char));
int res = av_image_fill_arrays(
rgbFrame->data, rgbFrame->linesize,
out_buffer, AV_PIX_FMT_RGB32,
videoCodecCtx->width, videoCodecCtx->height, 1);
if (res < 0) {
printf("failed array fill\n");
return -1;
}
int w = videoCodecCtx->width;//视频宽度
int h = videoCodecCtx->height;//视频高度
while (av_read_frame(fmtCtx, pkt) >= 0)
{
if (pkt->stream_index == videoStreamIndex) {
if (avcodec_send_packet(videoCodecCtx, pkt) >= 0) {
int ret;
while ((ret = avcodec_receive_frame(videoCodecCtx, yuvFrame)) >= 0) {
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return -1;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
exit(1);
}
if (yuvFrame->format == videoCodecCtx->pix_fmt) {
if ((ret = av_hwframe_transfer_data(nv12Frame, yuvFrame, 0)) < 0) {
continue;
}
}
//sws_scale(img_ctx,
// (const uint8_t* const*)nv12Frame->data,
// (const int*)nv12Frame->linesize,
// 0,
// nv12Frame->height,
// rgbFrame->data, rgbFrame->linesize);
fwrite(yuvFrame->data[0], 1, w * h, fp);//y
fwrite(yuvFrame->data[1], 1, w * h / 4, fp);//u
fwrite(yuvFrame->data[2], 1, w * h / 4, fp);//v
}
}
av_packet_unref(pkt);
}
}
if (!pkt) av_packet_free(&pkt);
if (!yuvFrame) av_frame_free(&yuvFrame);
if (!rgbFrame) av_frame_free(&rgbFrame);
if (!nv12Frame) av_frame_free(&nv12Frame);
if (!videoCodecCtx) avcodec_free_context(&videoCodecCtx);
if (!videoCodecCtx) avcodec_close(videoCodecCtx);
if (!fmtCtx) avformat_close_input(&fmtCtx);
return 0;
}
5.2说明
- 关于硬件加速的官方示例见hw_decode.c
- 查看ffmpeg支持的硬件加速选项：
ffmpeg -hwaccels - 网上资料看到速度可以提升3-4倍左右
6. 视频编码为h264
也就是把yuv的文件编码为h264的文件，文件会大幅度压缩。
6.1 代码
#include <iostream>
#include <stdio.h>
#include <stdlib.h>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libavutil/ffversion.h"
#include "libswresample/swresample.h"
#include "libswscale/swscale.h"
#include "libpostproc/postprocess.h"
#include "libavutil/imgutils.h"
#include "libavutil/opt.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include "libavutil/channel_layout.h"
#include "libavutil/pixdesc.h"
}
int main()
{
AVFormatContext* fmtCtx = NULL;
AVOutputFormat* outFmt = NULL;
AVStream* vStream = NULL;
AVCodecContext* codecCtx = NULL;
AVCodec* codec = NULL;
AVPacket* pkt = av_packet_alloc(); //创建已编码帧
uint8_t* picture_buf = NULL;
AVFrame* picFrame = NULL;
size_t size;
int ret = -1;
//[1]!打开视频文件
FILE* in_file = fopen("/mnt/d/media/akiyo_qcif.yuv", "rb");
if (!in_file) {
printf("can not open file!\n");
return -1;
}
//[1]!
do {
//[2]!打开输出文件，并填充fmtCtx数据
int in_w = 176, in_h = 144, frameCnt = 300;
const char* outFile = "/mnt/d/media/encoderesult.h264";
if (avformat_alloc_output_context2(&fmtCtx, NULL, NULL, outFile) < 0) {
printf("Cannot alloc output file context.\n");
break;
}
outFmt = fmtCtx->oformat;
//[2]!
//[3]!打开输出文件
if (avio_open(&fmtCtx->pb, outFile, AVIO_FLAG_READ_WRITE) < 0) {
printf("output file open failed.\n");
break;
}
//[3]!
//[4]!创建h264视频流，并设置参数
vStream = avformat_new_stream(fmtCtx, codec);
if (vStream == NULL) {
printf("failed create new video stream.\n");
break;
}
vStream->time_base.den = 25;
vStream->time_base.num = 1;
//[4]!
//[5]!编码参数相关
AVCodecParameters* codecPara = fmtCtx->streams[vStream->index]->codecpar;
codecPara->codec_type = AVMEDIA_TYPE_VIDEO;
codecPara->width = in_w;
codecPara->height = in_h;
//[5]!
//[6]!查找编码器
codec = avcodec_find_encoder(outFmt->video_codec);
if (codec == NULL) {
printf("Cannot find any endcoder.\n");
break;
}
//[6]!
//[7]!设置编码器内容
codecCtx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(codecCtx, codecPara);
if (codecCtx == NULL) {
printf("Cannot alloc context.\n");
break;
}
codecCtx->codec_id = outFmt->video_codec;
codecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
codecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
codecCtx->width = in_w;
codecCtx->height = in_h;
codecCtx->time_base.num = 1;
codecCtx->time_base.den = 25;
codecCtx->bit_rate = 400000;
codecCtx->gop_size = 12;
if (codecCtx->codec_id == AV_CODEC_ID_H264) {
codecCtx->qmin = 10;
codecCtx->qmax = 51;
codecCtx->qcompress = (float)0.6;
}
//codecCtx->max_b_frames = 0;
if (codecCtx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
codecCtx->max_b_frames = 2;
if (codecCtx->codec_id == AV_CODEC_ID_MPEG1VIDEO)
codecCtx->mb_decision = 2;
//[7]!
//[8]!打开编码器
if (avcodec_open2(codecCtx, codec, NULL) < 0) {
printf("Open encoder failed.\n");
break;
}
//[8]!
av_dump_format(fmtCtx, 0, outFile, 1);//输出 输出文件流信息
//初始化帧
picFrame = av_frame_alloc();
picFrame->width = codecCtx->width;
picFrame->height = codecCtx->height;
picFrame->format = codecCtx->pix_fmt;
size = (size_t)av_image_get_buffer_size(codecCtx->pix_fmt, codecCtx->width, codecCtx->height, 1);
picture_buf = (uint8_t*)av_malloc(size);
av_image_fill_arrays(picFrame->data, picFrame->linesize,
picture_buf, codecCtx->pix_fmt,
codecCtx->width, codecCtx->height, 1);
//[9] --写头文件
ret = avformat_write_header(fmtCtx, NULL);
//[9]
int y_size = codecCtx->width * codecCtx->height;
av_new_packet(pkt, (int)(size * 3));
//[10] --循环编码每一帧
for (int i = 0; i < frameCnt; i++) {
//读入YUV
if (fread(picture_buf, 1, (unsigned long)(y_size * 3 / 2), in_file) <= 0) {
printf("read file fail!\n");
return -1;
}
else if (feof(in_file))
break;
picFrame->data[0] = picture_buf; //亮度Y
picFrame->data[1] = picture_buf + y_size; // U
picFrame->data[2] = picture_buf + y_size * 5 / 4; // V
// AVFrame PTS
picFrame->pts = i;
//编码
if (avcodec_send_frame(codecCtx, picFrame) >= 0) {
while (avcodec_receive_packet(codecCtx, pkt) >= 0) {
printf("encoder success!\n");
// parpare packet for muxing
pkt->stream_index = vStream->index;
av_packet_rescale_ts(pkt, codecCtx->time_base, vStream->time_base);
pkt->pos = -1;
ret = av_interleaved_write_frame(fmtCtx, pkt);
if (ret < 0) {
//printf("error is: %s.\n", av_err2str(ret));
}
av_packet_unref(pkt);//刷新缓存
}
}
}
//[10]
//[11] --Flush encoder
//ret = flush_encoder(fmtCtx, codecCtx, vStream->index);
//if (ret < 0) {
// printf("flushing encoder failed!\n");
// break;
//}
//[11]
//[12] --写文件尾
av_write_trailer(fmtCtx);
//[12]
} while (0);
//释放内存
av_packet_free(&pkt);
avcodec_close(codecCtx);
av_free(picFrame);
av_free(picture_buf);
if (fmtCtx) {
avio_close(fmtCtx->pb);
avformat_free_context(fmtCtx);
}
fclose(in_file);
return 0;
}
6.2 说明
- 控制压缩效率可以通过以下两个参数进行调节：
- gop_size：指定两个关键帧之间的距离，如果设置为0，则全部都是I帧，压缩效果最差。
- max_b_frames： 最大B帧数，控制B帧的数量，如果设置为0则不会有B帧
- 压缩前后文件大小比较，可见MP4和h264文件大小基本上保持一致，二者比起未经压缩的yuv文件缩小24倍左右。
| mp4 | h264 | yuv |
|---|---|---|
| 459k | 457k | 11M |
- 编码相较于解码耗时更长，在微星笔记本（12代i9）上编码fps约为15，用显卡（RTX3080）加速后效率提升非常明显，h264_nvenc编码的fps约为120.
7. h264封装为MP4文件
MP4就是一个盒子，里边可以封装视频、音频、字幕，这里我们只封装视频，也就是最后产出的是只有视频没有音频的MP4文件。音频的添加到音频编解码时再进行。
7.1 代码
#include <stdio.h>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavcodec/avcodec.h"
#include "libswscale/swscale.h"
#include "libavformat/avformat.h"
#include "libavutil/avutil.h"
#include "libavutil/mathematics.h"
}
// 将H264转封装为MP4
int main() {
int frame_index = 0;//统计帧数
int inVStreamIndex = -1, outVStreamIndex = -1;//输入输出视频流在文件中的索引位置
const char* inVFileName = "/mnt/d/media/encoderesult.h264";
const char* outFileName = "/mnt/d/media/akiyo_qcif.mp4";
AVFormatContext* inVFmtCtx = NULL, * outFmtCtx = NULL;
AVCodecParameters* codecPara = NULL;
AVStream* outVStream = NULL;
const AVCodec* outCodec = NULL;
AVCodecContext* outCodecCtx = NULL;
AVCodecParameters* outCodecPara = NULL;
AVStream* inVStream = NULL;
AVPacket* pkt = av_packet_alloc();
do {
//======================输入部分============================//
//打开输入文件
if (avformat_open_input(&inVFmtCtx, inVFileName, NULL, NULL) < 0) {
printf("Cannot open input file.\n");
break;
}
//查找输入文件中的流
if (avformat_find_stream_info(inVFmtCtx, NULL) < 0) {
printf("Cannot find stream info in input file.\n");
break;
}
//查找视频流在文件中的位置
for (size_t i = 0; i < inVFmtCtx->nb_streams; i++) {
if (inVFmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
inVStreamIndex = (int)i;
break;
}
}
codecPara = inVFmtCtx->streams[inVStreamIndex]->codecpar;//输入视频流的编码参数
printf("===============Input information========>\n");
av_dump_format(inVFmtCtx, 0, inVFileName, 0);
printf("===============Input information========<\n");
//=====================输出部分=========================//
//打开输出文件并填充格式数据
if (avformat_alloc_output_context2(&outFmtCtx, NULL, NULL, outFileName) < 0) {
printf("Cannot alloc output file context.\n");
break;
}
//打开输出文件并填充数据
if (avio_open(&outFmtCtx->pb, outFileName, AVIO_FLAG_READ_WRITE) < 0) {
printf("output file open failed.\n");
break;
}
//在输出的mp4文件中创建一条视频流
outVStream = avformat_new_stream(outFmtCtx, NULL);
if (!outVStream) {
printf("Failed allocating output stream.\n");
break;
}
outVStream->time_base.den = 25;
outVStream->time_base.num = 1;
outVStreamIndex = outVStream->index;
//查找编码器
outCodec = avcodec_find_encoder(codecPara->codec_id);
if (outCodec == NULL) {
printf("Cannot find any encoder.\n");
break;
}
//从输入的h264编码器数据复制一份到输出文件的编码器中
outCodecCtx = avcodec_alloc_context3(outCodec);
outCodecPara = outFmtCtx->streams[outVStream->index]->codecpar;
if (avcodec_parameters_copy(outCodecPara, codecPara) < 0) {
printf("Cannot copy codec para.\n");
break;
}
if (avcodec_parameters_to_context(outCodecCtx, outCodecPara) < 0) {
printf("Cannot alloc codec ctx from para.\n");
break;
}
outCodecCtx->time_base.den = 25;
outCodecCtx->time_base.num = 1;
//打开输出文件需要的编码器
if (avcodec_open2(outCodecCtx, outCodec, NULL) < 0) {
printf("Cannot open output codec.\n");
break;
}
printf("============Output Information=============>\n");
av_dump_format(outFmtCtx, 0, outFileName, 1);
printf("============Output Information=============<\n");
//写入文件头
if (avformat_write_header(outFmtCtx, NULL) < 0) {
printf("Cannot write header to file.\n");
return -1;
}
//===============编码部分===============//
inVStream = inVFmtCtx->streams[inVStreamIndex];
while (av_read_frame(inVFmtCtx, pkt) >= 0) {
//循环读取每一帧直到读完
if (pkt->stream_index == inVStreamIndex) {
//确保处理的是视频流
//FIXME：No PTS (Example: Raw H.264)
//Simple Write PTS
//如果当前处理帧的显示时间戳为0或者没有等等不是正常值
if (pkt->pts == AV_NOPTS_VALUE) {
printf("frame_index:%d\n", frame_index);
//Write PTS
AVRational time_base1 = inVStream->time_base;
//Duration between 2 frames (us)
int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(inVStream->r_frame_rate);
//Parameters
pkt->pts = (double)(frame_index * calc_duration) / (double)(av_q2d(time_base1) * AV_TIME_BASE);
pkt->dts = pkt->pts;
pkt->duration = (double)calc_duration / (double)(av_q2d(time_base1) * AV_TIME_BASE);
frame_index++;
}
//Convert PTS/DTS
pkt->pts = av_rescale_q_rnd(pkt->pts, inVStream->time_base, outVStream->time_base, (enum AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt->dts = av_rescale_q_rnd(pkt->dts, inVStream->time_base, outVStream->time_base, (enum AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt->duration = av_rescale_q(pkt->duration, inVStream->time_base, outVStream->time_base);
pkt->pos = -1;
pkt->stream_index = outVStreamIndex;
printf("Write 1 Packet. size:%5d\tpts:%ld\n", pkt->size, pkt->pts);//每一帧的字节数
//Write
if (av_interleaved_write_frame(outFmtCtx, pkt) < 0) {
printf("Error muxing packet\n");
break;
}
//pkt->pts = pkt->dts;
av_packet_unref(pkt);
}
}
av_write_trailer(outFmtCtx);
} while (0);
//=================释放所有指针=======================
av_packet_free(&pkt);
avformat_close_input(&outFmtCtx);
avcodec_close(outCodecCtx);
avcodec_free_context(&outCodecCtx);
avformat_close_input(&inVFmtCtx);
avformat_free_context(inVFmtCtx);
//avio_close(outFmtCtx->pb);
return 0;
}
7.2 说明
- 整个过程中未作实际的编解码，只是进行了时间戳的转换。
- 关于ffmpeg中的时间戳可以参考《ffmpeg中timebase理解》，其实就是把一秒划分为
n个单元，以1/n秒作为一个单位。
8. 摄像头数据编码为h264
8.1 代码
#include <unistd.h>
#include <libavcodec/avcodec.h>
#include <libavdevice/avdevice.h>
#include <libavutil/channel_layout.h>
#include <libavutil/common.h>
#include <libavutil/frame.h>
#include <libavutil/samplefmt.h>
#include <libavutil/opt.h>
#include <libavutil/imgutils.h>
#include <libavutil/parseutils.h>
#include <libavutil/mem.h>
#include <libswscale/swscale.h>
#include <libavformat/avformat.h>
int flush_encoder(AVFormatContext *fmtCtx, AVCodecContext *codecCtx, int vStreamIndex){
int ret=0;
AVPacket *enc_pkt=av_packet_alloc();
enc_pkt->data = NULL;
enc_pkt->size = 0;
if (!(codecCtx->codec->capabilities & AV_CODEC_CAP_DELAY))
return 0;
printf("Flushing stream #%u encoder\n",vStreamIndex);
if(avcodec_send_frame(codecCtx,0)>=0){
while(avcodec_receive_packet(codecCtx,enc_pkt)>=0){
printf("success encoder 1 frame.\n");
// parpare packet for muxing
enc_pkt->stream_index = vStreamIndex;
av_packet_rescale_ts(enc_pkt,codecCtx->time_base,
fmtCtx->streams[ vStreamIndex ]->time_base);
ret = av_interleaved_write_frame(fmtCtx, enc_pkt);
if(ret<0){
break;
}
}
}
av_packet_unref(enc_pkt);
return ret;
}
int main()
{
int ret = 0;
avdevice_register_all();
AVFormatContext *inFmtCtx = avformat_alloc_context();
AVCodecContext *inCodecCtx = NULL;
const AVCodec *inCodec =NULL;
AVPacket *inPkt =av_packet_alloc();
AVFrame *srcFrame =av_frame_alloc();
AVFrame *yuvFrame =av_frame_alloc();
//打开输出文件，并填充fmtCtx数据
AVFormatContext *outFmtCtx = avformat_alloc_context();
const AVOutputFormat *outFmt = NULL;
AVCodecContext *outCodecCtx=NULL;
const AVCodec *outCodec = NULL;
AVStream *outVStream = NULL;
AVPacket *outPkt = av_packet_alloc();
struct SwsContext *img_ctx = NULL;
int inVideoStreamIndex = -1;
do{
/解码器部分//
//打开摄像头
const AVInputFormat *inFmt = av_find_input_format("v4l2");
if(avformat_open_input(&inFmtCtx,"/dev/video0",inFmt,NULL)<0){
printf("Cannot open camera.\n");
return -1;
}
if(avformat_find_stream_info(inFmtCtx,NULL)<0){
printf("Cannot find any stream in file.\n");
return -1;
}
for(size_t i=0;i<inFmtCtx->nb_streams;i++){
if(inFmtCtx->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_VIDEO){
inVideoStreamIndex=i;
break;
}
}
if(inVideoStreamIndex==-1){
printf("Cannot find video stream in file.\n");
return -1;
}
AVCodecParameters *inVideoCodecPara = inFmtCtx->streams[inVideoStreamIndex]->codecpar;
if(!(inCodec=avcodec_find_decoder(inVideoCodecPara->codec_id))){
printf("Cannot find valid video decoder.\n");
return -1;
}
if(!(inCodecCtx = avcodec_alloc_context3(inCodec))){
printf("Cannot alloc valid decode codec context.\n");
return -1;
}
if(avcodec_parameters_to_context(inCodecCtx,inVideoCodecPara)<0){
printf("Cannot initialize parameters.\n");
return -1;
}
if(avcodec_open2(inCodecCtx,inCodec,NULL)<0){
printf("Cannot open codec.\n");
return -1;
}
img_ctx = sws_getContext(inCodecCtx->width,
inCodecCtx->height,
inCodecCtx->pix_fmt,
inCodecCtx->width,
inCodecCtx->height,
AV_PIX_FMT_YUV420P,
SWS_BICUBIC,
NULL,NULL,NULL);
int numBytes = av_image_get_buffer_size(AV_PIX_FMT_YUV420P,
inCodecCtx->width,
inCodecCtx->height,1);
uint8_t* out_buffer = (unsigned char*)av_malloc(numBytes*sizeof(unsigned char));
ret = av_image_fill_arrays(yuvFrame->data,
yuvFrame->linesize,
out_buffer,
AV_PIX_FMT_YUV420P,
inCodecCtx->width,
inCodecCtx->height,
1);
if(ret<0){
printf("Fill arrays failed.\n");
return -1;
}
//解码器部分结束/
//编码器部分开始/
const char* outFile = "result.h264";
if(avformat_alloc_output_context2(&outFmtCtx,NULL,NULL,outFile)<0){
printf("Cannot alloc output file context.\n");
return -1;
}
outFmt = outFmtCtx->oformat;
//打开输出文件
if(avio_open(&outFmtCtx->pb,outFile,AVIO_FLAG_READ_WRITE)<0){
printf("output file open failed.\n");
return -1;
}
//创建h264视频流，并设置参数
outVStream = avformat_new_stream(outFmtCtx,outCodec);
if(outVStream==NULL){
printf("create new video stream fialed.\n");
return -1;
}
outVStream->time_base.den=30;
outVStream->time_base.num=1;
//编码参数相关
AVCodecParameters *outCodecPara = outFmtCtx->streams[outVStream->index]->codecpar;
outCodecPara->codec_type=AVMEDIA_TYPE_VIDEO;
outCodecPara->codec_id = outFmt->video_codec;
outCodecPara->width = 480;
outCodecPara->height = 360;
outCodecPara->bit_rate = 110000;
//查找编码器
outCodec = avcodec_find_encoder(outFmt->video_codec);
if(outCodec==NULL){
printf("Cannot find any encoder.\n");
return -1;
}
//设置编码器内容
outCodecCtx = avcodec_alloc_context3(outCodec);
avcodec_parameters_to_context(outCodecCtx,outCodecPara);
if(outCodecCtx==NULL){
printf("Cannot alloc output codec content.\n");
return -1;
}
outCodecCtx->codec_id = outFmt->video_codec;
outCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
outCodecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
outCodecCtx->width = inCodecCtx->width;
outCodecCtx->height = inCodecCtx->height;
outCodecCtx->time_base.num=1;
outCodecCtx->time_base.den=30;
outCodecCtx->bit_rate=110000;
outCodecCtx->gop_size=10;
if(outCodecCtx->codec_id==AV_CODEC_ID_H264){
outCodecCtx->qmin=10;
outCodecCtx->qmax=51;
outCodecCtx->qcompress=(float)0.6;
}else if(outCodecCtx->codec_id==AV_CODEC_ID_MPEG2VIDEO){
outCodecCtx->max_b_frames=2;
}else if(outCodecCtx->codec_id==AV_CODEC_ID_MPEG1VIDEO){
outCodecCtx->mb_decision=2;
}
//打开编码器
if(avcodec_open2(outCodecCtx,outCodec,NULL)<0){
printf("Open encoder failed.\n");
return -1;
}
///编码器部分结束
///编解码部分//
yuvFrame->format = outCodecCtx->pix_fmt;
yuvFrame->width = outCodecCtx->width;
yuvFrame->height = outCodecCtx->height;
ret = avformat_write_header(outFmtCtx,NULL);
int count = 0;
while(av_read_frame(inFmtCtx,inPkt)>=0 && count<50){
if(inPkt->stream_index == inVideoStreamIndex){
if(avcodec_send_packet(inCodecCtx,inPkt)>=0){
while((ret=avcodec_receive_frame(inCodecCtx,srcFrame))>=0){
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return -1;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
exit(1);
}
sws_scale(img_ctx,
(const uint8_t* const*)srcFrame->data,
srcFrame->linesize,
0,inCodecCtx->height,
yuvFrame->data,yuvFrame->linesize);
yuvFrame->pts=srcFrame->pts;
//encode
if(avcodec_send_frame(outCodecCtx,yuvFrame)>=0){
if(avcodec_receive_packet(outCodecCtx,outPkt)>=0){
printf("encode one frame.\n");
++count;
outPkt->stream_index = outVStream->index;
av_packet_rescale_ts(outPkt,outCodecCtx->time_base,
outVStream->time_base);
outPkt->pos=-1;
av_interleaved_write_frame(outFmtCtx,outPkt);
av_packet_unref(outPkt);
}
}
usleep(1000*24);
}
}
av_packet_unref(inPkt);
}
}
ret = flush_encoder(outFmtCtx,outCodecCtx,outVStream->index);
if(ret<0){
printf("flushing encoder failed.\n");
return -1;
}
av_write_trailer(outFmtCtx);
编解码部分结束
}while(0);
///内存释放部分/
av_packet_free(&inPkt);
avcodec_free_context(&inCodecCtx);
avcodec_close(inCodecCtx);
avformat_close_input(&inFmtCtx);
av_frame_free(&srcFrame);
av_frame_free(&yuvFrame);
av_packet_free(&outPkt);
avcodec_free_context(&outCodecCtx);
avcodec_close(outCodecCtx);
avformat_close_input(&outFmtCtx);
return 0;
}
8.2 说明
V4L2是Video for linux2的简称,为linux中关于视频设备的内核驱动- 步骤为读取摄像头YUYV422数据->解码为YUV420P->编码为H264流->保存文件