FFmpeg provides codec capabilities in the libavcodec module. The use process is: find codec, assign codec context, open codec, encode into AVPacket/decode into AVFrame, close codec. This article focuses on avcodec_open() to open the codec, and analyzes the overall process of codec.
1. Find the codec
The function to find the decoder is located in libavcodec/allcodecs.c. You can use avcodec_find_encoder() to find the encoder according to AVCodecID, or you can use avcodec_find_encoder_by_name() to search according to the name. Likewise, avcodec_find_decoder() and avcodec_find_decoder_by_name() can be used to find decoders. For the specific search process, please refer to: FFmpeg source code analysis - looking for codecs .
2. Assign codec context
After finding the codec, use AVCodec to assign the codec context, the code is located in libavcodec/options.c. First use av_malloc() to allocate memory, and then call init_context_defaults() to initialize the default parameters. The code is as follows:
AVCodecContext *avcodec_alloc_context3(const AVCodec *codec)
{
AVCodecContext *avctx= av_malloc(sizeof(AVCodecContext));
if (!avctx)
return NULL;
if (init_context_defaults(avctx, codec) < 0) {
av_free(avctx);
return NULL;
}
return avctx;
}3. Open the codec
1、avcodec_open
After assigning the codec context, use AVCodecContext to open the codec. Its function declaration is located in libavcodec/avcodec.h:
/**
* Initialize the AVCodecContext to use the given AVCodec. Prior to using this
* function the context has to be allocated with avcodec_alloc_context3().
*
* The functions avcodec_find_decoder_by_name(), avcodec_find_encoder_by_name(),
* avcodec_find_decoder() and avcodec_find_encoder() provide an easy way for
* retrieving a codec.
*
* @warning This function is not thread safe!
*
* @note Always call this function before using decoding routines (such as
* @ref avcodec_receive_frame()).
*
* @code
* av_dict_set(&opts, "b", "2.5M", 0);
* codec = avcodec_find_decoder(AV_CODEC_ID_H264);
* if (!codec)
* exit(1);
*
* context = avcodec_alloc_context3(codec);
*
* if (avcodec_open2(context, codec, opts) < 0)
* exit(1);
* @endcode
*
* @param avctx The context to initialize.
* @param codec The codec to open this context for.
* @param options A dictionary filled with AVCodecContext and codec-private options.
*
* @return zero on success, a negative value on error
*/
int avcodec_open2(AVCodecContext *avctx, const AVCodec *codec, AVDictionary **options);As can be seen from the description, this function is not thread-safe, and this function is called before the codec operation is performed. The function implementation is located in libavcodec/avcodec.c, the code is as follows:
int avcodec_open2(AVCodecContext *avctx, const AVCodec *codec, AVDictionary **options)
{
int ret = 0;
int codec_init_ok = 0;
AVDictionary *tmp = NULL;
AVCodecInternal *avci;
// 检查codec是否已经打开
if (avcodec_is_open(avctx))
return 0;
// 检查codec是否有效
if (!codec && !avctx->codec) {
return AVERROR(EINVAL);
}
if (codec && avctx->codec && codec != avctx->codec) {
return AVERROR(EINVAL);
}
if (!codec)
codec = avctx->codec;
if (avctx->extradata_size < 0 || avctx->extradata_size >= FF_MAX_EXTRADATA_SIZE)
return AVERROR(EINVAL);
// 拷贝options参数集合
if (options)
av_dict_copy(&tmp, *options, 0);
lock_avcodec(codec);
// 分配AVCodecInternal内存
avci = av_mallocz(sizeof(*avci));
if (!avci) {
ret = AVERROR(ENOMEM);
goto end;
}
// 分配frame、packet、fifo内存
avctx->internal = avci;
avci->buffer_frame = av_frame_alloc();
avci->buffer_pkt = av_packet_alloc();
avci->es.in_frame = av_frame_alloc();
avci->ds.in_pkt = av_packet_alloc();
avci->last_pkt_props = av_packet_alloc();
avci->pkt_props = av_fifo_alloc(sizeof(*avci->last_pkt_props));
if (!avci->buffer_frame || !avci->buffer_pkt ||
!avci->es.in_frame || !avci->ds.in_pkt ||
!avci->last_pkt_props || !avci->pkt_props) {
ret = AVERROR(ENOMEM);
goto free_and_end;
}
avci->skip_samples_multiplier = 1;
if (codec->priv_data_size > 0) {
if (!avctx->priv_data) {
avctx->priv_data = av_mallocz(codec->priv_data_size);
if (!avctx->priv_data) {
ret = AVERROR(ENOMEM);
goto free_and_end;
}
if (codec->priv_class) {
*(const AVClass **)avctx->priv_data = codec->priv_class;
av_opt_set_defaults(avctx->priv_data);
}
}
if (codec->priv_class && (ret = av_opt_set_dict(avctx->priv_data, &tmp)) < 0)
goto free_and_end;
} else {
avctx->priv_data = NULL;
}
if ((ret = av_opt_set_dict(avctx, &tmp)) < 0)
goto free_and_end;
if (avctx->codec_whitelist && av_match_list(codec->name, avctx->codec_whitelist, ',') <= 0) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
if (!(avctx->coded_width && avctx->coded_height && avctx->width && avctx->height &&
(avctx->codec_id == AV_CODEC_ID_H264 || avctx->codec_id == AV_CODEC_ID_VP6F
|| avctx->codec_id == AV_CODEC_ID_DXV))) {
if (avctx->coded_width && avctx->coded_height)
ret = ff_set_dimensions(avctx, avctx->coded_width, avctx->coded_height);
else if (avctx->width && avctx->height)
ret = ff_set_dimensions(avctx, avctx->width, avctx->height);
if (ret < 0)
goto free_and_end;
}
// 检查查视频宽高
if ((avctx->coded_width || avctx->coded_height || avctx->width || avctx->height)
&& (av_image_check_size2(avctx->coded_width, avctx->coded_height,
avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0
|| av_image_check_size2(avctx->width, avctx->height,
avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0)) {
ff_set_dimensions(avctx, 0, 0);
}
// 检查视频宽高比
if (avctx->width > 0 && avctx->height > 0) {
if (av_image_check_sar(avctx->width, avctx->height,
avctx->sample_aspect_ratio) < 0) {
avctx->sample_aspect_ratio = (AVRational){ 0, 1 };
}
}
// 检查音频声道
if (avctx->channels > FF_SANE_NB_CHANNELS || avctx->channels < 0) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
// 如果是音频解码器，检查声道数
if (av_codec_is_decoder(codec) &&
codec->type == AVMEDIA_TYPE_AUDIO &&
!(codec->capabilities & AV_CODEC_CAP_CHANNEL_CONF) &&
avctx->channels == 0) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
// 检查音频采样率
if (avctx->sample_rate < 0) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
// 检查块对齐
if (avctx->block_align < 0) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
avctx->codec = codec;
if ((avctx->codec_type == AVMEDIA_TYPE_UNKNOWN || avctx->codec_type == codec->type) &&
avctx->codec_id == AV_CODEC_ID_NONE) {
avctx->codec_type = codec->type;
avctx->codec_id = codec->id;
}
// 检查codec_id和codec_type
if (avctx->codec_id != codec->id || (avctx->codec_type != codec->type
&& avctx->codec_type != AVMEDIA_TYPE_ATTACHMENT)) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
avctx->frame_number = 0;
avctx->codec_descriptor = avcodec_descriptor_get(avctx->codec_id);
// 检查codec能力集，codec是否属于实验性的
if ((avctx->codec->capabilities & AV_CODEC_CAP_EXPERIMENTAL) &&
avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
const char *codec_string = av_codec_is_encoder(codec) ? "encoder" : "decoder";
const AVCodec *codec2;
codec2 = av_codec_is_encoder(codec) ?
avcodec_find_encoder(codec->id) : avcodec_find_decoder(codec->id);
if (!(codec2->capabilities & AV_CODEC_CAP_EXPERIMENTAL))
ret = AVERROR_EXPERIMENTAL;
goto free_and_end;
}
// 检查音频时间基
if (avctx->codec_type == AVMEDIA_TYPE_AUDIO &&
(!avctx->time_base.num || !avctx->time_base.den)) {
avctx->time_base.num = 1;
avctx->time_base.den = avctx->sample_rate;
}
if (av_codec_is_encoder(avctx->codec))
ret = ff_encode_preinit(avctx);
else
ret = ff_decode_preinit(avctx);
if (ret < 0)
goto free_and_end;
// 初始化编码器线程
if (CONFIG_FRAME_THREAD_ENCODER && av_codec_is_encoder(avctx->codec)) {
unlock_avcodec(codec);
ret = ff_frame_thread_encoder_init(avctx, options ? *options : NULL);
lock_avcodec(codec);
if (ret < 0)
goto free_and_end;
}
if (HAVE_THREADS
&& !(avci->frame_thread_encoder && (avctx->active_thread_type&FF_THREAD_FRAME))) {
ret = ff_thread_init(avctx);
if (ret < 0) {
goto free_and_end;
}
}
if (!HAVE_THREADS && !(codec->caps_internal & FF_CODEC_CAP_AUTO_THREADS))
avctx->thread_count = 1;
if ( avctx->codec->init && (!(avctx->active_thread_type&FF_THREAD_FRAME)
|| avci->frame_thread_encoder)) {
ret = avctx->codec->init(avctx);
if (ret < 0) {
codec_init_ok = -1;
goto free_and_end;
}
codec_init_ok = 1;
}
ret=0;
// 如果是解码器
if (av_codec_is_decoder(avctx->codec)) {
// 检查码率
if (!avctx->bit_rate)
avctx->bit_rate = get_bit_rate(avctx);
// 检查音频声道布局
if (avctx->channel_layout) {
int channels = av_get_channel_layout_nb_channels(avctx->channel_layout);
if (!avctx->channels)
avctx->channels = channels;
else if (channels != avctx->channels) {
char buf[512];
av_get_channel_layout_string(buf, sizeof(buf), -1, avctx->channel_layout);
avctx->channel_layout = 0;
}
}
if (avctx->channels && avctx->channels < 0 ||
avctx->channels > FF_SANE_NB_CHANNELS) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
// 检查每个sample的位数
if (avctx->bits_per_coded_sample < 0) {
ret = AVERROR(EINVAL);
goto free_and_end;
}
if (avctx->sub_charenc) {
if (avctx->codec_type != AVMEDIA_TYPE_SUBTITLE) {
ret = AVERROR(EINVAL);
goto free_and_end;
} else if (avctx->codec_descriptor->props & AV_CODEC_PROP_BITMAP_SUB) {
avctx->sub_charenc_mode = FF_SUB_CHARENC_MODE_DO_NOTHING;
} else {
if (avctx->sub_charenc_mode == FF_SUB_CHARENC_MODE_AUTOMATIC)
avctx->sub_charenc_mode = FF_SUB_CHARENC_MODE_PRE_DECODER;
if (avctx->sub_charenc_mode == FF_SUB_CHARENC_MODE_PRE_DECODER) {
ret = AVERROR(ENOSYS);
goto free_and_end;
}
}
}
}
if (codec->priv_data_size > 0 && avctx->priv_data && codec->priv_class) {
av_assert0(*(const AVClass **)avctx->priv_data == codec->priv_class);
}
end:
unlock_avcodec(codec);
if (options) {
av_dict_free(options);
*options = tmp;
}
return ret;
free_and_end:
if (avctx->codec && avctx->codec->close &&
(codec_init_ok > 0 || (codec_init_ok < 0 &&
avctx->codec->caps_internal & FF_CODEC_CAP_INIT_CLEANUP)))
avctx->codec->close(avctx);
if (HAVE_THREADS && avci->thread_ctx)
ff_thread_free(avctx);
if (codec->priv_class && avctx->priv_data)
av_opt_free(avctx->priv_data);
av_opt_free(avctx);
if (av_codec_is_encoder(avctx->codec)) {
av_freep(&avctx->extradata);
avctx->extradata_size = 0;
}
av_dict_free(&tmp);
av_freep(&avctx->priv_data);
av_freep(&avctx->subtitle_header);
av_frame_free(&avci->buffer_frame);
av_packet_free(&avci->buffer_pkt);
av_packet_free(&avci->last_pkt_props);
av_fifo_freep(&avci->pkt_props);
av_packet_free(&avci->ds.in_pkt);
av_frame_free(&avci->es.in_frame);
av_bsf_free(&avci->bsf);
av_buffer_unref(&avci->pool);
av_freep(&avci);
avctx->internal = NULL;
avctx->codec = NULL;
goto end;
}
As can be seen from the source code, avcodec_open() mainly allocates various memory, and then checks various parameters, including: video width and height, audio sampling rate, number of channels, channel layout, time base, codec capability set, etc.
2、avcodec_parameters_from_context
Since AVCodecContext has been marked as obsolete, it is recommended to use AVCodecParameters instead. Provide avcodec_parameters_from_context() in avcodec_par.c to copy AVCodecContext parameters to AVCodecParameters. Conversely, if you need to copy the AVCodecParameters parameters to the AVCodecContext, call avcodec_parameters_to_context().
Fourth, audio and video codec
In the FFmpeg 3.4 version, the old codec API has been marked as obsolete, and the new version is changed to asynchronous mode. We introduce the old and new versions of the API:
1. Coding
The old video encoding API is avcodec_encode_video2(), and the audio encoding API is avcodec_encode_audio2(). The new version API is unified as avcodec_send_frame/avcodec_receive_packet.
2. Decoding
The old video encoding API is avcodec_decode_video2(), and the audio encoding API is avcodec_decode_audio4(). The new version API is unified as avcodec_send_packet/avcodec_receive_frame.
5. Turn off the codec
1、avcodec_close
The avcodec_close() function is used to close the codec. The function declaration is located in libavcodec/avcodec.h and is declared as follows:
/**
* Close a given AVCodecContext and free all the data associated with it
* (but not the AVCodecContext itself).
*
* @note Do not use this function. Use avcodec_free_context() to destroy a
* codec context (either open or closed).
*/
int avcodec_close(AVCodecContext *avctx);As can be seen from the description, this function is used to close the given codec context, and the associated data, but not AVCodecContext itself. code show as below:
int avcodec_close(AVCodecContext *avctx)
{
int i;
if (!avctx)
return 0;
if (avcodec_is_open(avctx)) {
// 退出线程
if (CONFIG_FRAME_THREAD_ENCODER &&
avctx->internal->frame_thread_encoder && avctx->thread_count > 1) {
ff_frame_thread_encoder_free(avctx);
}
if (HAVE_THREADS && avctx->internal->thread_ctx)
ff_thread_free(avctx);
if (avctx->codec && avctx->codec->close)
avctx->codec->close(avctx);
avctx->internal->byte_buffer_size = 0;
av_freep(&avctx->internal->byte_buffer);
av_frame_free(&avctx->internal->buffer_frame);
av_packet_free(&avctx->internal->buffer_pkt);
av_packet_unref(avctx->internal->last_pkt_props);
// 清空fifo队列
while (av_fifo_size(avctx->internal->pkt_props) >=
sizeof(*avctx->internal->last_pkt_props)) {
av_fifo_generic_read(avctx->internal->pkt_props,
avctx->internal->last_pkt_props,
sizeof(*avctx->internal->last_pkt_props),
NULL);
av_packet_unref(avctx->internal->last_pkt_props);
}
av_packet_free(&avctx->internal->last_pkt_props);
av_fifo_freep(&avctx->internal->pkt_props);
av_packet_free(&avctx->internal->ds.in_pkt);
av_frame_free(&avctx->internal->es.in_frame);
av_buffer_unref(&avctx->internal->pool);
// 硬件加速反初始化
if (avctx->hwaccel && avctx->hwaccel->uninit)
avctx->hwaccel->uninit(avctx);
av_freep(&avctx->internal->hwaccel_priv_data);
// 释放bitstream filter资源
av_bsf_free(&avctx->internal->bsf);
av_freep(&avctx->internal);
}
for (i = 0; i < avctx->nb_coded_side_data; i++)
av_freep(&avctx->coded_side_data[i].data);
av_freep(&avctx->coded_side_data);
avctx->nb_coded_side_data = 0;
av_buffer_unref(&avctx->hw_frames_ctx);
av_buffer_unref(&avctx->hw_device_ctx);
if (avctx->priv_data && avctx->codec && avctx->codec->priv_class)
av_opt_free(avctx->priv_data);
av_opt_free(avctx);
av_freep(&avctx->priv_data);
if (av_codec_is_encoder(avctx->codec)) {
av_freep(&avctx->extradata);
}
avctx->codec = NULL;
avctx->active_thread_type = 0;
return 0;
}2、avcodec_free_context
The avcodec_free_context() function is used to release the AVCodecContext context, which corresponds to the allocation context avcodec_alloc_context3() above. Also located in options.c, first call the avcodec_close() function to close the codec, and finally release the AVCodecContext structure, the specific code is as follows:
void avcodec_free_context(AVCodecContext **pavctx)
{
AVCodecContext *avctx = *pavctx;
if (!avctx)
return;
// 调用avcodec_close关闭编解码器
avcodec_close(avctx);
av_freep(&avctx->extradata);
av_freep(&avctx->subtitle_header);
av_freep(&avctx->intra_matrix);
av_freep(&avctx->inter_matrix);
av_freep(&avctx->rc_override);
// 释放AVCodecContext结构体
av_freep(pavctx);
}