FFmpeg source code analysis: AVFilterGraph and AVFilter

FFmpeg provides simple and complex audio and video filters in the libavfilter module, all filters are connected by the AVFilterGraph filter graph. Simple filters are one-to-one output, and complex filters are many-to-one output. Important structures include: AVFilterGraph, AVFilterLink, AVFilterContext, AVFilter. Supports inserting AVFilter filters at the specified position in the filter chart, and then connects the filters by AVFilterLink.

1 、 AVFilterGraph

AVFilterGraph is a filter graph structure, which saves the filter context, filter number, thread type, thread number, callback function, etc. The code is located in the libavfilter/avfilter.h header file:

typedef struct AVFilterGraph {
    const AVClass *av_class;
	// 滤波器上下文
    AVFilterContext **filters;
	// 滤波器数量
    unsigned nb_filters;

    char *scale_sws_opts;
    int thread_type;
    int nb_threads;
    AVFilterGraphInternal *internal;
    void *opaque;
    // 用户设置的回调函数
    avfilter_execute_func *execute;

    char *aresample_swr_opts;
    AVFilterLink **sink_links;
    int sink_links_count;
    unsigned disable_auto_convert;
} AVFilterGraph;

2 、 AVFilterLink

AVFilterLink is the connector of the filter, responsible for connecting the two filters, and contains pointers to the source filter and the target filter. It should be noted that applications cannot directly access AVFilterLink, use buffersrc and buffersink API instead. The code is also in the avfilter.h header file:

struct AVFilterLink {
	// 源滤波器上下文
    AVFilterContext *src;
    AVFilterPad *srcpad;
    // 目标滤波器上下文
    AVFilterContext *dst;
    AVFilterPad *dstpad;
    // 媒体类型
    enum AVMediaType type;

    // 视频参数
    int w;
    int h;
    AVRational sample_aspect_ratio;
    // 音频参数
    uint64_t channel_layout;
    int sample_rate;
    int format;
    // pts的时间基
    AVRational time_base;

    AVFilterFormatsConfig incfg;
    AVFilterFormatsConfig outcfg;

    // 连接器的初始化状态
    enum {
        AVLINK_UNINIT = 0, // 没初始化
        AVLINK_STARTINIT,  // 已初始化，没完成
        AVLINK_INIT        // 已完成初始化
    } init_state;

    struct AVFilterGraph *graph;
    // 当前pts时间戳
    int64_t current_pts;
    int64_t current_pts_us;
    int age_index;
    AVRational frame_rate;
    AVFrame *partial_buf;
    int partial_buf_size;
    int min_samples;
    int max_samples;
    int channels;
    int64_t frame_count_in, frame_count_out;
    void *frame_pool;
    int frame_wanted_out;
    AVBufferRef *hw_frames_ctx;

#ifndef FF_INTERNAL_FIELDS

    // 内部数据结构(保留)
    char reserved[0xF000];

#else /* FF_INTERNAL_FIELDS */

    // 滤波器的帧队列
    FFFrameQueue fifo;
    frame_blocked_in;
    int status_in;
    int64_t status_in_pts;
    int status_out;

#endif /* FF_INTERNAL_FIELDS */
};

3 、 AVFilterContext

AVFilterContext is the structure of the filter context, including the filter instance, name, input Pad and Link, output Pad and Link, and the ready field to indicate that the filter is ready and has priority. The specific code is as follows:

struct AVFilterContext {
    const AVClass *av_class;
    // 滤波器实例
    const AVFilter *filter;
    // 滤波器名字
    char *name;

    AVFilterPad   *input_pads;
    AVFilterLink **inputs;
    unsigned    nb_inputs;
    AVFilterPad   *output_pads;
    AVFilterLink **outputs;
    unsigned    nb_outputs;

    void *priv;
    struct AVFilterGraph *graph;
    int thread_type;
    AVFilterInternal *internal;
    struct AVFilterCommand *command_queue;
    char *enable_str;
    void *enable;
    double *var_values;
    int is_disabled;
    AVBufferRef *hw_device_ctx;
    int nb_threads;

    // 滤波器准备就绪，带优先级
    unsigned ready;
    int extra_hw_frames;
};

4 、 AVFilter

AVFilter is the structure of the filter, including the filter name and description, the input and output AVFilterPad, and the function pointer of the filter operation. The specific code is as follows:

typedef struct AVFilter {
    // 滤波器名字
    const char *name;
    // 滤波器描述
    const char *description;
    const AVFilterPad *inputs;
    const AVFilterPad *outputs;
    const AVClass *priv_class;
    int flags;
    int priv_size;
    int flags_internal;

    // 预初始化函数指针
    int (*preinit)(AVFilterContext *ctx);
    // 初始化函数指针
    int (*init)(AVFilterContext *ctx);
    // 带字典参数的初始化函数指针
    int (*init_dict)(AVFilterContext *ctx, AVDictionary **options);
    // 反初始化函数指针
    void (*uninit)(AVFilterContext *ctx);
    // 查询滤波器支持的输入输出格式
    int (*query_formats)(AVFilterContext *);
    // 处理命令函数指针
    int (*process_command)(AVFilterContext *, const char *cmd, 
	    const char *arg, char *res, int res_len, int flags);
    // 带透传指针的初始化函数指针
    int (*init_opaque)(AVFilterContext *ctx, void *opaque);
    // 滤波器激活函数指针
    int (*activate)(AVFilterContext *ctx);
} AVFilter;

5 、 avfilter_graph_create_filter

Responsible for creating and adding filter instances and adding them to the AVFilterGraph graph. code show as below:

int avfilter_graph_create_filter(AVFilterContext **filt_ctx, const AVFilter *filt,
                                 const char *name, const char *args, void *opaque,
                                 AVFilterGraph *graph_ctx)
{
    int ret;
    // 分配滤波器上下文
    *filt_ctx = avfilter_graph_alloc_filter(graph_ctx, filt, name);
    if (!*filt_ctx)
        return AVERROR(ENOMEM);
    // 初始化滤波器
    ret = avfilter_init_str(*filt_ctx, args);
    if (ret < 0)
        goto fail;

    return 0;
fail:
    if (*filt_ctx)
        avfilter_free(*filt_ctx);
    *filt_ctx = NULL;
    return ret;
}

Call the avfilter_graph_alloc_filter() function to allocate the filter context, call ff_graph_thread_init() to create a filter thread, and call ff_filter_alloc() to actually allocate the filter context. code show as below:

AVFilterContext *avfilter_graph_alloc_filter(AVFilterGraph *graph,
                                             const AVFilter *filter,
                                             const char *name)
{
    AVFilterContext **filters, *s;
    // 创建滤波器线程
    if (graph->thread_type && !graph->internal->thread_execute) {
        if (graph->execute) {
            graph->internal->thread_execute = graph->execute;
        } else {
            int ret = ff_graph_thread_init(graph);
            if (ret < 0) {
                return NULL;
            }
        }
    }
    // 分配滤波器上下文
    s = ff_filter_alloc(filter, name);
    if (!s)
        return NULL;
    filters = av_realloc(graph->filters, sizeof(*filters) * (graph->nb_filters + 1));
    if (!filters) {
        avfilter_free(s);
        return NULL;
    }
    graph->filters = filters;
    graph->filters[graph->nb_filters++] = s;
    s->graph = graph;

    return s;
}

6 、 avfilter_init_str

 The avfilter_init_str() function is used to initialize the filter, including compatibility with the old command syntax and the new command syntax. The code is in avfilter.c:

int avfilter_init_str(AVFilterContext *filter, const char *args)
{
    AVDictionary *options = NULL;
    AVDictionaryEntry *e;
    int ret = 0;

    if (args && *args) {
        if (!filter->filter->priv_class) {
            return AVERROR(EINVAL);
        }
// 兼容旧版语法
#if FF_API_OLD_FILTER_OPTS_ERROR
            if (   !strcmp(filter->filter->name, "format")     ||
                   !strcmp(filter->filter->name, "noformat")   ||
                   !strcmp(filter->filter->name, "frei0r")     ||
                   !strcmp(filter->filter->name, "frei0r_src") ||
                   !strcmp(filter->filter->name, "ocv")        ||
                   !strcmp(filter->filter->name, "pan")        ||
                   !strcmp(filter->filter->name, "pp")         ||
                   !strcmp(filter->filter->name, "aevalsrc")) {

            ......
            if (deprecated) {
                ret = AVERROR(EINVAL);
            } else {
                ret = process_options(filter, &options, copy);
            }
            av_freep(&copy);
            if (ret < 0)
                goto fail;
        } else
#endif
        {
			// 处理参数选项
            ret = process_options(filter, &options, args);
            if (ret < 0)
                goto fail;
        }
    }
    // 初始化参数选项
    ret = avfilter_init_dict(filter, &options);
    if (ret < 0)
        goto fail;
    if ((e = av_dict_get(options, "", NULL, AV_DICT_IGNORE_SUFFIX))) {
        ret = AVERROR_OPTION_NOT_FOUND;
        goto fail;
    }

fail:
    av_dict_free(&options);
    return ret;
}

Among them, call the avfilter_init_dict() function to initialize the parameter options:

int avfilter_init_dict(AVFilterContext *ctx, AVDictionary **options)
{
    int ret = 0;

    ret = av_opt_set_dict(ctx, options);
    if (ret < 0) {
        return ret;
    }
    // 线程类型赋值
    if (ctx->filter->flags & AVFILTER_FLAG_SLICE_THREADS &&
        ctx->thread_type & ctx->graph->thread_type & AVFILTER_THREAD_SLICE &&
        ctx->graph->internal->thread_execute) {
        ctx->thread_type       = AVFILTER_THREAD_SLICE;
        ctx->internal->execute = ctx->graph->internal->thread_execute;
    } else {
        ctx->thread_type = 0;
    }
    if (ctx->filter->priv_class) {
        ret = av_opt_set_dict2(ctx->priv, options, AV_OPT_SEARCH_CHILDREN);
        if (ret < 0) {
            return ret;
        }
    }
    if (ctx->filter->init_opaque)
        ret = ctx->filter->init_opaque(ctx, NULL);
    else if (ctx->filter->init)
        ret = ctx->filter->init(ctx);
    else if (ctx->filter->init_dict)
        ret = ctx->filter->init_dict(ctx, options);

    if (ctx->enable_str) {
        ret = set_enable_expr(ctx, ctx->enable_str);
        if (ret < 0)
            return ret;
    }
    return ret;
}

 7 、 avfilter_insert_filter

The avfilter_insert_filter() function inserts a new filter between the two filters of the filter chain, and calls the avfilter_link() function to perform the real connection. code show as below:

int avfilter_insert_filter(AVFilterLink *link, AVFilterContext *filt,
                           unsigned filt_srcpad_idx, unsigned filt_dstpad_idx)
{
    int ret;
    unsigned dstpad_idx = link->dstpad - link->dst->input_pads;

    link->dst->inputs[dstpad_idx] = NULL;
	// 开始连接滤波器
    if ((ret = avfilter_link(filt, filt_dstpad_idx, link->dst, dstpad_idx)) < 0) {
        // 连接滤波器失败
        link->dst->inputs[dstpad_idx] = link;
        return ret;
    }
    link->dst                     = filt;
    link->dstpad                  = &filt->input_pads[filt_srcpad_idx];
    filt->inputs[filt_srcpad_idx] = link;
    // 保存相关的媒体格式信息
    if (link->outcfg.formats)
        ff_formats_changeref(&link->outcfg.formats,
                             &filt->outputs[filt_dstpad_idx]->outcfg.formats);
    if (link->outcfg.samplerates)
        ff_formats_changeref(&link->outcfg.samplerates,
                             &filt->outputs[filt_dstpad_idx]->outcfg.samplerates);
    if (link->outcfg.channel_layouts)
        ff_channel_layouts_changeref(&link->outcfg.channel_layouts,
                                     &filt->outputs[filt_dstpad_idx]->outcfg.channel_layouts);

    return 0;
}

The avfilter_link() function is called internally to connect the filter, mainly to perform assignment operations and initialize the frame queue:

int avfilter_link(AVFilterContext *src, unsigned srcpad,
                  AVFilterContext *dst, unsigned dstpad)
{
    AVFilterLink *link;
    av_assert0(src->graph);
    av_assert0(dst->graph);
    av_assert0(src->graph == dst->graph);

    if (src->nb_outputs <= srcpad || dst->nb_inputs <= dstpad ||
        src->outputs[srcpad]      || dst->inputs[dstpad])
        return AVERROR(EINVAL);
    if (src->output_pads[srcpad].type != dst->input_pads[dstpad].type) {
        return AVERROR(EINVAL);
    }
    link = av_mallocz(sizeof(*link));
    if (!link)
        return AVERROR(ENOMEM);

    src->outputs[srcpad] = dst->inputs[dstpad] = link;
	// 输入输出的AVFilterContext、pad赋值
    link->src     = src;
    link->dst     = dst;
    link->srcpad  = &src->output_pads[srcpad];
    link->dstpad  = &dst->input_pads[dstpad];
    link->type    = src->output_pads[srcpad].type;
    link->format  = -1;
	// 初始化帧队列
    ff_framequeue_init(&link->fifo, &src->graph->internal->frame_queues);

    return 0;
}

8、graph_config_formats

Configuring all formats connected to the filter graph consists of four steps: Find Supported Formats, Merge Formats, Confirm Formats, and Select Formats. details as follows:

static int graph_config_formats(AVFilterGraph *graph, AVClass *log_ctx)
{
    int ret;

    // 寻找支持的格式
    while ((ret = query_formats(graph, log_ctx)) == AVERROR(EAGAIN))
        av_log(graph, AV_LOG_DEBUG, "query_formats not finished\n");
    if (ret < 0)
        return ret;

    // 合并格式
    if ((ret = reduce_formats(graph)) < 0)
        return ret;

    // 音频：确认选中最佳的采样格式、采样率、声道布局
    swap_sample_fmts(graph);
    swap_samplerates(graph);
    swap_channel_layouts(graph);
    // 选择格式
    if ((ret = pick_formats(graph)) < 0)
        return ret;

    return 0;
}

Among them, the pick_formats() function will traverse all the filters of the chart and call the pick_format() function to select the format. For video, choose the best pixel format; for audio, choose the best sample format. details as follows:

static int pick_format(AVFilterLink *link, AVFilterLink *ref)
{
    if (!link || !link->incfg.formats)
        return 0;

    if (link->type == AVMEDIA_TYPE_VIDEO) {
		// 视频：选择最佳的像素格式
        if(ref && ref->type == AVMEDIA_TYPE_VIDEO){
            int has_alpha= av_pix_fmt_desc_get(ref->format)->nb_components % 2 == 0;
            enum AVPixelFormat best= AV_PIX_FMT_NONE;
            int i;
            for (i = 0; i < link->incfg.formats->nb_formats; i++) {
                enum AVPixelFormat p = link->incfg.formats->formats[i];
                best= av_find_best_pix_fmt_of_2(best, p, ref->format, has_alpha, NULL);
            }
            link->incfg.formats->formats[0] = best;
        }
    } else if (link->type == AVMEDIA_TYPE_AUDIO) {
		// 音频：选择最佳的采样格式
        if(ref && ref->type == AVMEDIA_TYPE_AUDIO){
            enum AVSampleFormat best= AV_SAMPLE_FMT_NONE;
            int i;
            for (i = 0; i < link->incfg.formats->nb_formats; i++) {
                enum AVSampleFormat p = link->incfg.formats->formats[i];
                best = find_best_sample_fmt_of_2(best, p, ref->format);
            }
            link->incfg.formats->formats[0] = best;
        }
    }
    ......

    return 0;
}

 When choosing the best sampling format, call the find_best_sample_fmt_of_2() function, compare the scores of the two formats, and finally call the get_fmt_score() function to calculate the score of the format:

static int get_fmt_score(enum AVSampleFormat dst_fmt, enum AVSampleFormat src_fmt)
{
    int score = 0;
    // 音频采样格式：无交错(平面存储)
    if (av_sample_fmt_is_planar(dst_fmt) != av_sample_fmt_is_planar(src_fmt))
        score ++;
    // 获取每个采样的字节数
    if (av_get_bytes_per_sample(dst_fmt) < av_get_bytes_per_sample(src_fmt)) {
        score += 100 * (av_get_bytes_per_sample(src_fmt) - av_get_bytes_per_sample(dst_fmt));
    }else
        score += 10  * (av_get_bytes_per_sample(dst_fmt) - av_get_bytes_per_sample(src_fmt));
    // 音频采样格式：交错存储，并且是32位或浮点数
    if (av_get_packed_sample_fmt(dst_fmt) == AV_SAMPLE_FMT_S32 &&
        av_get_packed_sample_fmt(src_fmt) == AV_SAMPLE_FMT_FLT)
        score += 20;
    if (av_get_packed_sample_fmt(dst_fmt) == AV_SAMPLE_FMT_FLT &&
        av_get_packed_sample_fmt(src_fmt) == AV_SAMPLE_FMT_S32)
        score += 2;

    return score;
}

9 、 avfilter_graph_free

The avfilter_graph_free() function is used to release the AVFilterGraph structure:

void avfilter_graph_free(AVFilterGraph **graph)
{
    if (!*graph)
        return;
    // 释放滤波器
    while ((*graph)->nb_filters)
        avfilter_free((*graph)->filters[0]);
    // 释放滤波线程
    ff_graph_thread_free(*graph);
    av_freep(&(*graph)->sink_links);
    av_freep(&(*graph)->scale_sws_opts);
    av_freep(&(*graph)->aresample_swr_opts);
#if FF_API_LAVR_OPTS
    av_freep(&(*graph)->resample_lavr_opts);
#endif
    av_freep(&(*graph)->filters);
    av_freep(&(*graph)->internal);
    // 最终释放AVFilterGraph
    av_freep(graph);
}