Función 1.av1_encode_strategy
La función av1_encode_strategy implementará una estrategia de codificación de alto nivel, seleccionará el tipo de marco, la posición del marco, etc. Llena la estructura EncodeFrameParams con los resultados de estas decisiones y luego llama a la función av1_encode (o función denoise_and_encode).
El proceso básico de esta función es:
- inicialización
- Obtenga el siguiente búfer de origen para codificar (en el primer proceso de codificación, no hay codificación real, todas las fuentes son NULL)
- De acuerdo con los datos del primer proceso de codificación, obtenga los parámetros de codificación del marco actual ( av1_get_second_pass_params ) y establezca el tipo de marco actual
- Obtener lista de marcos de referencia
- Llame a la función denoise_and_encode o av1_encode para codificar el marco actual
- Actualizar la información correspondiente
El código y los comentarios son los siguientes:
int av1_encode_strategy(AV1_COMP *const cpi, size_t *const size,
uint8_t *const dest, unsigned int *frame_flags,
int64_t *const time_stamp, int64_t *const time_end,
const aom_rational64_t *const timestamp_ratio,
int flush) {
const AV1EncoderConfig *const oxcf = &cpi->oxcf;
AV1_COMMON *const cm = &cpi->common;
GF_GROUP *gf_group = &cpi->gf_group;
ExternalFlags *const ext_flags = &cpi->ext_flags; //由帧级外部接口发出信号的标志。
EncodeFrameInput frame_input;
// EncodeFrameParams包含由av1_encode_strategy决定并传递给av1_encode的每帧编码参数
EncodeFrameParams frame_params;
// EncodeFrameResults包含有关对单个帧进行编码的结果的信息
EncodeFrameResults frame_results;
memset(&frame_input, 0, sizeof(frame_input));
memset(&frame_params, 0, sizeof(frame_params));
memset(&frame_results, 0, sizeof(frame_results));
// TODO(sarahparker) finish bit allocation for one pass pyramid
if (has_no_stats_stage(cpi) && oxcf->rc_mode != AOM_Q) {
cpi->oxcf.gf_max_pyr_height =
AOMMIN(cpi->oxcf.gf_max_pyr_height, USE_ALTREF_FOR_ONE_PASS);
cpi->oxcf.gf_min_pyr_height =
AOMMIN(cpi->oxcf.gf_min_pyr_height, cpi->oxcf.gf_max_pyr_height);
}
if (!is_stat_generation_stage(cpi)) {
// 如果不是统计信息的阶段
// If this is a forward keyframe, mark as a show_existing_frame
// 如果这是一个向前的关键帧，标记为show_existing_frame
if (cpi->oxcf.fwd_kf_enabled && (gf_group->index == gf_group->size) &&
gf_group->update_type[1] == ARF_UPDATE && cpi->rc.frames_to_key == 0) {
frame_params.show_existing_frame = 1;
} else {
frame_params.show_existing_frame =
((oxcf->enable_overlay == 0 || cpi->sf.hl_sf.disable_overlay_frames ||
cpi->show_existing_alt_ref) &&
gf_group->update_type[gf_group->index] == OVERLAY_UPDATE) ||
gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE;
}
frame_params.show_existing_frame &= allow_show_existing(cpi, *frame_flags);
// Reset show_existing_alt_ref decision to 0 after it is used.
if (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE) {
cpi->show_existing_alt_ref = 0;
}
} else {
frame_params.show_existing_frame = 0;
}
int code_arf = 0;
struct lookahead_entry *source = NULL;
struct lookahead_entry *last_source = NULL;
if (frame_params.show_existing_frame) {
source = av1_lookahead_pop(cpi->lookahead, flush, cpi->compressor_stage);
frame_params.show_frame = 1;
} else {
int show_existing_alt_ref = 0;
//检查我们是应该编码一个ARF还是internal ARF。如果没有，试试LAST
//执行与所选源关联的某些设置
//输出的是temporal_filtered, flush和frame_update_type类型。
//返回帧源，如果找不到，则返回NULL
source = choose_frame_source(cpi, &code_arf, &flush, &last_source,
&frame_params, &show_existing_alt_ref);
if (gf_group->update_type[gf_group->index] == ARF_UPDATE)
cpi->show_existing_alt_ref = show_existing_alt_ref;//告诉OVERLAY帧是否显示现有的alt_ref帧。
}
if (source == NULL) { // If no source was found, we can't encode a frame.
// 在第一次编码过程中，没有真正进行编码，所有source为NULL
#if !CONFIG_REALTIME_ONLY
if (flush && oxcf->pass == 1 && !cpi->twopass.first_pass_done) {
// 第一次编码过程结束时，会执行该函数，用来输出第一次编码过程的统计数据
av1_end_first_pass(cpi); /* get last stats packet */
cpi->twopass.first_pass_done = 1;
}
#endif
return -1;
}
frame_input.source = code_arf ? &cpi->alt_ref_buffer : &source->img;
frame_input.last_source = last_source != NULL ? &last_source->img : NULL;
frame_input.ts_duration = source->ts_end - source->ts_start;
// Save unfiltered source. It is used in av1_get_second_pass_params().
cpi->unfiltered_source = frame_input.source; //保存为滤波的原图像
*time_stamp = source->ts_start;
*time_end = source->ts_end;
if (source->ts_start < cpi->time_stamps.first_ever) {
cpi->time_stamps.first_ever = source->ts_start;
cpi->time_stamps.prev_end_seen = source->ts_start;
}
av1_apply_encoding_flags(cpi, source->flags);
if (!frame_params.show_existing_frame)
*frame_flags = (source->flags & AOM_EFLAG_FORCE_KF) ? FRAMEFLAGS_KEY : 0;
// Shown frames and arf-overlay frames need frame-rate considering
if (frame_params.show_frame)
adjust_frame_rate(cpi, source->ts_start, source->ts_end);
if (!frame_params.show_existing_frame) {
if (cpi->film_grain_table) {
cm->cur_frame->film_grain_params_present = aom_film_grain_table_lookup(
cpi->film_grain_table, *time_stamp, *time_end, 0 /* =erase */,
&cm->film_grain_params);
} else {
cm->cur_frame->film_grain_params_present =
cm->seq_params.film_grain_params_present;
}
// only one operating point supported now
const int64_t pts64 = ticks_to_timebase_units(timestamp_ratio, *time_stamp);
if (pts64 < 0 || pts64 > UINT32_MAX) return AOM_CODEC_ERROR;
cm->frame_presentation_time = (uint32_t)pts64;
}
#if CONFIG_REALTIME_ONLY
av1_get_one_pass_rt_params(cpi, &frame_params, *frame_flags);
#else
if (has_no_stats_stage(cpi) && oxcf->mode == REALTIME &&
oxcf->lag_in_frames == 0)
av1_get_one_pass_rt_params(cpi, &frame_params, *frame_flags);
else if (!is_stat_generation_stage(cpi))
//获取第二次编码过程的当前帧的编码参数（根据第一次编码过程的数据）
av1_get_second_pass_params(cpi, &frame_params, &frame_input, *frame_flags);
#endif
FRAME_UPDATE_TYPE frame_update_type = get_frame_update_type(gf_group);
// 设置当前帧的帧类型
if (frame_params.show_existing_frame &&
frame_params.frame_type != KEY_FRAME) {
// Force show-existing frames to be INTER, except forward keyframes
frame_params.frame_type = INTER_FRAME;
}
// TODO([email protected]): Move all the encode strategy
// (largely near av1_get_compressed_data) in here
// TODO([email protected]): Change all the encode strategy to
// modify frame_params instead of cm or cpi.
// Per-frame encode speed. In theory this can vary, but things may have been
// written assuming speed-level will not change within a sequence, so this
// parameter should be used with caution.
frame_params.speed = oxcf->speed;
// Work out some encoding parameters specific to the pass:
// 计算出一些特定于过程的编码参数：
if (has_no_stats_stage(cpi) && cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) {
av1_cyclic_refresh_update_parameters(cpi);
} else if (is_stat_generation_stage(cpi)) {
cpi->td.mb.e_mbd.lossless[0] = is_lossless_requested(&cpi->oxcf);
const int kf_requested = (cm->current_frame.frame_number == 0 ||
(*frame_flags & FRAMEFLAGS_KEY));
if (kf_requested && frame_update_type != OVERLAY_UPDATE &&
frame_update_type != INTNL_OVERLAY_UPDATE) {
frame_params.frame_type = KEY_FRAME;
} else {
frame_params.frame_type = INTER_FRAME;
}
} else if (is_stat_consumption_stage(cpi)) {
#if CONFIG_MISMATCH_DEBUG
mismatch_move_frame_idx_w();
#endif
#if TXCOEFF_COST_TIMER
cm->txcoeff_cost_timer = 0;
cm->txcoeff_cost_count = 0;
#endif
}
if (!is_stat_generation_stage(cpi))
set_ext_overrides(cm, &frame_params, ext_flags);
// Shown keyframes and S frames refresh all reference buffers
// 显示的关键帧和S帧刷新所有引用缓冲区
const int force_refresh_all =
((frame_params.frame_type == KEY_FRAME && frame_params.show_frame) ||
frame_params.frame_type == S_FRAME) &&
!frame_params.show_existing_frame;
av1_configure_buffer_updates(cpi, &frame_params, frame_update_type,
force_refresh_all);
// 获取参考帧列表
if (!is_stat_generation_stage(cpi)) {
const RefCntBuffer *ref_frames[INTER_REFS_PER_FRAME];//参考帧
const YV12_BUFFER_CONFIG *ref_frame_buf[INTER_REFS_PER_FRAME];
if (!ext_flags->refresh_frame_flags_pending) {
av1_get_ref_frames(cpi, &cpi->ref_buffer_stack);
} else if (cpi->svc.external_ref_frame_config) {
for (unsigned int i = 0; i < INTER_REFS_PER_FRAME; i++)
cm->remapped_ref_idx[i] = cpi->svc.ref_idx[i];
}
// Get the reference frames 获取参考帧
for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
ref_frames[i] = get_ref_frame_buf(cm, ref_frame_priority_order[i]);
ref_frame_buf[i] = ref_frames[i] != NULL ? &ref_frames[i]->buf : NULL;
}
// Work out which reference frame slots may be used. 找出可以使用的参考帧槽。
frame_params.ref_frame_flags = get_ref_frame_flags(
&cpi->sf, ref_frame_buf, ext_flags->ref_frame_flags);
frame_params.primary_ref_frame =
choose_primary_ref_frame(cpi, &frame_params);
frame_params.order_offset = get_order_offset(&cpi->gf_group, &frame_params);
frame_params.refresh_frame_flags = av1_get_refresh_frame_flags(
cpi, &frame_params, frame_update_type, &cpi->ref_buffer_stack);
frame_params.existing_fb_idx_to_show =
frame_params.show_existing_frame
? (frame_update_type == INTNL_OVERLAY_UPDATE
? get_ref_frame_map_idx(cm, BWDREF_FRAME)
: get_ref_frame_map_idx(cm, ALTREF_FRAME))
: INVALID_IDX;
}
// The way frame_params->remapped_ref_idx is setup is a placeholder.
// Currently, reference buffer assignment is done by update_ref_frame_map()
// which is called by high-level strategy AFTER encoding a frame. It modifies
// cm->remapped_ref_idx. If you want to use an alternative method to
// determine reference buffer assignment, just put your assignments into
// frame_params->remapped_ref_idx here and they will be used when encoding
// this frame. If frame_params->remapped_ref_idx is setup independently of
// cm->remapped_ref_idx then update_ref_frame_map() will have no effect.
memcpy(frame_params.remapped_ref_idx, cm->remapped_ref_idx,
REF_FRAMES * sizeof(*cm->remapped_ref_idx));
cpi->td.mb.e_mbd.delta_qindex = 0;
if (!frame_params.show_existing_frame) {
cm->quant_params.using_qmatrix = cpi->oxcf.using_qm;
#if !CONFIG_REALTIME_ONLY
if (oxcf->lag_in_frames > 0 && !is_stat_generation_stage(cpi)) {
if (cpi->gf_group.index == 1 && cpi->oxcf.enable_tpl_model) {
av1_configure_buffer_updates(cpi, &frame_params, frame_update_type, 0);
av1_set_frame_size(cpi, cm->width, cm->height);
av1_tpl_setup_stats(cpi, 0, &frame_params, &frame_input);
assert(cpi->num_gf_group_show_frames == 1);
}
}
#endif
}
#if CONFIG_REALTIME_ONLY
if (av1_encode(cpi, dest, &frame_input, &frame_params, &frame_results) !=
AOM_CODEC_OK) {
return AOM_CODEC_ERROR;
}
#else
// 对当前帧进行编码
// 对关键帧应用时间滤波并对过滤后的帧进行编码。
// 如果当前帧不是关键帧，则此函数与av1_encode相同。
if (denoise_and_encode(cpi, dest, &frame_input, &frame_params,
&frame_results) != AOM_CODEC_OK) {
return AOM_CODEC_ERROR;
}
#endif // CONFIG_REALTIME_ONLY
if (!is_stat_generation_stage(cpi))
cpi->num_gf_group_show_frames += frame_params.show_frame;
if (!is_stat_generation_stage(cpi)) {
// First pass doesn't modify reference buffer assignment or produce frame
// flags 第一个过程不修改引用缓冲区分配或生成帧标志
update_frame_flags(cpi, frame_flags);
if (!ext_flags->refresh_frame_flags_pending) {
int ref_map_index =
av1_get_refresh_ref_frame_map(cm->current_frame.refresh_frame_flags);
av1_update_ref_frame_map(cpi, frame_update_type, cm->show_existing_frame,
ref_map_index, &cpi->ref_buffer_stack);
}
}
#if !CONFIG_REALTIME_ONLY
if (!is_stat_generation_stage(cpi)) {
#if TXCOEFF_COST_TIMER
cm->cum_txcoeff_cost_timer += cm->txcoeff_cost_timer;
fprintf(stderr,
"\ntxb coeff cost block number: %ld, frame time: %ld, cum time %ld "
"in us\n",
cm->txcoeff_cost_count, cm->txcoeff_cost_timer,
cm->cum_txcoeff_cost_timer);
#endif
av1_twopass_postencode_update(cpi);
}
#endif // !CONFIG_REALTIME_ONLY
if (!is_stat_generation_stage(cpi)) {
update_fb_of_context_type(cpi, &frame_params, cpi->fb_of_context_type);
set_additional_frame_flags(cm, frame_flags);
update_rc_counts(cpi);
}
// Unpack frame_results:
*size = frame_results.size;
// Leave a signal for a higher level caller about if this frame is droppable
if (*size > 0) {
cpi->droppable = is_frame_droppable(&cpi->svc, ext_flags);
}
if (cpi->use_svc) av1_save_layer_context(cpi);
return AOM_CODEC_OK;
}Función 2.denoise_and_encode
Esta función codifica el fotograma actual, dividido en las dos situaciones siguientes:
- Si el fotograma actual es un fotograma clave, aplique un filtrado temporal y codifique el fotograma filtrado.
- Si el fotograma actual no es un fotograma clave, esta función es la misma que av1_encode.
static int denoise_and_encode(AV1_COMP *const cpi, uint8_t *const dest,
EncodeFrameInput *const frame_input,
EncodeFrameParams *const frame_params,
EncodeFrameResults *const frame_results) {
const AV1EncoderConfig *const oxcf = &cpi->oxcf;
AV1_COMMON *const cm = &cpi->common;
// Decide whether to apply temporal filtering to the source frame.
// 决定是否对源帧应用时域滤波。
int apply_filtering =
frame_params->frame_type == KEY_FRAME &&
oxcf->enable_keyframe_filtering && !is_stat_generation_stage(cpi) &&
!frame_params->show_existing_frame &&
cpi->rc.frames_to_key > TF_NUM_FILTERING_FRAMES_FOR_KEY_FRAME &&
!is_lossless_requested(oxcf) && oxcf->arnr_max_frames > 0;
if (apply_filtering) {
const double y_noise_level = av1_estimate_noise_from_single_plane(
frame_input->source, 0, cm->seq_params.bit_depth);
apply_filtering = y_noise_level > 0;
}
// Save the pointer to the original source image.
YV12_BUFFER_CONFIG *source_kf_buffer = frame_input->source;
// Apply filtering to key frame.对关键帧应用过滤。
if (apply_filtering) {
// Initialization for frame motion estimation.
MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
av1_init_mi_buffers(&cm->mi_params);
setup_mi(cpi, frame_input->source);
av1_init_macroblockd(cm, xd, NULL);
memset(
cpi->mbmi_ext_info.frame_base, 0,
cpi->mbmi_ext_info.alloc_size * sizeof(*cpi->mbmi_ext_info.frame_base));
av1_set_speed_features_framesize_independent(cpi, oxcf->speed);
av1_set_speed_features_framesize_dependent(cpi, oxcf->speed);
av1_set_rd_speed_thresholds(cpi);
av1_setup_frame_buf_refs(cm);
av1_setup_frame_sign_bias(cm);
av1_frame_init_quantizer(cpi);
av1_setup_past_independence(cm);
if (!frame_params->show_frame) {
int arf_src_index = get_arf_src_index(&cpi->gf_group, cpi->oxcf.pass);
av1_temporal_filter(cpi, -1 * arf_src_index, NULL);
} else {
av1_temporal_filter(cpi, -1, NULL);
}
aom_extend_frame_borders(&cpi->alt_ref_buffer, av1_num_planes(cm));
// Use the filtered frame for encoding.
frame_input->source = &cpi->alt_ref_buffer;
// Copy metadata info to alt-ref buffer.
aom_remove_metadata_from_frame_buffer(frame_input->source);
aom_copy_metadata_to_frame_buffer(frame_input->source,
source_kf_buffer->metadata);
if (oxcf->enable_tpl_model && oxcf->lag_in_frames > 0 &&
frame_params->show_frame) {
av1_tpl_setup_stats(cpi, 0, frame_params, frame_input);
}
}
if (av1_encode(cpi, dest, frame_input, frame_params, frame_results) !=
AOM_CODEC_OK) {
return AOM_CODEC_ERROR;
}
// Set frame_input source to true source for psnr calculation.
if (apply_filtering) {
cpi->source = source_kf_buffer;
cpi->unscaled_source = source_kf_buffer;
}
return AOM_CODEC_OK;
}Función 3.av1_encode
La función av1_encode usa los parámetros actualizados (EncodeFrameInput y EncodeFrameParams) en av1_encode_strategy para actualizar los parámetros correspondientes en AV1_COMP y AV1_COMMON. Si es la primera codificación, ingrese la función av1_first_pass ; de lo contrario, ingrese la función encode_frame_to_data_rate para la codificación.
int av1_encode(AV1_COMP *const cpi, uint8_t *const dest,
const EncodeFrameInput *const frame_input,
const EncodeFrameParams *const frame_params,
EncodeFrameResults *const frame_results) {
AV1_COMMON *const cm = &cpi->common;
CurrentFrame *const current_frame = &cm->current_frame;
cpi->unscaled_source = frame_input->source;
cpi->source = frame_input->source;
cpi->unscaled_last_source = frame_input->last_source;
current_frame->refresh_frame_flags = frame_params->refresh_frame_flags;
cm->features.error_resilient_mode = frame_params->error_resilient_mode;
cm->features.primary_ref_frame = frame_params->primary_ref_frame;
cm->current_frame.frame_type = frame_params->frame_type;
cm->show_frame = frame_params->show_frame;
cpi->ref_frame_flags = frame_params->ref_frame_flags;
cpi->speed = frame_params->speed;
cm->show_existing_frame = frame_params->show_existing_frame;
cpi->existing_fb_idx_to_show = frame_params->existing_fb_idx_to_show;
memcpy(cm->remapped_ref_idx, frame_params->remapped_ref_idx,
REF_FRAMES * sizeof(*cm->remapped_ref_idx));
cpi->refresh_golden_frame = frame_params->refresh_golden_frame;
cpi->refresh_bwd_ref_frame = frame_params->refresh_bwd_ref_frame;
cpi->refresh_alt_ref_frame = frame_params->refresh_alt_ref_frame;
if (current_frame->frame_type == KEY_FRAME && cm->show_frame)
current_frame->frame_number = 0;
current_frame->order_hint =
current_frame->frame_number + frame_params->order_offset;
current_frame->display_order_hint = current_frame->order_hint;
current_frame->order_hint %=
(1 << (cm->seq_params.order_hint_info.order_hint_bits_minus_1 + 1));
if (is_stat_generation_stage(cpi)) {
#if !CONFIG_REALTIME_ONLY
av1_first_pass(cpi, frame_input->ts_duration);
#endif
} else if (cpi->oxcf.pass == 0 || cpi->oxcf.pass == 2) {
if (encode_frame_to_data_rate(cpi, &frame_results->size, dest) !=
AOM_CODEC_OK) {
return AOM_CODEC_ERROR;
}
} else {
return AOM_CODEC_ERROR;
}
return AOM_CODEC_OK;
}