void av1_vaq_frame_setup(AV1_COMP *cpi) {
AV1_COMMON *cm = &cpi->common;
struct segmentation *seg = &cm->seg;
int i;
int resolution_change =
cm->prev_frame && (cm->width != cm->prev_frame->width ||
cm->height != cm->prev_frame->height);
if (resolution_change) {
memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
av1_clearall_segfeatures(seg);
aom_clear_system_state();
av1_disable_segmentation(seg);
return;
}
if (frame_is_intra_only(cm) || cm->error_resilient_mode ||
cpi->refresh_alt_ref_frame ||
(cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) {
cpi->vaq_refresh = 1;
av1_enable_segmentation(seg);
av1_clearall_segfeatures(seg);
aom_clear_system_state();
for (i = 0; i < MAX_SEGMENTS; ++i) {
int qindex_delta =
av1_compute_qdelta_by_rate(&cpi->rc, cm->frame_type, cm->base_qindex,
rate_ratio[i], cm->bit_depth);
// We don't allow qindex 0 in a segment if the base value is not 0.
// Q index 0 (lossless) implies 4x4 encoding only and in AQ mode a segment
// Q delta is sometimes applied without going back around the rd loop.
// This could lead to an illegal combination of partition size and q.
if ((cm->base_qindex != 0) && ((cm->base_qindex + qindex_delta) == 0)) {
qindex_delta = -cm->base_qindex + 1;
}
av1_set_segdata(seg, i, SEG_LVL_ALT_Q, qindex_delta);
av1_enable_segfeature(seg, i, SEG_LVL_ALT_Q);
}
}
}
void av1_vaq_frame_setup(AV1_COMP *cpi) {
AV1_COMMON *cm = &cpi->common;
struct segmentation *seg = &cm->seg;
int i;
if (frame_is_intra_only(cm) || cm->error_resilient_mode ||
cpi->refresh_alt_ref_frame ||
(cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) {
cpi->vaq_refresh = 1;
av1_enable_segmentation(seg);
av1_clearall_segfeatures(seg);
seg->abs_delta = SEGMENT_DELTADATA;
aom_clear_system_state();
for (i = 0; i < MAX_SEGMENTS; ++i) {
int qindex_delta =
av1_compute_qdelta_by_rate(&cpi->rc, cm->frame_type, cm->base_qindex,
rate_ratio[i], cm->bit_depth);
// We don't allow qindex 0 in a segment if the base value is not 0.
// Q index 0 (lossless) implies 4x4 encoding only and in AQ mode a segment
// Q delta is sometimes applied without going back around the rd loop.
// This could lead to an illegal combination of partition size and q.
if ((cm->base_qindex != 0) && ((cm->base_qindex + qindex_delta) == 0)) {
qindex_delta = -cm->base_qindex + 1;
}
// No need to enable SEG_LVL_ALT_Q for this segment.
if (rate_ratio[i] == 1.0) {
continue;
}
av1_set_segdata(seg, i, SEG_LVL_ALT_Q, qindex_delta);
av1_enable_segfeature(seg, i, SEG_LVL_ALT_Q);
}
}
}
double aom_get_ssim_metrics(uint8_t *img1, int img1_pitch, uint8_t *img2,
int img2_pitch, int width, int height, Ssimv *sv2,
Metrics *m, int do_inconsistency) {
double dssim_total = 0;
double ssim_total = 0;
double ssim2_total = 0;
double inconsistency_total = 0;
int i, j;
int c = 0;
double norm;
double old_ssim_total = 0;
aom_clear_system_state();
// We can sample points as frequently as we like start with 1 per 4x4.
for (i = 0; i < height;
i += 4, img1 += img1_pitch * 4, img2 += img2_pitch * 4) {
for (j = 0; j < width; j += 4, ++c) {
Ssimv sv = { 0 };
double ssim;
double ssim2;
double dssim;
uint32_t var_new;
uint32_t var_old;
uint32_t mean_new;
uint32_t mean_old;
double ssim_new;
double ssim_old;
// Not sure there's a great way to handle the edge pixels
// in ssim when using a window. Seems biased against edge pixels
// however you handle this. This uses only samples that are
// fully in the frame.
if (j + 8 <= width && i + 8 <= height) {
ssimv_parms(img1 + j, img1_pitch, img2 + j, img2_pitch, &sv);
}
ssim = ssimv_similarity(&sv, 64);
ssim2 = ssimv_similarity2(&sv, 64);
sv.ssim = ssim2;
// dssim is calculated to use as an actual error metric and
// is scaled up to the same range as sum square error.
// Since we are subsampling every 16th point maybe this should be
// *16 ?
dssim = 255 * 255 * (1 - ssim2) / 2;
// Here I introduce a new error metric: consistency-weighted
// SSIM-inconsistency. This metric isolates frames where the
// SSIM 'suddenly' changes, e.g. if one frame in every 8 is much
// sharper or blurrier than the others. Higher values indicate a
// temporally inconsistent SSIM. There are two ideas at work:
//
// 1) 'SSIM-inconsistency': the total inconsistency value
// reflects how much SSIM values are changing between this
// source / reference frame pair and the previous pair.
//
// 2) 'consistency-weighted': weights de-emphasize areas in the
// frame where the scene content has changed. Changes in scene
// content are detected via changes in local variance and local
// mean.
//
// Thus the overall measure reflects how inconsistent the SSIM
// values are, over consistent regions of the frame.
//
// The metric has three terms:
//
// term 1 -> uses change in scene Variance to weight error score
// 2 * var(Fi)*var(Fi-1) / (var(Fi)^2+var(Fi-1)^2)
// larger changes from one frame to the next mean we care
// less about consistency.
//
// term 2 -> uses change in local scene luminance to weight error
// 2 * avg(Fi)*avg(Fi-1) / (avg(Fi)^2+avg(Fi-1)^2)
// larger changes from one frame to the next mean we care
// less about consistency.
//
// term3 -> measures inconsistency in ssim scores between frames
// 1 - ( 2 * ssim(Fi)*ssim(Fi-1)/(ssim(Fi)^2+sssim(Fi-1)^2).
//
// This term compares the ssim score for the same location in 2
// subsequent frames.
var_new = sv.sum_sq_s - sv.sum_s * sv.sum_s / 64;
var_old = sv2[c].sum_sq_s - sv2[c].sum_s * sv2[c].sum_s / 64;
mean_new = sv.sum_s;
mean_old = sv2[c].sum_s;
ssim_new = sv.ssim;
ssim_old = sv2[c].ssim;
if (do_inconsistency) {
// We do the metric once for every 4x4 block in the image. Since
// we are scaling the error to SSE for use in a psnr calculation
// 1.0 = 4x4x255x255 the worst error we can possibly have.
static const double kScaling = 4. * 4 * 255 * 255;
// The constants have to be non 0 to avoid potential divide by 0
// issues other than that they affect kind of a weighting between
// the terms. No testing of what the right terms should be has been
// done.
static const double c1 = 1, c2 = 1, c3 = 1;
// This measures how much consistent variance is in two consecutive
// source frames. 1.0 means they have exactly the same variance.
const double variance_term =
(2.0 * var_old * var_new + c1) /
(1.0 * var_old * var_old + 1.0 * var_new * var_new + c1);
// This measures how consistent the local mean are between two
// consecutive frames. 1.0 means they have exactly the same mean.
const double mean_term =
(2.0 * mean_old * mean_new + c2) /
(1.0 * mean_old * mean_old + 1.0 * mean_new * mean_new + c2);
// This measures how consistent the ssims of two
// consecutive frames is. 1.0 means they are exactly the same.
double ssim_term =
pow((2.0 * ssim_old * ssim_new + c3) /
(ssim_old * ssim_old + ssim_new * ssim_new + c3),
5);
double this_inconsistency;
// Floating point math sometimes makes this > 1 by a tiny bit.
// We want the metric to scale between 0 and 1.0 so we can convert
// it to an snr scaled value.
if (ssim_term > 1) ssim_term = 1;
// This converts the consistency metric to an inconsistency metric
// ( so we can scale it like psnr to something like sum square error.
// The reason for the variance and mean terms is the assumption that
// if there are big changes in the source we shouldn't penalize
// inconsistency in ssim scores a bit less as it will be less visible
// to the user.
this_inconsistency = (1 - ssim_term) * variance_term * mean_term;
this_inconsistency *= kScaling;
inconsistency_total += this_inconsistency;
}
sv2[c] = sv;
ssim_total += ssim;
ssim2_total += ssim2;
dssim_total += dssim;
old_ssim_total += ssim_old;
}
old_ssim_total += 0;
}
norm = 1. / (width / 4) / (height / 4);
ssim_total *= norm;
ssim2_total *= norm;
m->ssim2 = ssim2_total;
m->ssim = ssim_total;
if (old_ssim_total == 0) inconsistency_total = 0;
m->ssimc = inconsistency_total;
m->dssim = dssim_total;
return inconsistency_total;
}
int av1_receive_compressed_data(AV1Decoder *pbi, size_t size,
const uint8_t **psource) {
AV1_COMMON *volatile const cm = &pbi->common;
BufferPool *volatile const pool = cm->buffer_pool;
RefCntBuffer *volatile const frame_bufs = cm->buffer_pool->frame_bufs;
const uint8_t *source = *psource;
int retcode = 0;
cm->error.error_code = AOM_CODEC_OK;
if (size == 0) {
// This is used to signal that we are missing frames.
// We do not know if the missing frame(s) was supposed to update
// any of the reference buffers, but we act conservative and
// mark only the last buffer as corrupted.
//
// TODO(jkoleszar): Error concealment is undefined and non-normative
// at this point, but if it becomes so, [0] may not always be the correct
// thing to do here.
if (cm->frame_refs[0].idx > 0) {
assert(cm->frame_refs[0].buf != NULL);
cm->frame_refs[0].buf->corrupted = 1;
}
}
pbi->ready_for_new_data = 0;
// Find a free buffer for the new frame, releasing the reference previously
// held.
// Check if the previous frame was a frame without any references to it.
// Release frame buffer if not decoding in frame parallel mode.
if (!cm->frame_parallel_decode && cm->new_fb_idx >= 0 &&
frame_bufs[cm->new_fb_idx].ref_count == 0)
pool->release_fb_cb(pool->cb_priv,
&frame_bufs[cm->new_fb_idx].raw_frame_buffer);
// Find a free frame buffer. Return error if can not find any.
cm->new_fb_idx = get_free_fb(cm);
if (cm->new_fb_idx == INVALID_IDX) return AOM_CODEC_MEM_ERROR;
// Assign a MV array to the frame buffer.
cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx];
pbi->hold_ref_buf = 0;
if (cm->frame_parallel_decode) {
AVxWorker *const worker = pbi->frame_worker_owner;
av1_frameworker_lock_stats(worker);
frame_bufs[cm->new_fb_idx].frame_worker_owner = worker;
// Reset decoding progress.
pbi->cur_buf = &frame_bufs[cm->new_fb_idx];
pbi->cur_buf->row = -1;
pbi->cur_buf->col = -1;
av1_frameworker_unlock_stats(worker);
} else {
pbi->cur_buf = &frame_bufs[cm->new_fb_idx];
}
if (setjmp(cm->error.jmp)) {
const AVxWorkerInterface *const winterface = aom_get_worker_interface();
int i;
cm->error.setjmp = 0;
pbi->ready_for_new_data = 1;
// Synchronize all threads immediately as a subsequent decode call may
// cause a resize invalidating some allocations.
winterface->sync(&pbi->lf_worker);
for (i = 0; i < pbi->num_tile_workers; ++i) {
winterface->sync(&pbi->tile_workers[i]);
}
lock_buffer_pool(pool);
// Release all the reference buffers if worker thread is holding them.
if (pbi->hold_ref_buf == 1) {
int ref_index = 0, mask;
for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) {
const int old_idx = cm->ref_frame_map[ref_index];
// Current thread releases the holding of reference frame.
decrease_ref_count(old_idx, frame_bufs, pool);
// Release the reference frame holding in the reference map for the
// decoding of the next frame.
if (mask & 1) decrease_ref_count(old_idx, frame_bufs, pool);
++ref_index;
}
// Current thread releases the holding of reference frame.
for (; ref_index < REF_FRAMES && !cm->show_existing_frame; ++ref_index) {
const int old_idx = cm->ref_frame_map[ref_index];
decrease_ref_count(old_idx, frame_bufs, pool);
}
pbi->hold_ref_buf = 0;
}
// Release current frame.
decrease_ref_count(cm->new_fb_idx, frame_bufs, pool);
unlock_buffer_pool(pool);
aom_clear_system_state();
return -1;
}
// Setup cyclic background refresh: set delta q and segmentation map.
void av1_cyclic_refresh_setup(AV1_COMP *const cpi) {
AV1_COMMON *const cm = &cpi->common;
const RATE_CONTROL *const rc = &cpi->rc;
CYCLIC_REFRESH *const cr = cpi->cyclic_refresh;
struct segmentation *const seg = &cm->seg;
const int apply_cyclic_refresh = apply_cyclic_refresh_bitrate(cm, rc);
if (cm->current_video_frame == 0) cr->low_content_avg = 0.0;
// Don't apply refresh on key frame or enhancement layer frames.
if (!apply_cyclic_refresh || cm->frame_type == KEY_FRAME) {
// Set segmentation map to 0 and disable.
unsigned char *const seg_map = cpi->segmentation_map;
memset(seg_map, 0, cm->mi_rows * cm->mi_cols);
av1_disable_segmentation(&cm->seg);
if (cm->frame_type == KEY_FRAME) {
memset(cr->last_coded_q_map, MAXQ,
cm->mi_rows * cm->mi_cols * sizeof(*cr->last_coded_q_map));
cr->sb_index = 0;
}
return;
} else {
int qindex_delta = 0;
int qindex2;
const double q = av1_convert_qindex_to_q(cm->base_qindex, cm->bit_depth);
aom_clear_system_state();
// Set rate threshold to some multiple (set to 2 for now) of the target
// rate (target is given by sb64_target_rate and scaled by 256).
cr->thresh_rate_sb = ((int64_t)(rc->sb64_target_rate) << 8) << 2;
// Distortion threshold, quadratic in Q, scale factor to be adjusted.
// q will not exceed 457, so (q * q) is within 32bit; see:
// av1_convert_qindex_to_q(), av1_ac_quant(), ac_qlookup*[].
cr->thresh_dist_sb = ((int64_t)(q * q)) << 2;
// Set up segmentation.
// Clear down the segment map.
av1_enable_segmentation(&cm->seg);
av1_clearall_segfeatures(seg);
// Select delta coding method.
seg->abs_delta = SEGMENT_DELTADATA;
// Note: setting temporal_update has no effect, as the seg-map coding method
// (temporal or spatial) is determined in
// av1_choose_segmap_coding_method(),
// based on the coding cost of each method. For error_resilient mode on the
// last_frame_seg_map is set to 0, so if temporal coding is used, it is
// relative to 0 previous map.
// seg->temporal_update = 0;
// Segment BASE "Q" feature is disabled so it defaults to the baseline Q.
av1_disable_segfeature(seg, CR_SEGMENT_ID_BASE, SEG_LVL_ALT_Q);
// Use segment BOOST1 for in-frame Q adjustment.
av1_enable_segfeature(seg, CR_SEGMENT_ID_BOOST1, SEG_LVL_ALT_Q);
// Use segment BOOST2 for more aggressive in-frame Q adjustment.
av1_enable_segfeature(seg, CR_SEGMENT_ID_BOOST2, SEG_LVL_ALT_Q);
// Set the q delta for segment BOOST1.
qindex_delta = compute_deltaq(cpi, cm->base_qindex, cr->rate_ratio_qdelta);
cr->qindex_delta[1] = qindex_delta;
// Compute rd-mult for segment BOOST1.
qindex2 = clamp(cm->base_qindex + cm->y_dc_delta_q + qindex_delta, 0, MAXQ);
cr->rdmult = av1_compute_rd_mult(cpi, qindex2);
av1_set_segdata(seg, CR_SEGMENT_ID_BOOST1, SEG_LVL_ALT_Q, qindex_delta);
// Set a more aggressive (higher) q delta for segment BOOST2.
qindex_delta = compute_deltaq(
cpi, cm->base_qindex,
AOMMIN(CR_MAX_RATE_TARGET_RATIO,
0.1 * cr->rate_boost_fac * cr->rate_ratio_qdelta));
cr->qindex_delta[2] = qindex_delta;
av1_set_segdata(seg, CR_SEGMENT_ID_BOOST2, SEG_LVL_ALT_Q, qindex_delta);
// Update the segmentation and refresh map.
cyclic_refresh_update_map(cpi);
}
}
