int vp9_get_raw_frame(VP9Decoder *pbi, YV12_BUFFER_CONFIG *sd, int64_t *time_stamp, int64_t *time_end_stamp, vp9_ppflags_t *flags) { int ret = -1; if (pbi->ready_for_new_data == 1) return ret; /* ie no raw frame to show!!! */ if (pbi->common.show_frame == 0) return ret; pbi->ready_for_new_data = 1; *time_stamp = pbi->last_time_stamp; *time_end_stamp = 0; #if CONFIG_VP9_POSTPROC ret = vp9_post_proc_frame(&pbi->common, sd, flags); #else *sd = *pbi->common.frame_to_show; sd->y_width = pbi->common.width; sd->y_height = pbi->common.height; sd->uv_width = sd->y_width >> pbi->common.subsampling_x; sd->uv_height = sd->y_height >> pbi->common.subsampling_y; ret = 0; #endif /*!CONFIG_POSTPROC*/ vp9_clear_system_state(); return ret; }
int vp9_get_raw_frame(VP9Decoder *pbi, YV12_BUFFER_CONFIG *sd, vp9_ppflags_t *flags) { VP9_COMMON *const cm = &pbi->common; int ret = -1; #if !CONFIG_VP9_POSTPROC (void)*flags; #endif if (pbi->ready_for_new_data == 1) return ret; pbi->ready_for_new_data = 1; /* no raw frame to show!!! */ if (!cm->show_frame) return ret; #if CONFIG_VP9_POSTPROC if (!cm->show_existing_frame) { ret = vp9_post_proc_frame(cm, sd, flags); } else { *sd = *cm->frame_to_show; ret = 0; } #else *sd = *cm->frame_to_show; ret = 0; #endif /*!CONFIG_POSTPROC*/ vp9_clear_system_state(); return ret; }
double vp9_vaq_inv_q_ratio(int energy) { ENERGY_IN_BOUNDS(energy); vp9_clear_system_state(); // __asm emms; return Q_RATIO(-energy); }
static void calc_iframe_target_size(VP9_COMP *cpi) { const VP9_CONFIG *oxcf = &cpi->oxcf; RATE_CONTROL *const rc = &cpi->rc; int target; vp9_clear_system_state(); // __asm emms; // For 1-pass. if (cpi->pass == 0) { if (cpi->common.current_video_frame == 0) { target = oxcf->starting_buffer_level / 2; } else { // TODO(marpan): Add in adjustment based on Q. // If this keyframe was forced, use a more recent Q estimate. // int Q = (cpi->common.frame_flags & FRAMEFLAGS_KEY) ? // cpi->rc.avg_frame_qindex : cpi->rc.ni_av_qi; int initial_boost = 32; // Boost depends somewhat on frame rate. int kf_boost = MAX(initial_boost, (int)(2 * cpi->output_framerate - 16)); // Adjustment up based on q: need to fix. // kf_boost = kf_boost * kfboost_qadjust(Q) / 100; // Frame separation adjustment (down). if (rc->frames_since_key < cpi->output_framerate / 2) { kf_boost = (int)(kf_boost * rc->frames_since_key / (cpi->output_framerate / 2)); } kf_boost = (kf_boost < 16) ? 16 : kf_boost; target = ((16 + kf_boost) * rc->per_frame_bandwidth) >> 4; } rc->active_worst_quality = rc->worst_quality; } else {
double vp9_vaq_rdmult_ratio(int energy) { ENERGY_IN_BOUNDS(energy); vp9_clear_system_state(); // __asm emms; return RDMULT_RATIO(energy); }
void vp9_setup_in_frame_q_adj(VP9_COMP *cpi) { VP9_COMMON *const cm = &cpi->common; struct segmentation *const seg = &cm->seg; // Make SURE use of floating point in this function is safe. vp9_clear_system_state(); if (cm->frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame || (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) { int segment; const int aq_strength = get_aq_c_strength(cm->base_qindex, cm->bit_depth); // Clear down the segment map. memset(cpi->segmentation_map, DEFAULT_AQ2_SEG, cm->mi_rows * cm->mi_cols); vp9_clearall_segfeatures(seg); // Segmentation only makes sense if the target bits per SB is above a // threshold. Below this the overheads will usually outweigh any benefit. if (cpi->rc.sb64_target_rate < 256) { vp9_disable_segmentation(seg); return; } vp9_enable_segmentation(seg); // Select delta coding method. seg->abs_delta = SEGMENT_DELTADATA; // Default segment "Q" feature is disabled so it defaults to the baseline Q. vp9_disable_segfeature(seg, DEFAULT_AQ2_SEG, SEG_LVL_ALT_Q); // Use some of the segments for in frame Q adjustment. for (segment = 0; segment < AQ_C_SEGMENTS; ++segment) { int qindex_delta; if (segment == DEFAULT_AQ2_SEG) continue; qindex_delta = vp9_compute_qdelta_by_rate(&cpi->rc, cm->frame_type, cm->base_qindex, aq_c_q_adj_factor[aq_strength][segment], cm->bit_depth); // For AQ complexity mode, we dont allow Q0 in a segment if the base // Q is not 0. Q0 (lossless) implies 4x4 only and in AQ mode 2 a segment // Q delta is sometimes applied without going back around the rd loop. // This could lead to an illegal combination of partition size and q. if ((cm->base_qindex != 0) && ((cm->base_qindex + qindex_delta) == 0)) { qindex_delta = -cm->base_qindex + 1; } if ((cm->base_qindex + qindex_delta) > 0) { vp9_enable_segfeature(seg, segment, SEG_LVL_ALT_Q); vp9_set_segdata(seg, segment, SEG_LVL_ALT_Q, qindex_delta); } } } }
int vp9_block_energy(VP9_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs) { double energy; unsigned int var = block_variance(cpi, x, bs); vp9_clear_system_state(); // __asm emms; // if (var <= 1000) // return 0; energy = 0.9*(logf(var + 1) - 10.0); return clamp(round(energy), ENERGY_MIN, ENERGY_MAX); }
void vp9_transform_mby_16x16(MACROBLOCK *x) { MACROBLOCKD *xd = &x->e_mbd; BLOCK *b = &x->block[0]; TX_TYPE tx_type = get_tx_type_16x16(xd, &xd->block[0]); vp9_clear_system_state(); if (tx_type != DCT_DCT) { vp9_fht_c(b->src_diff, 32, b->coeff, tx_type, 16); } else { x->vp9_short_fdct16x16(&x->block[0].src_diff[0], &x->block[0].coeff[0], 32); } }
// Select a segment for the current block. // The choice of segment for a block depends on the ratio of the projected // bits for the block vs a target average and its spatial complexity. void vp9_caq_select_segment(VP9_COMP *cpi, MACROBLOCK *mb, BLOCK_SIZE bs, int mi_row, int mi_col, int projected_rate) { VP9_COMMON *const cm = &cpi->common; const int mi_offset = mi_row * cm->mi_cols + mi_col; const int bw = num_8x8_blocks_wide_lookup[BLOCK_64X64]; const int bh = num_8x8_blocks_high_lookup[BLOCK_64X64]; const int xmis = MIN(cm->mi_cols - mi_col, num_8x8_blocks_wide_lookup[bs]); const int ymis = MIN(cm->mi_rows - mi_row, num_8x8_blocks_high_lookup[bs]); int x, y; int i; unsigned char segment; if (0) { segment = DEFAULT_AQ2_SEG; } else { // Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh). // It is converted to bits * 256 units. const int target_rate = (cpi->rc.sb64_target_rate * xmis * ymis * 256) / (bw * bh); double logvar; double low_var_thresh; const int aq_strength = get_aq_c_strength(cm->base_qindex, cm->bit_depth); vp9_clear_system_state(); low_var_thresh = (cpi->oxcf.pass == 2) ? MAX(cpi->twopass.mb_av_energy, MIN_DEFAULT_LV_THRESH) : DEFAULT_LV_THRESH; vp9_setup_src_planes(mb, cpi->Source, mi_row, mi_col); logvar = vp9_log_block_var(cpi, mb, bs); segment = AQ_C_SEGMENTS - 1; // Just in case no break out below. for (i = 0; i < AQ_C_SEGMENTS; ++i) { // Test rate against a threshold value and variance against a threshold. // Increasing segment number (higher variance and complexity) = higher Q. if ((projected_rate < target_rate * aq_c_transitions[aq_strength][i]) && (logvar < (low_var_thresh + aq_c_var_thresholds[aq_strength][i]))) { segment = i; break; } } } // Fill in the entires in the segment map corresponding to this SB64. for (y = 0; y < ymis; y++) { for (x = 0; x < xmis; x++) { cpi->segmentation_map[mi_offset + y * cm->mi_cols + x] = segment; } } }
void vp9_vaq_init() { int i; double base_ratio; assert(ENERGY_SPAN <= MAX_SEGMENTS); vp9_clear_system_state(); // __asm emms; base_ratio = 1.5; for (i = ENERGY_MIN; i <= ENERGY_MAX; i++) { Q_RATIO(i) = pow(base_ratio, i/3.0); } }
static void calc_iframe_target_size(VP9_COMP *cpi) { // boost defaults to half second int target; // Clear down mmx registers to allow floating point in what follows vp9_clear_system_state(); // __asm emms; // New Two pass RC target = cpi->rc.per_frame_bandwidth; // For 1-pass. if (cpi->pass == 0) { if (cpi->common.current_video_frame == 0) { target = cpi->oxcf.starting_buffer_level / 2; } else { // TODO(marpan): Add in adjustment based on Q. // If this keyframe was forced, use a more recent Q estimate. // int Q = (cpi->common.frame_flags & FRAMEFLAGS_KEY) ? // cpi->rc.avg_frame_qindex : cpi->rc.ni_av_qi; int initial_boost = 32; // Boost depends somewhat on frame rate. int kf_boost = MAX(initial_boost, (int)(2 * cpi->output_framerate - 16)); // Adjustment up based on q: need to fix. // kf_boost = kf_boost * kfboost_qadjust(Q) / 100; // Frame separation adjustment (down). if (cpi->rc.frames_since_key < cpi->output_framerate / 2) { kf_boost = (int)(kf_boost * cpi->rc.frames_since_key / (cpi->output_framerate / 2)); } kf_boost = (kf_boost < 16) ? 16 : kf_boost; target = ((16 + kf_boost) * cpi->rc.per_frame_bandwidth) >> 4; } cpi->rc.active_worst_quality = cpi->rc.worst_quality; } if (cpi->oxcf.rc_max_intra_bitrate_pct) { int max_rate = cpi->rc.per_frame_bandwidth * cpi->oxcf.rc_max_intra_bitrate_pct / 100; if (target > max_rate) target = max_rate; } cpi->rc.this_frame_target = target; }
void vp9_update_mbgraph_stats ( VP9_COMP *cpi ) { VP9_COMMON *const cm = &cpi->common; int i, n_frames = vp9_lookahead_depth(cpi->lookahead); YV12_BUFFER_CONFIG *golden_ref = &cm->yv12_fb[cm->gld_fb_idx]; // we need to look ahead beyond where the ARF transitions into // being a GF - so exit if we don't look ahead beyond that if (n_frames <= cpi->frames_till_gf_update_due) return; if (n_frames > (int)cpi->common.frames_till_alt_ref_frame) n_frames = cpi->common.frames_till_alt_ref_frame; if (n_frames > MAX_LAG_BUFFERS) n_frames = MAX_LAG_BUFFERS; cpi->mbgraph_n_frames = n_frames; for (i = 0; i < n_frames; i++) { MBGRAPH_FRAME_STATS *frame_stats = &cpi->mbgraph_stats[i]; vpx_memset(frame_stats->mb_stats, 0, cm->mb_rows * cm->mb_cols * sizeof(*cpi->mbgraph_stats[i].mb_stats)); } // do motion search to find contribution of each reference to data // later on in this GF group // FIXME really, the GF/last MC search should be done forward, and // the ARF MC search backwards, to get optimal results for MV caching for (i = 0; i < n_frames; i++) { MBGRAPH_FRAME_STATS *frame_stats = &cpi->mbgraph_stats[i]; struct lookahead_entry *q_cur = vp9_lookahead_peek(cpi->lookahead, i); assert(q_cur != NULL); update_mbgraph_frame_stats(cpi, frame_stats, &q_cur->img, golden_ref, cpi->Source); } vp9_clear_system_state(); // __asm emms; separate_arf_mbs(cpi); }
void vp9_vaq_frame_setup(VP9_COMP *cpi) { VP9_COMMON *cm = &cpi->common; struct segmentation *seg = &cm->seg; int base_q = vp9_convert_qindex_to_q(cm->base_qindex); int base_rdmult = vp9_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q); int i; if (cm->frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame || (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) { vp9_enable_segmentation((VP9_PTR)cpi); vp9_clearall_segfeatures(seg); seg->abs_delta = SEGMENT_DELTADATA; vp9_clear_system_state(); // __asm emms; for (i = ENERGY_MIN; i <= ENERGY_MAX; i++) { int qindex_delta, segment_rdmult; if (Q_RATIO(i) == 1) { // No need to enable SEG_LVL_ALT_Q for this segment RDMULT_RATIO(i) = 1; continue; } qindex_delta = vp9_compute_qdelta(cpi, base_q, base_q * Q_RATIO(i)); vp9_set_segdata(seg, SEGMENT_ID(i), SEG_LVL_ALT_Q, qindex_delta); vp9_enable_segfeature(seg, SEGMENT_ID(i), SEG_LVL_ALT_Q); segment_rdmult = vp9_compute_rd_mult(cpi, cm->base_qindex + qindex_delta + cm->y_dc_delta_q); RDMULT_RATIO(i) = (double) segment_rdmult / base_rdmult; } } }
int vp9_receive_compressed_data(VP9Decoder *pbi, size_t size, const uint8_t **psource) { VP9_COMMON *const cm = &pbi->common; const uint8_t *source = *psource; int retcode = 0; cm->error.error_code = VPX_CODEC_OK; if (size == 0) { // This is used to signal that we are missing frames. // We do not know if the missing frame(s) was supposed to update // any of the reference buffers, but we act conservative and // mark only the last buffer as corrupted. // // TODO(jkoleszar): Error concealment is undefined and non-normative // at this point, but if it becomes so, [0] may not always be the correct // thing to do here. if (cm->frame_refs[0].idx != INT_MAX) cm->frame_refs[0].buf->corrupted = 1; } pbi->ready_for_new_data = 0; // Check if the previous frame was a frame without any references to it. if (cm->new_fb_idx >= 0 && cm->frame_bufs[cm->new_fb_idx].ref_count == 0) cm->release_fb_cb(cm->cb_priv, &cm->frame_bufs[cm->new_fb_idx].raw_frame_buffer); cm->new_fb_idx = get_free_fb(cm); // Assign a MV array to the frame buffer. cm->cur_frame = &cm->frame_bufs[cm->new_fb_idx]; if (setjmp(cm->error.jmp)) { pbi->need_resync = 1; cm->error.setjmp = 0; vp9_clear_system_state(); // We do not know if the missing frame(s) was supposed to update // any of the reference buffers, but we act conservative and // mark only the last buffer as corrupted. // // TODO(jkoleszar): Error concealment is undefined and non-normative // at this point, but if it becomes so, [0] may not always be the correct // thing to do here. if (cm->frame_refs[0].idx != INT_MAX && cm->frame_refs[0].buf != NULL) cm->frame_refs[0].buf->corrupted = 1; if (cm->new_fb_idx > 0 && cm->frame_bufs[cm->new_fb_idx].ref_count > 0) cm->frame_bufs[cm->new_fb_idx].ref_count--; return -1; } cm->error.setjmp = 1; vp9_decode_frame(pbi, source, source + size, psource); swap_frame_buffers(pbi); vp9_clear_system_state(); cm->last_width = cm->width; cm->last_height = cm->height; if (!cm->show_existing_frame) { cm->last_show_frame = cm->show_frame; cm->prev_frame = cm->cur_frame; } if (cm->show_frame) cm->current_video_frame++; cm->error.setjmp = 0; return retcode; }
// Setup cyclic background refresh: set delta q and segmentation map. void vp9_cyclic_refresh_setup(VP9_COMP *const cpi) { VP9_COMMON *const cm = &cpi->common; const RATE_CONTROL *const rc = &cpi->rc; CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; struct segmentation *const seg = &cm->seg; const int apply_cyclic_refresh = apply_cyclic_refresh_bitrate(cm, rc); if (cm->current_video_frame == 0) cr->low_content_avg = 0.0; // Don't apply refresh on key frame or enhancement layer frames. if (!apply_cyclic_refresh || (cm->frame_type == KEY_FRAME) || (cpi->svc.temporal_layer_id > 0) || (cpi->svc.spatial_layer_id > 0)) { // Set segmentation map to 0 and disable. unsigned char *const seg_map = cpi->segmentation_map; memset(seg_map, 0, cm->mi_rows * cm->mi_cols); vp9_disable_segmentation(&cm->seg); if (cm->frame_type == KEY_FRAME) { memset(cr->last_coded_q_map, MAXQ, cm->mi_rows * cm->mi_cols * sizeof(*cr->last_coded_q_map)); cr->sb_index = 0; } return; } else { int qindex_delta = 0; int qindex2; const double q = vp9_convert_qindex_to_q(cm->base_qindex, cm->bit_depth); vp9_clear_system_state(); // Set rate threshold to some multiple (set to 2 for now) of the target // rate (target is given by sb64_target_rate and scaled by 256). cr->thresh_rate_sb = ((int64_t)(rc->sb64_target_rate) << 8) << 2; // Distortion threshold, quadratic in Q, scale factor to be adjusted. // q will not exceed 457, so (q * q) is within 32bit; see: // vp9_convert_qindex_to_q(), vp9_ac_quant(), ac_qlookup*[]. cr->thresh_dist_sb = ((int64_t)(q * q)) << 2; // Set up segmentation. // Clear down the segment map. vp9_enable_segmentation(&cm->seg); vp9_clearall_segfeatures(seg); // Select delta coding method. seg->abs_delta = SEGMENT_DELTADATA; // Note: setting temporal_update has no effect, as the seg-map coding method // (temporal or spatial) is determined in vp9_choose_segmap_coding_method(), // based on the coding cost of each method. For error_resilient mode on the // last_frame_seg_map is set to 0, so if temporal coding is used, it is // relative to 0 previous map. // seg->temporal_update = 0; // Segment BASE "Q" feature is disabled so it defaults to the baseline Q. vp9_disable_segfeature(seg, CR_SEGMENT_ID_BASE, SEG_LVL_ALT_Q); // Use segment BOOST1 for in-frame Q adjustment. vp9_enable_segfeature(seg, CR_SEGMENT_ID_BOOST1, SEG_LVL_ALT_Q); // Use segment BOOST2 for more aggressive in-frame Q adjustment. vp9_enable_segfeature(seg, CR_SEGMENT_ID_BOOST2, SEG_LVL_ALT_Q); // Set the q delta for segment BOOST1. qindex_delta = compute_deltaq(cpi, cm->base_qindex, cr->rate_ratio_qdelta); cr->qindex_delta[1] = qindex_delta; // Compute rd-mult for segment BOOST1. qindex2 = clamp(cm->base_qindex + cm->y_dc_delta_q + qindex_delta, 0, MAXQ); cr->rdmult = vp9_compute_rd_mult(cpi, qindex2); vp9_set_segdata(seg, CR_SEGMENT_ID_BOOST1, SEG_LVL_ALT_Q, qindex_delta); // Set a more aggressive (higher) q delta for segment BOOST2. qindex_delta = compute_deltaq( cpi, cm->base_qindex, MIN(CR_MAX_RATE_TARGET_RATIO, 0.1 * cr->rate_boost_fac * cr->rate_ratio_qdelta)); cr->qindex_delta[2] = qindex_delta; vp9_set_segdata(seg, CR_SEGMENT_ID_BOOST2, SEG_LVL_ALT_Q, qindex_delta); // Update the segmentation and refresh map. cyclic_refresh_update_map(cpi); } }
int vp9_receive_compressed_data(VP9Decoder *pbi, size_t size, const uint8_t **psource, int64_t time_stamp) { VP9_COMMON *const cm = &pbi->common; const uint8_t *source = *psource; int retcode = 0; cm->error.error_code = VPX_CODEC_OK; if (size == 0) { // This is used to signal that we are missing frames. // We do not know if the missing frame(s) was supposed to update // any of the reference buffers, but we act conservative and // mark only the last buffer as corrupted. // // TODO(jkoleszar): Error concealment is undefined and non-normative // at this point, but if it becomes so, [0] may not always be the correct // thing to do here. if (cm->frame_refs[0].idx != INT_MAX) cm->frame_refs[0].buf->corrupted = 1; } // Check if the previous frame was a frame without any references to it. if (cm->new_fb_idx >= 0 && cm->frame_bufs[cm->new_fb_idx].ref_count == 0) cm->release_fb_cb(cm->cb_priv, &cm->frame_bufs[cm->new_fb_idx].raw_frame_buffer); cm->new_fb_idx = get_free_fb(cm); if (setjmp(cm->error.jmp)) { cm->error.setjmp = 0; // We do not know if the missing frame(s) was supposed to update // any of the reference buffers, but we act conservative and // mark only the last buffer as corrupted. // // TODO(jkoleszar): Error concealment is undefined and non-normative // at this point, but if it becomes so, [0] may not always be the correct // thing to do here. if (cm->frame_refs[0].idx != INT_MAX) cm->frame_refs[0].buf->corrupted = 1; if (cm->frame_bufs[cm->new_fb_idx].ref_count > 0) cm->frame_bufs[cm->new_fb_idx].ref_count--; return -1; } cm->error.setjmp = 1; retcode = vp9_decode_frame(pbi, source, source + size, psource); if (retcode < 0) { cm->error.error_code = VPX_CODEC_ERROR; cm->error.setjmp = 0; if (cm->frame_bufs[cm->new_fb_idx].ref_count > 0) cm->frame_bufs[cm->new_fb_idx].ref_count--; return retcode; } swap_frame_buffers(pbi); #if WRITE_RECON_BUFFER == 2 if (cm->show_frame) write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame); else write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame + 1000); #endif if (!pbi->do_loopfilter_inline) { // If multiple threads are used to decode tiles, then we use those threads // to do parallel loopfiltering. if (pbi->num_tile_workers) { vp9_loop_filter_frame_mt(pbi, cm, cm->lf.filter_level, 0, 0); } else { vp9_loop_filter_frame(cm, &pbi->mb, cm->lf.filter_level, 0, 0); } } #if WRITE_RECON_BUFFER == 2 if (cm->show_frame) write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame + 2000); else write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame + 3000); #endif #if WRITE_RECON_BUFFER == 1 if (cm->show_frame) recon_write_yuv_frame("recon.yuv", cm->frame_to_show, cm->width, cm->height); #endif vp9_clear_system_state(); cm->last_width = cm->width; cm->last_height = cm->height; if (!cm->show_existing_frame) cm->last_show_frame = cm->show_frame; if (cm->show_frame) { if (!cm->show_existing_frame) vp9_swap_mi_and_prev_mi(cm); cm->current_video_frame++; } pbi->ready_for_new_data = 0; pbi->last_time_stamp = time_stamp; cm->error.setjmp = 0; return retcode; }
double vp9_get_ssim_metrics(uint8_t *img1, int img1_pitch, uint8_t *img2, int img2_pitch, int width, int height, Ssimv *sv2, Metrics *m, int do_inconsistency) { double dssim_total = 0; double ssim_total = 0; double ssim2_total = 0; double inconsistency_total = 0; int i, j; int c = 0; double norm; double old_ssim_total = 0; vp9_clear_system_state(); // We can sample points as frequently as we like start with 1 per 4x4. for (i = 0; i < height; i += 4, img1 += img1_pitch * 4, img2 += img2_pitch * 4) { for (j = 0; j < width; j += 4, ++c) { Ssimv sv = {0}; double ssim; double ssim2; double dssim; uint32_t var_new; uint32_t var_old; uint32_t mean_new; uint32_t mean_old; double ssim_new; double ssim_old; // Not sure there's a great way to handle the edge pixels // in ssim when using a window. Seems biased against edge pixels // however you handle this. This uses only samples that are // fully in the frame. if (j + 8 <= width && i + 8 <= height) { ssimv_parms(img1 + j, img1_pitch, img2 + j, img2_pitch, &sv); } ssim = ssimv_similarity(&sv, 64); ssim2 = ssimv_similarity2(&sv, 64); sv.ssim = ssim2; // dssim is calculated to use as an actual error metric and // is scaled up to the same range as sum square error. // Since we are subsampling every 16th point maybe this should be // *16 ? dssim = 255 * 255 * (1 - ssim2) / 2; // Here I introduce a new error metric: consistency-weighted // SSIM-inconsistency. This metric isolates frames where the // SSIM 'suddenly' changes, e.g. if one frame in every 8 is much // sharper or blurrier than the others. Higher values indicate a // temporally inconsistent SSIM. There are two ideas at work: // // 1) 'SSIM-inconsistency': the total inconsistency value // reflects how much SSIM values are changing between this // source / reference frame pair and the previous pair. // // 2) 'consistency-weighted': weights de-emphasize areas in the // frame where the scene content has changed. Changes in scene // content are detected via changes in local variance and local // mean. // // Thus the overall measure reflects how inconsistent the SSIM // values are, over consistent regions of the frame. // // The metric has three terms: // // term 1 -> uses change in scene Variance to weight error score // 2 * var(Fi)*var(Fi-1) / (var(Fi)^2+var(Fi-1)^2) // larger changes from one frame to the next mean we care // less about consistency. // // term 2 -> uses change in local scene luminance to weight error // 2 * avg(Fi)*avg(Fi-1) / (avg(Fi)^2+avg(Fi-1)^2) // larger changes from one frame to the next mean we care // less about consistency. // // term3 -> measures inconsistency in ssim scores between frames // 1 - ( 2 * ssim(Fi)*ssim(Fi-1)/(ssim(Fi)^2+sssim(Fi-1)^2). // // This term compares the ssim score for the same location in 2 // subsequent frames. var_new = sv.sum_sq_s - sv.sum_s * sv.sum_s / 64; var_old = sv2[c].sum_sq_s - sv2[c].sum_s * sv2[c].sum_s / 64; mean_new = sv.sum_s; mean_old = sv2[c].sum_s; ssim_new = sv.ssim; ssim_old = sv2[c].ssim; if (do_inconsistency) { // We do the metric once for every 4x4 block in the image. Since // we are scaling the error to SSE for use in a psnr calculation // 1.0 = 4x4x255x255 the worst error we can possibly have. static const double kScaling = 4. * 4 * 255 * 255; // The constants have to be non 0 to avoid potential divide by 0 // issues other than that they affect kind of a weighting between // the terms. No testing of what the right terms should be has been // done. static const double c1 = 1, c2 = 1, c3 = 1; // This measures how much consistent variance is in two consecutive // source frames. 1.0 means they have exactly the same variance. const double variance_term = (2.0 * var_old * var_new + c1) / (1.0 * var_old * var_old + 1.0 * var_new * var_new + c1); // This measures how consistent the local mean are between two // consecutive frames. 1.0 means they have exactly the same mean. const double mean_term = (2.0 * mean_old * mean_new + c2) / (1.0 * mean_old * mean_old + 1.0 * mean_new * mean_new + c2); // This measures how consistent the ssims of two // consecutive frames is. 1.0 means they are exactly the same. double ssim_term = pow((2.0 * ssim_old * ssim_new + c3) / (ssim_old * ssim_old + ssim_new * ssim_new + c3), 5); double this_inconsistency; // Floating point math sometimes makes this > 1 by a tiny bit. // We want the metric to scale between 0 and 1.0 so we can convert // it to an snr scaled value. if (ssim_term > 1) ssim_term = 1; // This converts the consistency metric to an inconsistency metric // ( so we can scale it like psnr to something like sum square error. // The reason for the variance and mean terms is the assumption that // if there are big changes in the source we shouldn't penalize // inconsistency in ssim scores a bit less as it will be less visible // to the user. this_inconsistency = (1 - ssim_term) * variance_term * mean_term; this_inconsistency *= kScaling; inconsistency_total += this_inconsistency; } sv2[c] = sv; ssim_total += ssim; ssim2_total += ssim2; dssim_total += dssim; old_ssim_total += ssim_old; } old_ssim_total += 0; } norm = 1. / (width / 4) / (height / 4); ssim_total *= norm; ssim2_total *= norm; m->ssim2 = ssim2_total; m->ssim = ssim_total; if (old_ssim_total == 0) inconsistency_total = 0; m->ssimc = inconsistency_total; m->dssim = dssim_total; return inconsistency_total; }
int vp9_receive_compressed_data(VP9Decoder *pbi, size_t size, const uint8_t **psource) { VP9_COMMON *volatile const cm = &pbi->common; BufferPool *volatile const pool = cm->buffer_pool; RefCntBuffer *volatile const frame_bufs = cm->buffer_pool->frame_bufs; const uint8_t *source = *psource; int retcode = 0; cm->error.error_code = VPX_CODEC_OK; if (size == 0) { // This is used to signal that we are missing frames. // We do not know if the missing frame(s) was supposed to update // any of the reference buffers, but we act conservative and // mark only the last buffer as corrupted. // // TODO(jkoleszar): Error concealment is undefined and non-normative // at this point, but if it becomes so, [0] may not always be the correct // thing to do here. if (cm->frame_refs[0].idx > 0) { assert(cm->frame_refs[0].buf != NULL); cm->frame_refs[0].buf->corrupted = 1; } } pbi->ready_for_new_data = 0; // Check if the previous frame was a frame without any references to it. // Release frame buffer if not decoding in frame parallel mode. if (!pbi->frame_parallel_decode && cm->new_fb_idx >= 0 && frame_bufs[cm->new_fb_idx].ref_count == 0) pool->release_fb_cb(pool->cb_priv, &frame_bufs[cm->new_fb_idx].raw_frame_buffer); cm->new_fb_idx = get_free_fb(cm); // Assign a MV array to the frame buffer. cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx]; pbi->hold_ref_buf = 0; if (pbi->frame_parallel_decode) { VP9Worker *const worker = pbi->frame_worker_owner; vp9_frameworker_lock_stats(worker); frame_bufs[cm->new_fb_idx].frame_worker_owner = worker; // Reset decoding progress. pbi->cur_buf = &frame_bufs[cm->new_fb_idx]; pbi->cur_buf->row = -1; pbi->cur_buf->col = -1; vp9_frameworker_unlock_stats(worker); } else { pbi->cur_buf = &frame_bufs[cm->new_fb_idx]; } if (setjmp(cm->error.jmp)) { const VP9WorkerInterface *const winterface = vp9_get_worker_interface(); int i; cm->error.setjmp = 0; pbi->ready_for_new_data = 1; // Synchronize all threads immediately as a subsequent decode call may // cause a resize invalidating some allocations. winterface->sync(&pbi->lf_worker); for (i = 0; i < pbi->num_tile_workers; ++i) { winterface->sync(&pbi->tile_workers[i]); } lock_buffer_pool(pool); // Release all the reference buffers if worker thread is holding them. if (pbi->hold_ref_buf == 1) { int ref_index = 0, mask; for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) { const int old_idx = cm->ref_frame_map[ref_index]; // Current thread releases the holding of reference frame. decrease_ref_count(old_idx, frame_bufs, pool); // Release the reference frame in reference map. if ((mask & 1) && old_idx >= 0) { decrease_ref_count(old_idx, frame_bufs, pool); } ++ref_index; } // Current thread releases the holding of reference frame. for (; ref_index < REF_FRAMES && !cm->show_existing_frame; ++ref_index) { const int old_idx = cm->ref_frame_map[ref_index]; decrease_ref_count(old_idx, frame_bufs, pool); } pbi->hold_ref_buf = 0; } // Release current frame. decrease_ref_count(cm->new_fb_idx, frame_bufs, pool); unlock_buffer_pool(pool); vp9_clear_system_state(); return -1; }
int vp9_receive_compressed_data(VP9D_PTR ptr, size_t size, const uint8_t **psource, int64_t time_stamp) { VP9D_COMP *pbi = (VP9D_COMP *) ptr; VP9_COMMON *cm = &pbi->common; const uint8_t *source = *psource; int retcode = 0; /*if(pbi->ready_for_new_data == 0) return -1;*/ if (ptr == 0) return -1; cm->error.error_code = VPX_CODEC_OK; pbi->source = source; pbi->source_sz = size; if (pbi->source_sz == 0) { /* This is used to signal that we are missing frames. * We do not know if the missing frame(s) was supposed to update * any of the reference buffers, but we act conservative and * mark only the last buffer as corrupted. * * TODO(jkoleszar): Error concealment is undefined and non-normative * at this point, but if it becomes so, [0] may not always be the correct * thing to do here. */ if (cm->active_ref_idx[0] != INT_MAX) get_frame_ref_buffer(cm, 0)->corrupted = 1; } cm->new_fb_idx = get_free_fb(cm); if (setjmp(cm->error.jmp)) { cm->error.setjmp = 0; /* We do not know if the missing frame(s) was supposed to update * any of the reference buffers, but we act conservative and * mark only the last buffer as corrupted. * * TODO(jkoleszar): Error concealment is undefined and non-normative * at this point, but if it becomes so, [0] may not always be the correct * thing to do here. */ if (cm->active_ref_idx[0] != INT_MAX) get_frame_ref_buffer(cm, 0)->corrupted = 1; if (cm->fb_idx_ref_cnt[cm->new_fb_idx] > 0) cm->fb_idx_ref_cnt[cm->new_fb_idx]--; return -1; } cm->error.setjmp = 1; retcode = vp9_decode_frame(pbi, psource); if (retcode < 0) { cm->error.error_code = VPX_CODEC_ERROR; cm->error.setjmp = 0; if (cm->fb_idx_ref_cnt[cm->new_fb_idx] > 0) cm->fb_idx_ref_cnt[cm->new_fb_idx]--; return retcode; } swap_frame_buffers(pbi); #if WRITE_RECON_BUFFER == 2 if (cm->show_frame) write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame); else write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame + 1000); #endif if (!pbi->do_loopfilter_inline) { vp9_loop_filter_frame(cm, &pbi->mb, pbi->common.lf.filter_level, 0, 0); } #if WRITE_RECON_BUFFER == 2 if (cm->show_frame) write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame + 2000); else write_dx_frame_to_file(cm->frame_to_show, cm->current_video_frame + 3000); #endif vp9_extend_frame_inner_borders(cm->frame_to_show, cm->subsampling_x, cm->subsampling_y); #if WRITE_RECON_BUFFER == 1 if (cm->show_frame) recon_write_yuv_frame("recon.yuv", cm->frame_to_show, cm->width, cm->height); #endif vp9_clear_system_state(); cm->last_show_frame = cm->show_frame; if (cm->show_frame) { // current mip will be the prev_mip for the next frame MODE_INFO *temp = cm->prev_mip; MODE_INFO **temp2 = cm->prev_mi_grid_base; cm->prev_mip = cm->mip; cm->mip = temp; cm->prev_mi_grid_base = cm->mi_grid_base; cm->mi_grid_base = temp2; // update the upper left visible macroblock ptrs cm->mi = cm->mip + cm->mode_info_stride + 1; cm->prev_mi = cm->prev_mip + cm->mode_info_stride + 1; cm->mi_grid_visible = cm->mi_grid_base + cm->mode_info_stride + 1; cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mode_info_stride + 1; pbi->mb.mi_8x8 = cm->mi_grid_visible; pbi->mb.mi_8x8[0] = cm->mi; cm->current_video_frame++; } pbi->ready_for_new_data = 0; pbi->last_time_stamp = time_stamp; pbi->source_sz = 0; cm->error.setjmp = 0; return retcode; }
// Setup cyclic background refresh: set delta q and segmentation map. void vp9_cyclic_refresh_setup(VP9_COMP *const cpi) { VP9_COMMON *const cm = &cpi->common; const RATE_CONTROL *const rc = &cpi->rc; CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; struct segmentation *const seg = &cm->seg; unsigned char *const seg_map = cpi->segmentation_map; const int apply_cyclic_refresh = apply_cyclic_refresh_bitrate(cm, rc); // Don't apply refresh on key frame or enhancement layer frames. if (!apply_cyclic_refresh || (cm->frame_type == KEY_FRAME) || (cpi->svc.temporal_layer_id > 0)) { // Set segmentation map to 0 and disable. vpx_memset(seg_map, 0, cm->mi_rows * cm->mi_cols); vp9_disable_segmentation(&cm->seg); if (cm->frame_type == KEY_FRAME) cr->sb_index = 0; return; } else { int qindex_delta = 0; int i, block_count, bl_index, sb_rows, sb_cols, sbs_in_frame; int xmis, ymis, x, y, qindex2; // Rate target ratio to set q delta. const float rate_ratio_qdelta = 2.0; const double q = vp9_convert_qindex_to_q(cm->base_qindex); vp9_clear_system_state(); // Some of these parameters may be set via codec-control function later. cr->max_sbs_perframe = 10; cr->max_qdelta_perc = 50; cr->min_block_size = BLOCK_8X8; cr->time_for_refresh = 1; // Set rate threshold to some fraction of target (and scaled by 256). cr->thresh_rate_sb = (rc->sb64_target_rate * 256) >> 2; // Distortion threshold, quadratic in Q, scale factor to be adjusted. cr->thresh_dist_sb = 8 * (int)(q * q); if (cpi->sf.use_nonrd_pick_mode) { // May want to be more conservative with thresholds in non-rd mode for now // as rate/distortion are derived from model based on prediction residual. cr->thresh_rate_sb = (rc->sb64_target_rate * 256) >> 3; cr->thresh_dist_sb = 4 * (int)(q * q); } cr->num_seg_blocks = 0; // Set up segmentation. // Clear down the segment map. vpx_memset(seg_map, 0, cm->mi_rows * cm->mi_cols); vp9_enable_segmentation(&cm->seg); vp9_clearall_segfeatures(seg); // Select delta coding method. seg->abs_delta = SEGMENT_DELTADATA; // Note: setting temporal_update has no effect, as the seg-map coding method // (temporal or spatial) is determined in vp9_choose_segmap_coding_method(), // based on the coding cost of each method. For error_resilient mode on the // last_frame_seg_map is set to 0, so if temporal coding is used, it is // relative to 0 previous map. // seg->temporal_update = 0; // Segment 0 "Q" feature is disabled so it defaults to the baseline Q. vp9_disable_segfeature(seg, 0, SEG_LVL_ALT_Q); // Use segment 1 for in-frame Q adjustment. vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q); // Set the q delta for segment 1. qindex_delta = vp9_compute_qdelta_by_rate(rc, cm->frame_type, cm->base_qindex, rate_ratio_qdelta); // TODO(marpan): Incorporate the actual-vs-target rate over/undershoot from // previous encoded frame. if (-qindex_delta > cr->max_qdelta_perc * cm->base_qindex / 100) qindex_delta = -cr->max_qdelta_perc * cm->base_qindex / 100; // Compute rd-mult for segment 1. qindex2 = clamp(cm->base_qindex + cm->y_dc_delta_q + qindex_delta, 0, MAXQ); cr->rdmult = vp9_compute_rd_mult(cpi, qindex2); vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qindex_delta); sb_cols = (cm->mi_cols + MI_BLOCK_SIZE - 1) / MI_BLOCK_SIZE; sb_rows = (cm->mi_rows + MI_BLOCK_SIZE - 1) / MI_BLOCK_SIZE; sbs_in_frame = sb_cols * sb_rows; // Number of target superblocks to get the q delta (segment 1). block_count = cr->max_sbs_perframe * sbs_in_frame / 100; // Set the segmentation map: cycle through the superblocks, starting at // cr->mb_index, and stopping when either block_count blocks have been found // to be refreshed, or we have passed through whole frame. assert(cr->sb_index < sbs_in_frame); i = cr->sb_index; do { int sum_map = 0; // Get the mi_row/mi_col corresponding to superblock index i. int sb_row_index = (i / sb_cols); int sb_col_index = i - sb_row_index * sb_cols; int mi_row = sb_row_index * MI_BLOCK_SIZE; int mi_col = sb_col_index * MI_BLOCK_SIZE; assert(mi_row >= 0 && mi_row < cm->mi_rows); assert(mi_col >= 0 && mi_col < cm->mi_cols); bl_index = mi_row * cm->mi_cols + mi_col; // Loop through all 8x8 blocks in superblock and update map. xmis = MIN(cm->mi_cols - mi_col, num_8x8_blocks_wide_lookup[BLOCK_64X64]); ymis = MIN(cm->mi_rows - mi_row, num_8x8_blocks_high_lookup[BLOCK_64X64]); for (y = 0; y < ymis; y++) { for (x = 0; x < xmis; x++) { const int bl_index2 = bl_index + y * cm->mi_cols + x; // If the block is as a candidate for clean up then mark it // for possible boost/refresh (segment 1). The segment id may get // reset to 0 later if block gets coded anything other than ZEROMV. if (cr->map[bl_index2] == 0) { seg_map[bl_index2] = 1; sum_map++; } else if (cr->map[bl_index2] < 0) { cr->map[bl_index2]++; } } } // Enforce constant segment over superblock. // If segment is partial over superblock, reset to either all 1 or 0. if (sum_map > 0 && sum_map < xmis * ymis) { const int new_value = (sum_map >= xmis * ymis / 2); for (y = 0; y < ymis; y++) for (x = 0; x < xmis; x++) seg_map[bl_index + y * cm->mi_cols + x] = new_value; } i++; if (i == sbs_in_frame) { i = 0; } if (sum_map >= xmis * ymis /2) block_count--; } while (block_count && i != cr->sb_index); cr->sb_index = i; }