/**
* \brief RDO function to calculate cost for intra
* \returns cost to code pred block
** Only for luma
*/
uint32_t rdo_cost_intra(encoder_state * const encoder_state, pixel *pred, pixel *orig_block, int width, int8_t mode, int tr_depth)
{
const encoder_control * const encoder = encoder_state->encoder_control;
coefficient pre_quant_coeff[LCU_WIDTH*LCU_WIDTH>>2];
int16_t block[LCU_WIDTH*LCU_WIDTH>>2];
int16_t temp_block[LCU_WIDTH*LCU_WIDTH>>2];
coefficient temp_coeff[LCU_WIDTH*LCU_WIDTH>>2];
int8_t luma_scan_mode = SCAN_DIAG;
int i = 0,x,y;
for (y = 0; y < width; y++) {
for (x = 0; x < width; x++) {
block[i++] = orig_block[x + y*width]- pred[x + y*width];
}
}
// Scan mode is diagonal, except for 4x4 and 8x8, where:
// - angular 6-14 = vertical
// - angular 22-30 = horizontal
if (width <= 8) {
if (mode >= 6 && mode <= 14) {
luma_scan_mode = SCAN_VER;
} else if (mode >= 22 && mode <= 30) {
luma_scan_mode = SCAN_HOR;
}
}
transform2d(encoder, block,pre_quant_coeff,width,0);
if(encoder->rdoq_enable) {
rdoq(encoder_state, pre_quant_coeff, temp_coeff, width, width, 0, luma_scan_mode, CU_INTRA, tr_depth);
} else {
quant(encoder_state, pre_quant_coeff, temp_coeff, width, width, 0, luma_scan_mode, CU_INTRA);
}
dequant(encoder_state, temp_coeff, pre_quant_coeff, width, width, 0, CU_INTRA);
itransform2d(encoder, temp_block,pre_quant_coeff,width,0);
unsigned ssd = 0;
// SSD between original and reconstructed
for (i = 0; i < width*width; i++) {
//int diff = temp_block[i]-block[i];
int diff = orig_block[i] - CLIP(0, 255, pred[i] + temp_block[i]);
ssd += diff*diff;
}
double coeff_bits = 0;
// Simple RDO
if(encoder->rdo == 1) {
// SSD between reconstruction and original + sum of coeffs
int coeff_abs = 0;
for (i = 0; i < width*width; i++) {
coeff_abs += abs((int)temp_coeff[i]);
}
coeff_bits += 1 + 1.5 * coeff_abs;
// Full RDO
} else if(encoder->rdo >= 2) {
coeff_bits = get_coeff_cost(encoder_state, temp_coeff, width, 0, luma_scan_mode);
}
return (uint32_t)(0.5 + ssd + coeff_bits * encoder_state->global->cur_lambda_cost);
}
/**
* \brief RDO function to calculate cost for intra
* \returns cost to code pred block
** Only for luma
*/
uint32_t rdo_cost_intra(encoder_state * const encoder_state, pixel *pred, pixel *orig_block, int width, int8_t mode)
{
const encoder_control * const encoder = encoder_state->encoder_control;
coefficient pre_quant_coeff[LCU_WIDTH*LCU_WIDTH>>2];
int16_t block[LCU_WIDTH*LCU_WIDTH>>2];
int16_t temp_block[LCU_WIDTH*LCU_WIDTH>>2];
coefficient temp_coeff[LCU_WIDTH*LCU_WIDTH>>2];
uint32_t ac_sum;
uint32_t cost = 0;
uint32_t coeffcost = 0;
int8_t luma_scan_mode = SCAN_DIAG;
int i = 0,x,y;
for (y = 0; y < width; y++) {
for (x = 0; x < width; x++) {
block[i++] = orig_block[x + y*width]- pred[x + y*width];
}
}
// Scan mode is diagonal, except for 4x4 and 8x8, where:
// - angular 6-14 = vertical
// - angular 22-30 = horizontal
if (width <= 8) {
if (mode >= 6 && mode <= 14) {
luma_scan_mode = SCAN_VER;
} else if (mode >= 22 && mode <= 30) {
luma_scan_mode = SCAN_HOR;
}
}
transform2d(encoder, block,pre_quant_coeff,width,0);
if(encoder->rdoq_enable) {
rdoq(encoder_state, pre_quant_coeff, temp_coeff, width, width, &ac_sum, 0, luma_scan_mode, CU_INTRA,0);
} else {
quant(encoder_state, pre_quant_coeff, temp_coeff, width, width, &ac_sum, 0, luma_scan_mode, CU_INTRA);
}
dequant(encoder_state, temp_coeff, pre_quant_coeff, width, width, 0, CU_INTRA);
itransform2d(encoder, temp_block,pre_quant_coeff,width,0);
// SSD between original and reconstructed
for (i = 0; i < width*width; i++) {
int diff = temp_block[i]-block[i];
cost += diff*diff;
}
// Simple RDO
if(encoder->rdo == 1) {
// SSD between reconstruction and original + sum of coeffs
for (i = 0; i < width*width; i++) {
coeffcost += abs((int)temp_coeff[i]);
}
cost += (1 + coeffcost + (coeffcost>>1))*((int)encoder_state->global->cur_lambda_cost+0.5);
// Full RDO
} else if(encoder->rdo == 2) {