static int GetResidualCost(int ctx0, const VP8Residual* const res) {
int n = res->first;
// should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1
const int p0 = res->prob[n][ctx0][0];
const uint16_t* t = res->cost[n][ctx0];
// bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0
// (as required by the syntax). For ctx0 == 0, we need to add it here or it'll
// be missing during the loop.
int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0;
if (res->last < 0) {
return VP8BitCost(0, p0);
}
for (; n < res->last; ++n) {
const int v = abs(res->coeffs[n]);
const int b = VP8EncBands[n + 1];
const int ctx = (v >= 2) ? 2 : v;
cost += VP8LevelCost(t, v);
t = res->cost[b][ctx];
}
// Last coefficient is always non-zero
{
const int v = abs(res->coeffs[n]);
assert(v != 0);
cost += VP8LevelCost(t, v);
if (n < 15) {
const int b = VP8EncBands[n + 1];
const int ctx = (v == 1) ? 1 : 2;
const int last_p0 = res->prob[b][ctx][0];
cost += VP8BitCost(0, last_p0);
}
}
return cost;
}
static int TrellisQuantizeBlock(const VP8EncIterator* const it,
int16_t in[16], int16_t out[16],
int ctx0, int coeff_type,
const VP8Matrix* const mtx,
int lambda) {
ProbaArray* const last_costs = it->enc_->proba_.coeffs_[coeff_type];
CostArray* const costs = it->enc_->proba_.level_cost_[coeff_type];
const int first = (coeff_type == 0) ? 1 : 0;
Node nodes[17][NUM_NODES];
int best_path[3] = {-1, -1, -1}; // store best-last/best-level/best-previous
score_t best_score;
int best_node;
int last = first - 1;
int n, m, p, nz;
{
score_t cost;
score_t max_error;
const int thresh = mtx->q_[1] * mtx->q_[1] / 4;
const int last_proba = last_costs[VP8EncBands[first]][ctx0][0];
// compute maximal distortion.
max_error = 0;
for (n = first; n < 16; ++n) {
const int j = kZigzag[n];
const int err = in[j] * in[j];
max_error += kWeightTrellis[j] * err;
if (err > thresh) last = n;
}
// we don't need to go inspect up to n = 16 coeffs. We can just go up
// to last + 1 (inclusive) without losing much.
if (last < 15) ++last;
// compute 'skip' score. This is the max score one can do.
cost = VP8BitCost(0, last_proba);
best_score = RDScoreTrellis(lambda, cost, max_error);
// initialize source node.
n = first - 1;
for (m = -MIN_DELTA; m <= MAX_DELTA; ++m) {
NODE(n, m).cost = 0;
NODE(n, m).error = max_error;
NODE(n, m).ctx = ctx0;
}
}
// traverse trellis.
for (n = first; n <= last; ++n) {
const int j = kZigzag[n];
const int Q = mtx->q_[j];
const int iQ = mtx->iq_[j];
const int B = BIAS(0x00); // neutral bias
// note: it's important to take sign of the _original_ coeff,
// so we don't have to consider level < 0 afterward.
const int sign = (in[j] < 0);
int coeff0 = (sign ? -in[j] : in[j]) + mtx->sharpen_[j];
int level0;
if (coeff0 > 2047) coeff0 = 2047;
level0 = QUANTDIV(coeff0, iQ, B);
// test all alternate level values around level0.
for (m = -MIN_DELTA; m <= MAX_DELTA; ++m) {
Node* const cur = &NODE(n, m);
int delta_error, new_error;
score_t cur_score = MAX_COST;
int level = level0 + m;
int last_proba;
cur->sign = sign;
cur->level = level;
cur->ctx = (level == 0) ? 0 : (level == 1) ? 1 : 2;
if (level >= 2048 || level < 0) { // node is dead?
cur->cost = MAX_COST;
continue;
}
last_proba = last_costs[VP8EncBands[n + 1]][cur->ctx][0];
// Compute delta_error = how much coding this level will
// subtract as distortion to max_error
new_error = coeff0 - level * Q;
delta_error =
kWeightTrellis[j] * (coeff0 * coeff0 - new_error * new_error);
// Inspect all possible non-dead predecessors. Retain only the best one.
for (p = -MIN_DELTA; p <= MAX_DELTA; ++p) {
const Node* const prev = &NODE(n - 1, p);
const int prev_ctx = prev->ctx;
const uint16_t* const tcost = costs[VP8EncBands[n]][prev_ctx];
const score_t total_error = prev->error - delta_error;
score_t cost, base_cost, score;
if (prev->cost >= MAX_COST) { // dead node?
continue;
}
// Base cost of both terminal/non-terminal
base_cost = prev->cost + VP8LevelCost(tcost, level);
// Examine node assuming it's a non-terminal one.
cost = base_cost;
if (level && n < 15) {
cost += VP8BitCost(1, last_proba);
}
score = RDScoreTrellis(lambda, cost, total_error);
if (score < cur_score) {
cur_score = score;
cur->cost = cost;
cur->error = total_error;
cur->prev = p;
}
// Now, record best terminal node (and thus best entry in the graph).
if (level) {
cost = base_cost;
if (n < 15) cost += VP8BitCost(0, last_proba);
score = RDScoreTrellis(lambda, cost, total_error);
if (score < best_score) {
best_score = score;
best_path[0] = n; // best eob position
best_path[1] = m; // best level
best_path[2] = p; // best predecessor
}
}
}
}
}
// Fresh start
memset(in + first, 0, (16 - first) * sizeof(*in));
memset(out + first, 0, (16 - first) * sizeof(*out));
if (best_path[0] == -1) {
return 0; // skip!
}
// Unwind the best path.
// Note: best-prev on terminal node is not necessarily equal to the
// best_prev for non-terminal. So we patch best_path[2] in.
n = best_path[0];
best_node = best_path[1];
NODE(n, best_node).prev = best_path[2]; // force best-prev for terminal
nz = 0;
for (; n >= first; --n) {
const Node* const node = &NODE(n, best_node);
const int j = kZigzag[n];
out[n] = node->sign ? -node->level : node->level;
nz |= (node->level != 0);
in[j] = out[n] * mtx->q_[j];
best_node = node->prev;
}
return nz;
}
