/* Builds a literal prefix code into "depths" and "bits" based on the statistics
of the "input" string and stores it into the bit stream.
Note that the prefix code here is built from the pre-LZ77 input, therefore
we can only approximate the statistics of the actual literal stream.
Moreover, for long inputs we build a histogram from a sample of the input
and thus have to assign a non-zero depth for each literal.
Returns estimated compression ratio millibytes/char for encoding given input
with generated code. */
static size_t BuildAndStoreLiteralPrefixCode(MemoryManager* m,
const uint8_t* input,
const size_t input_size,
uint8_t depths[256],
uint16_t bits[256],
size_t* storage_ix,
uint8_t* storage) {
uint32_t histogram[256] = { 0 };
size_t histogram_total;
size_t i;
if (input_size < (1 << 15)) {
for (i = 0; i < input_size; ++i) {
++histogram[input[i]];
}
histogram_total = input_size;
for (i = 0; i < 256; ++i) {
/* We weigh the first 11 samples with weight 3 to account for the
balancing effect of the LZ77 phase on the histogram. */
const uint32_t adjust = 2 * BROTLI_MIN(uint32_t, histogram[i], 11u);
histogram[i] += adjust;
histogram_total += adjust;
}
} else {
static const size_t kSampleRate = 29;
for (i = 0; i < input_size; i += kSampleRate) {
++histogram[input[i]];
}
histogram_total = (input_size + kSampleRate - 1) / kSampleRate;
for (i = 0; i < 256; ++i) {
/* We add 1 to each population count to avoid 0 bit depths (since this is
only a sample and we don't know if the symbol appears or not), and we
weigh the first 11 samples with weight 3 to account for the balancing
effect of the LZ77 phase on the histogram (more frequent symbols are
more likely to be in backward references instead as literals). */
const uint32_t adjust = 1 + 2 * BROTLI_MIN(uint32_t, histogram[i], 11u);
histogram[i] += adjust;
histogram_total += adjust;
}
}
BrotliBuildAndStoreHuffmanTreeFast(m, histogram, histogram_total,
/* max_bits = */ 8,
depths, bits, storage_ix, storage);
if (BROTLI_IS_OOM(m)) return 0;
{
size_t literal_ratio = 0;
for (i = 0; i < 256; ++i) {
if (histogram[i]) literal_ratio += histogram[i] * depths[i];
}
/* Estimated encoding ratio, millibytes per symbol. */
return (literal_ratio * 125) / histogram_total;
}
}
static size_t UTF8Position(size_t last, size_t c, size_t clamp) {
if (c < 128) {
return 0; /* Next one is the 'Byte 1' again. */
} else if (c >= 192) { /* Next one is the 'Byte 2' of utf-8 encoding. */
return BROTLI_MIN(size_t, 1, clamp);
} else {
/* Let's decide over the last byte if this ends the sequence. */
if (last < 0xe0) {
return 0; /* Completed two or three byte coding. */
} else { /* Next one is the 'Byte 3' of utf-8 encoding. */
return BROTLI_MIN(size_t, 2, clamp);
}
}
}
void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost) {
if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) {
EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost);
return;
} else {
size_t histogram[256] = { 0 };
size_t window_half = 2000;
size_t in_window = BROTLI_MIN(size_t, window_half, len);
/* Bootstrap histogram. */
size_t i;
for (i = 0; i < in_window; ++i) {
++histogram[data[(pos + i) & mask]];
}
/* Compute bit costs with sliding window. */
for (i = 0; i < len; ++i) {
size_t histo;
if (i >= window_half) {
/* Remove a byte in the past. */
--histogram[data[(pos + i - window_half) & mask]];
--in_window;
}
if (i + window_half < len) {
/* Add a byte in the future. */
++histogram[data[(pos + i + window_half) & mask]];
++in_window;
}
histo = histogram[data[(pos + i) & mask]];
if (histo == 0) {
histo = 1;
}
{
double lit_cost = FastLog2(in_window) - FastLog2(histo);
lit_cost += 0.029;
if (lit_cost < 1.0) {
lit_cost *= 0.5;
lit_cost += 0.5;
}
cost[i] = (float)lit_cost;
}
}
}
}
static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost) {
/* max_utf8 is 0 (normal ASCII single byte modeling),
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
size_t histogram[3][256] = { { 0 } };
size_t window_half = 495;
size_t in_window = BROTLI_MIN(size_t, window_half, len);
size_t in_window_utf8[3] = { 0 };
size_t i;
{ /* Bootstrap histograms. */
size_t last_c = 0;
size_t utf8_pos = 0;
for (i = 0; i < in_window; ++i) {
size_t c = data[(pos + i) & mask];
++histogram[utf8_pos][c];
++in_window_utf8[utf8_pos];
utf8_pos = UTF8Position(last_c, c, max_utf8);
last_c = c;
}
}
/* Compute bit costs with sliding window. */
for (i = 0; i < len; ++i) {
if (i >= window_half) {
/* Remove a byte in the past. */
size_t c =
i < window_half + 1 ? 0 : data[(pos + i - window_half - 1) & mask];
size_t last_c =
i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask];
size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
--histogram[utf8_pos2][data[(pos + i - window_half) & mask]];
--in_window_utf8[utf8_pos2];
}
if (i + window_half < len) {
/* Add a byte in the future. */
size_t c = data[(pos + i + window_half - 1) & mask];
size_t last_c = data[(pos + i + window_half - 2) & mask];
size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8);
++histogram[utf8_pos2][data[(pos + i + window_half) & mask]];
++in_window_utf8[utf8_pos2];
}
{
size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask];
size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask];
size_t utf8_pos = UTF8Position(last_c, c, max_utf8);
size_t masked_pos = (pos + i) & mask;
size_t histo = histogram[utf8_pos][data[masked_pos]];
double lit_cost;
if (histo == 0) {
histo = 1;
}
lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo);
lit_cost += 0.02905;
if (lit_cost < 1.0) {
lit_cost *= 0.5;
lit_cost += 0.5;
}
/* Make the first bytes more expensive -- seems to help, not sure why.
Perhaps because the entropy source is changing its properties
rapidly in the beginning of the file, perhaps because the beginning
of the data is a statistical "anomaly". */
if (i < 2000) {
lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35);
}
cost[i] = (float)lit_cost;
}
}
}
