double ZopfliCalculateBlockSize(const unsigned short* litlens,
const unsigned short* dists,
size_t lstart, size_t lend, int btype) {
size_t ll_counts[288];
size_t d_counts[32];
unsigned ll_lengths[288];
unsigned d_lengths[32];
double result = 3; /*bfinal and btype bits*/
assert(btype == 1 || btype == 2); /* This is not for uncompressed blocks. */
if(btype == 1) {
GetFixedTree(ll_lengths, d_lengths);
} else {
ZopfliLZ77Counts(litlens, dists, lstart, lend, ll_counts, d_counts);
ZopfliCalculateBitLengths(ll_counts, 288, 15, ll_lengths);
ZopfliCalculateBitLengths(d_counts, 32, 15, d_lengths);
PatchDistanceCodesForBuggyDecoders(d_lengths);
result += CalculateTreeSize(ll_lengths, d_lengths, ll_counts, d_counts);
}
result += CalculateBlockSymbolSize(
ll_lengths, d_lengths, litlens, dists, lstart, lend);
return result;
}
/*
Calculates the bit lengths for the symbols for dynamic blocks. Chooses bit
lengths that give the smallest size of tree encoding + encoding of all the
symbols to have smallest output size. This are not necessarily the ideal Huffman
bit lengths.
*/
static void GetDynamicLengths(const ZopfliLZ77Store* lz77,
size_t lstart, size_t lend,
unsigned* ll_lengths, unsigned* d_lengths) {
size_t ll_counts[ZOPFLI_NUM_LL];
size_t d_counts[ZOPFLI_NUM_D];
ZopfliLZ77GetHistogram(lz77, lstart, lend, ll_counts, d_counts);
ll_counts[256] = 1; /* End symbol. */
OptimizeHuffmanForRle(ZOPFLI_NUM_LL, ll_counts);
OptimizeHuffmanForRle(ZOPFLI_NUM_D, d_counts);
ZopfliCalculateBitLengths(ll_counts, ZOPFLI_NUM_LL, 15, ll_lengths);
ZopfliCalculateBitLengths(d_counts, ZOPFLI_NUM_D, 15, d_lengths);
PatchDistanceCodesForBuggyDecoders(d_lengths);
}
/*
Calculates the bit lengths for the symbols for dynamic blocks. Chooses bit
lengths that give the smallest size of tree encoding + encoding of all the
symbols to have smallest output size. This are not necessarily the ideal Huffman
bit lengths.
*/
static void GetDynamicLengths(const unsigned short* litlens,
const unsigned short* dists,
size_t lstart, size_t lend,
unsigned* ll_lengths, unsigned* d_lengths) {
size_t ll_counts[288];
size_t d_counts[32];
ZopfliLZ77Counts(litlens, dists, lstart, lend, ll_counts, d_counts);
OptimizeHuffmanForRle(288, ll_counts);
OptimizeHuffmanForRle(32, d_counts);
ZopfliCalculateBitLengths(ll_counts, 288, 15, ll_lengths);
ZopfliCalculateBitLengths(d_counts, 32, 15, d_lengths);
PatchDistanceCodesForBuggyDecoders(d_lengths);
}
/*
Tries out OptimizeHuffmanForRle for this block, if the result is smaller,
uses it, otherwise keeps the original. Returns size of encoded tree and data in
bits, not including the 3-bit block header.
*/
static double TryOptimizeHuffmanForRle(
const ZopfliLZ77Store* lz77, size_t lstart, size_t lend,
const size_t* ll_counts, const size_t* d_counts,
unsigned* ll_lengths, unsigned* d_lengths) {
size_t ll_counts2[ZOPFLI_NUM_LL];
size_t d_counts2[ZOPFLI_NUM_D];
unsigned ll_lengths2[ZOPFLI_NUM_LL];
unsigned d_lengths2[ZOPFLI_NUM_D];
double treesize;
double datasize;
double treesize2;
double datasize2;
treesize = CalculateTreeSize(ll_lengths, d_lengths);
datasize = CalculateBlockSymbolSizeGivenCounts(ll_counts, d_counts,
ll_lengths, d_lengths, lz77, lstart, lend);
memcpy(ll_counts2, ll_counts, sizeof(ll_counts2));
memcpy(d_counts2, d_counts, sizeof(d_counts2));
OptimizeHuffmanForRle(ZOPFLI_NUM_LL, ll_counts2);
OptimizeHuffmanForRle(ZOPFLI_NUM_D, d_counts2);
ZopfliCalculateBitLengths(ll_counts2, ZOPFLI_NUM_LL, 15, ll_lengths2);
ZopfliCalculateBitLengths(d_counts2, ZOPFLI_NUM_D, 15, d_lengths2);
PatchDistanceCodesForBuggyDecoders(d_lengths2);
treesize2 = CalculateTreeSize(ll_lengths2, d_lengths2);
datasize2 = CalculateBlockSymbolSizeGivenCounts(ll_counts, d_counts,
ll_lengths2, d_lengths2, lz77, lstart, lend);
if (treesize2 + datasize2 < treesize + datasize) {
memcpy(ll_lengths, ll_lengths2, sizeof(ll_lengths2));
memcpy(d_lengths, d_lengths2, sizeof(d_lengths2));
return treesize2 + datasize2;
}
return treesize + datasize;
}
/*
Encodes the Huffman tree and returns how many bits its encoding takes. If out
is a null pointer, only returns the size and runs faster.
*/
static size_t EncodeTree(const unsigned* ll_lengths,
const unsigned* d_lengths,
int use_16, int use_17, int use_18,
unsigned char* bp,
unsigned char** out, size_t* outsize) {
unsigned lld_total; /* Total amount of literal, length, distance codes. */
/* Runlength encoded version of lengths of litlen and dist trees. */
unsigned* rle = 0;
unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */
size_t rle_size = 0; /* Size of rle array. */
size_t rle_bits_size = 0; /* Should have same value as rle_size. */
unsigned hlit = 29; /* 286 - 257 */
unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/
unsigned hclen;
unsigned hlit2;
size_t i, j;
size_t clcounts[19];
unsigned clcl[19]; /* Code length code lengths. */
unsigned clsymbols[19];
/* The order in which code length code lengths are encoded as per deflate. */
static const unsigned order[19] = {
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
};
int size_only = !out;
size_t result_size = 0;
for(i = 0; i < 19; i++) clcounts[i] = 0;
/* Trim zeros. */
while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--;
while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--;
hlit2 = hlit + 257;
lld_total = hlit2 + hdist + 1;
for (i = 0; i < lld_total; i++) {
/* This is an encoding of a huffman tree, so now the length is a symbol */
unsigned char symbol = i < hlit2 ? ll_lengths[i] : d_lengths[i - hlit2];
unsigned count = 1;
if(use_16 || (symbol == 0 && (use_17 || use_18))) {
for (j = i + 1; j < lld_total && symbol ==
(j < hlit2 ? ll_lengths[j] : d_lengths[j - hlit2]); j++) {
count++;
}
}
i += count - 1;
/* Repetitions of zeroes */
if (symbol == 0 && count >= 3) {
if (use_18) {
while (count >= 11) {
unsigned count2 = count > 138 ? 138 : count;
if (!size_only) {
ZOPFLI_APPEND_DATA(18, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count2 - 11, &rle_bits, &rle_bits_size);
}
clcounts[18]++;
count -= count2;
}
}
if (use_17) {
while (count >= 3) {
unsigned count2 = count > 10 ? 10 : count;
if (!size_only) {
ZOPFLI_APPEND_DATA(17, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size);
}
clcounts[17]++;
count -= count2;
}
}
}
/* Repetitions of any symbol */
if (use_16 && count >= 4) {
count--; /* Since the first one is hardcoded. */
clcounts[symbol]++;
if (!size_only) {
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
}
while (count >= 3) {
unsigned count2 = count > 6 ? 6 : count;
if (!size_only) {
ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size);
}
clcounts[16]++;
count -= count2;
}
}
/* No or insufficient repetition */
clcounts[symbol] += count;
while (count > 0) {
if (!size_only) {
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
}
count--;
}
}
ZopfliCalculateBitLengths(clcounts, 19, 7, clcl);
if (!size_only) ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols);
hclen = 15;
/* Trim zeros. */
while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--;
if (!size_only) {
AddBits(hlit, 5, bp, out, outsize);
AddBits(hdist, 5, bp, out, outsize);
AddBits(hclen, 4, bp, out, outsize);
for (i = 0; i < hclen + 4; i++) {
AddBits(clcl[order[i]], 3, bp, out, outsize);
}
for (i = 0; i < rle_size; i++) {
unsigned symbol = clsymbols[rle[i]];
AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize);
/* Extra bits. */
if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize);
else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize);
else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize);
}
}
result_size += 14; /* hlit, hdist, hclen bits */
result_size += (hclen + 4) * 3; /* clcl bits */
for(i = 0; i < 19; i++) {
result_size += clcl[i] * clcounts[i];
}
/* Extra bits. */
result_size += clcounts[16] * 2;
result_size += clcounts[17] * 3;
result_size += clcounts[18] * 7;
/* Note: in case of "size_only" these are null pointers so no effect. */
free(rle);
free(rle_bits);
return result_size;
}
static void AddDynamicTree(const unsigned* ll_lengths,
const unsigned* d_lengths,
unsigned char* bp,
unsigned char** out, size_t* outsize) {
unsigned* lld_lengths = 0; /* All litlen and dist lengthts with ending zeros
trimmed together in one array. */
unsigned lld_total; /* Size of lld_lengths. */
unsigned* rle = 0; /* Runlength encoded version of lengths of litlen and dist
trees. */
unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */
size_t rle_size = 0; /* Size of rle array. */
size_t rle_bits_size = 0; /* Should have same value as rle_size. */
unsigned hlit = 29; /* 286 - 257 */
unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/
unsigned hclen;
size_t i, j;
size_t clcounts[19];
unsigned clcl[19]; /* Code length code lengths. */
unsigned clsymbols[19];
/* The order in which code length code lengths are encoded as per deflate. */
unsigned order[19] = {
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
};
/* Trim zeros. */
while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--;
while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--;
lld_total = hlit + 257 + hdist + 1;
lld_lengths = (unsigned*)malloc(sizeof(*lld_lengths) * lld_total);
if (!lld_lengths) exit(-1); /* Allocation failed. */
for (i = 0; i < lld_total; i++) {
lld_lengths[i] = i < 257 + hlit
? ll_lengths[i] : d_lengths[i - 257 - hlit];
assert(lld_lengths[i] < 16);
}
for (i = 0; i < lld_total; i++) {
size_t count = 0;
for (j = i; j < lld_total && lld_lengths[i] == lld_lengths[j]; j++) {
count++;
}
if (count >= 4 || (count >= 3 && lld_lengths[i] == 0)) {
if (lld_lengths[i] == 0) {
if (count > 10) {
if (count > 138) count = 138;
ZOPFLI_APPEND_DATA(18, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count - 11, &rle_bits, &rle_bits_size);
} else {
ZOPFLI_APPEND_DATA(17, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count - 3, &rle_bits, &rle_bits_size);
}
} else {
unsigned repeat = count - 1; /* Since the first one is hardcoded. */
ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
while (repeat >= 6) {
ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
ZOPFLI_APPEND_DATA(6 - 3, &rle_bits, &rle_bits_size);
repeat -= 6;
}
if (repeat >= 3) {
ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
ZOPFLI_APPEND_DATA(3 - 3, &rle_bits, &rle_bits_size);
repeat -= 3;
}
while (repeat != 0) {
ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
repeat--;
}
}
i += count - 1;
} else {
ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
}
assert(rle[rle_size - 1] <= 18);
}
for (i = 0; i < 19; i++) {
clcounts[i] = 0;
}
for (i = 0; i < rle_size; i++) {
clcounts[rle[i]]++;
}
ZopfliCalculateBitLengths(clcounts, 19, 7, clcl);
ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols);
hclen = 15;
/* Trim zeros. */
while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--;
AddBits(hlit, 5, bp, out, outsize);
AddBits(hdist, 5, bp, out, outsize);
AddBits(hclen, 4, bp, out, outsize);
for (i = 0; i < hclen + 4; i++) {
AddBits(clcl[order[i]], 3, bp, out, outsize);
}
for (i = 0; i < rle_size; i++) {
unsigned symbol = clsymbols[rle[i]];
AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize);
/* Extra bits. */
if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize);
else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize);
else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize);
}
free(lld_lengths);
free(rle);
free(rle_bits);
}
