static void PrintBlockSplitPoints(const unsigned short* litlens,
const unsigned short* dists,
size_t llsize, const size_t* lz77splitpoints,
size_t nlz77points) {
size_t* splitpoints = 0;
size_t npoints = 0;
size_t i;
size_t pos = 0;
/* The input is given as lz77 indices, but we want to see the uncompressed
index values. */
if (nlz77points > 0) {
for (i = 0; i < llsize; i++) {
size_t length = dists[i] == 0 ? 1 : litlens[i];
if (lz77splitpoints[npoints] == i) {
ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints);
if (npoints == nlz77points) break;
}
pos += length;
}
}
assert(npoints == nlz77points);
PrintPoints(&splitpoints,&npoints,0);
free(splitpoints);
}
/*
Prints the block split points as decimal and hex values in the terminal.
*/
static void PrintBlockSplitPoints(const ZopfliLZ77Store* lz77,
const size_t* lz77splitpoints,
size_t nlz77points) {
size_t* splitpoints = 0;
size_t npoints = 0;
size_t i;
/* The input is given as lz77 indices, but we want to see the uncompressed
index values. */
size_t pos = 0;
if (nlz77points > 0) {
for (i = 0; i < lz77->size; i++) {
size_t length = lz77->dists[i] == 0 ? 1 : lz77->litlens[i];
if (lz77splitpoints[npoints] == i) {
ZOPFLI_APPEND_DATA(pos, &splitpoints, &npoints);
if (npoints == nlz77points) break;
}
pos += length;
}
}
assert(npoints == nlz77points);
fprintf(stderr, "block split points: ");
for (i = 0; i < npoints; i++) {
fprintf(stderr, "%d ", (int)splitpoints[i]);
}
fprintf(stderr, "(hex:");
for (i = 0; i < npoints; i++) {
fprintf(stderr, " %x", (int)splitpoints[i]);
}
fprintf(stderr, ")\n");
free(splitpoints);
}
void ZopfliBlockSplitSimple(size_t instart, size_t inend, size_t blocksize,
size_t** splitpoints, size_t* npoints, int verbose) {
size_t i = instart>0? instart : blocksize;
while (i < inend) {
ZOPFLI_APPEND_DATA(i, splitpoints, npoints);
i += blocksize;
}
if(verbose>3) PrintPoints(splitpoints,npoints,0);
if(verbose>2) fprintf(stderr, "Total blocks: %lu \n\n",(unsigned long)(*npoints+1));
}
static void AddBits(unsigned symbol, unsigned length,
unsigned char* bp, unsigned char** out, size_t* outsize) {
/* TODO(lode): make more efficient (add more bits at once). */
unsigned i;
for (i = 0; i < length; i++) {
unsigned bit = (symbol >> i) & 1;
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
(*out)[*outsize - 1] |= bit << *bp;
*bp = (*bp + 1) & 7;
}
}
void ZopfliBlockSplitSimple(const unsigned char* in,
size_t instart, size_t inend,
size_t blocksize,
size_t** splitpoints, size_t* npoints) {
size_t i = instart;
while (i < inend) {
ZOPFLI_APPEND_DATA(i, splitpoints, npoints);
i += blocksize;
}
(void)in;
}
/*
Adds bits, like AddBits, but the order is inverted. The deflate specification
uses both orders in one standard.
*/
static void AddHuffmanBits(unsigned symbol, unsigned length,
unsigned char* bp, unsigned char** out,
size_t* outsize) {
/* TODO(lode): make more efficient (add more bits at once). */
unsigned i;
for (i = 0; i < length; i++) {
unsigned bit = (symbol >> (length - i - 1)) & 1;
if (((*bp) & 7) == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
(*out)[*outsize - 1] |= bit << ((*bp) & 7);
(*bp)++;
}
}
static void AddSorted(size_t value, size_t** out, size_t* outsize) {
size_t i;
ZOPFLI_APPEND_DATA(value, out, outsize);
for (i = 0; i + 1 < *outsize; i++) {
if ((*out)[i] > value) {
size_t j;
for (j = *outsize - 1; j > i; j--) {
(*out)[j] = (*out)[j - 1];
}
(*out)[i] = value;
break;
}
}
}
void ZopfliBlockSplit(const ZopfliOptions* options,
const unsigned char* in, size_t instart, size_t inend,
size_t maxblocks, size_t** splitpoints, size_t* npoints) {
size_t pos = 0;
size_t i;
ZopfliBlockState s;
size_t* lz77splitpoints = 0;
size_t nlz77points = 0;
ZopfliLZ77Store store;
ZopfliHash hash;
ZopfliHash* h = &hash;
ZopfliInitLZ77Store(in, &store);
ZopfliInitBlockState(options, instart, inend, 0, &s);
ZopfliAllocHash(ZOPFLI_WINDOW_SIZE, h);
*npoints = 0;
*splitpoints = 0;
/* Unintuitively, Using a simple LZ77 method here instead of ZopfliLZ77Optimal
results in better blocks. */
ZopfliLZ77Greedy(&s, in, instart, inend, &store, h);
ZopfliBlockSplitLZ77(options,
&store, maxblocks,
&lz77splitpoints, &nlz77points);
/* Convert LZ77 positions to positions in the uncompressed input. */
pos = instart;
if (nlz77points > 0) {
for (i = 0; i < store.size; i++) {
size_t length = store.dists[i] == 0 ? 1 : store.litlens[i];
if (lz77splitpoints[*npoints] == i) {
ZOPFLI_APPEND_DATA(pos, splitpoints, npoints);
if (*npoints == nlz77points) break;
}
pos += length;
}
}
assert(*npoints == nlz77points);
free(lz77splitpoints);
ZopfliCleanBlockState(&s);
ZopfliCleanLZ77Store(&store);
ZopfliCleanHash(h);
}
void ZopfliBlockSplit(const ZopfliOptions* options,
const unsigned char* in, size_t instart, size_t inend,
size_t maxblocks, size_t** splitpoints, size_t* npoints, size_t* startnpoints) {
size_t pos = 0;
size_t i;
ZopfliBlockState s;
size_t* lz77splitpoints = 0;
size_t nlz77points = 0;
ZopfliLZ77Store store;
ZopfliInitLZ77Store(&store);
s.options = options;
s.blockstart = instart;
s.blockend = inend;
#ifdef ZOPFLI_LONGEST_MATCH_CACHE
s.lmc = 0;
#endif
*npoints = 0;
*splitpoints = 0;
/* Unintuitively, Using a simple LZ77 method here instead of ZopfliLZ77Optimal
results in better blocks. */
ZopfliLZ77Greedy(&s, in, instart, inend, &store, options);
ZopfliBlockSplitLZ77(options,
store.litlens, store.dists, store.size, maxblocks,
&lz77splitpoints, &nlz77points, *startnpoints);
/* Convert LZ77 positions to positions in the uncompressed input. */
pos = instart;
if (nlz77points > 0) {
for (i = 0; i < store.size; i++) {
size_t length = store.dists[i] == 0 ? 1 : store.litlens[i];
if (lz77splitpoints[*npoints] == i) {
ZOPFLI_APPEND_DATA(pos, splitpoints, npoints);
if (*npoints == nlz77points) break;
}
pos += length;
}
}
assert(*npoints == nlz77points);
free(lz77splitpoints);
ZopfliCleanLZ77Store(&store);
}
void ZopfliZipCompress(const ZopfliOptions* options,
const unsigned char* in, size_t insize,
unsigned char** out, size_t* outsize,
ZopfliPredefinedSplits* sp,
const ZopfliAdditionalData* moredata) {
static const unsigned char filePKh[10] = { 80, 75, 3, 4, 20, 0, 2, 0, 8, 0};
static const unsigned char CDIRPKh[12] = { 80, 75, 1, 2, 20, 0, 20, 0, 2, 0, 8, 0};
static const unsigned char CDIRPKs[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0};
static const unsigned char EndCDIRPKh[12] = { 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0};
static const unsigned long defTimestamp = 50;
unsigned long crcvalue = CRC(in, insize);
unsigned long i;
char* tempfilename = NULL;
const char* infilename = NULL;
unsigned long fullsize = insize & 0xFFFFFFFFUL;
unsigned long rawdeflsize = 0;
unsigned char bp = 0;
size_t max = 0;
unsigned long cdirsize = 0;
unsigned long cdiroffset;
if(moredata==NULL) {
tempfilename = Zmalloc(9 * sizeof(char*));
sprintf(tempfilename,"%08lx",crcvalue & 0xFFFFFFFFUL);
infilename = tempfilename;
} else {
infilename = moredata->filename;
}
/* File PK STATIC DATA + CM */
for(i=0;i<sizeof(filePKh);++i) ZOPFLI_APPEND_DATA(filePKh[i],out,outsize);
/* MS-DOS TIME */
if(moredata == NULL) {
for(i=0;i<sizeof(defTimestamp);++i) ZOPFLI_APPEND_DATA(defTimestamp >> (i*8) % 256, out, outsize);
} else {
for(i=0;i<4;++i) ZOPFLI_APPEND_DATA((moredata->timestamp >> (i*8)) % 256, out, outsize);
/*
bp = bitpointer, always in range [0, 7].
The outsize is number of necessary bytes to encode the bits.
Given the value of bp and the amount of bytes, the amount of bits represented
is not simply bytesize * 8 + bp because even representing one bit requires a
whole byte. It is: (bp == 0) ? (bytesize * 8) : ((bytesize - 1) * 8 + bp)
*/
static void AddBit(int bit,
unsigned char* bp, unsigned char** out, size_t* outsize) {
if (*bp == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
(*out)[*outsize - 1] |= bit << *bp;
*bp = (*bp + 1) & 7;
}
/*
Encodes the Huffman tree and returns how many bits its encoding takes. If out
is a null pointer, only returns the size and runs faster.
*/
static size_t EncodeTree(const unsigned* ll_lengths,
const unsigned* d_lengths,
int use_16, int use_17, int use_18,
unsigned char* bp,
unsigned char** out, size_t* outsize) {
unsigned lld_total; /* Total amount of literal, length, distance codes. */
/* Runlength encoded version of lengths of litlen and dist trees. */
unsigned* rle = 0;
unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */
size_t rle_size = 0; /* Size of rle array. */
size_t rle_bits_size = 0; /* Should have same value as rle_size. */
unsigned hlit = 29; /* 286 - 257 */
unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/
unsigned hclen;
unsigned hlit2;
size_t i, j;
size_t clcounts[19];
unsigned clcl[19]; /* Code length code lengths. */
unsigned clsymbols[19];
/* The order in which code length code lengths are encoded as per deflate. */
static const unsigned order[19] = {
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
};
int size_only = !out;
size_t result_size = 0;
for(i = 0; i < 19; i++) clcounts[i] = 0;
/* Trim zeros. */
while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--;
while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--;
hlit2 = hlit + 257;
lld_total = hlit2 + hdist + 1;
for (i = 0; i < lld_total; i++) {
/* This is an encoding of a huffman tree, so now the length is a symbol */
unsigned char symbol = i < hlit2 ? ll_lengths[i] : d_lengths[i - hlit2];
unsigned count = 1;
if(use_16 || (symbol == 0 && (use_17 || use_18))) {
for (j = i + 1; j < lld_total && symbol ==
(j < hlit2 ? ll_lengths[j] : d_lengths[j - hlit2]); j++) {
count++;
}
}
i += count - 1;
/* Repetitions of zeroes */
if (symbol == 0 && count >= 3) {
if (use_18) {
while (count >= 11) {
unsigned count2 = count > 138 ? 138 : count;
if (!size_only) {
ZOPFLI_APPEND_DATA(18, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count2 - 11, &rle_bits, &rle_bits_size);
}
clcounts[18]++;
count -= count2;
}
}
if (use_17) {
while (count >= 3) {
unsigned count2 = count > 10 ? 10 : count;
if (!size_only) {
ZOPFLI_APPEND_DATA(17, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size);
}
clcounts[17]++;
count -= count2;
}
}
}
/* Repetitions of any symbol */
if (use_16 && count >= 4) {
count--; /* Since the first one is hardcoded. */
clcounts[symbol]++;
if (!size_only) {
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
}
while (count >= 3) {
unsigned count2 = count > 6 ? 6 : count;
if (!size_only) {
ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count2 - 3, &rle_bits, &rle_bits_size);
}
clcounts[16]++;
count -= count2;
}
}
/* No or insufficient repetition */
clcounts[symbol] += count;
while (count > 0) {
if (!size_only) {
ZOPFLI_APPEND_DATA(symbol, &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
}
count--;
}
}
ZopfliCalculateBitLengths(clcounts, 19, 7, clcl);
if (!size_only) ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols);
hclen = 15;
/* Trim zeros. */
while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--;
if (!size_only) {
AddBits(hlit, 5, bp, out, outsize);
AddBits(hdist, 5, bp, out, outsize);
AddBits(hclen, 4, bp, out, outsize);
for (i = 0; i < hclen + 4; i++) {
AddBits(clcl[order[i]], 3, bp, out, outsize);
}
for (i = 0; i < rle_size; i++) {
unsigned symbol = clsymbols[rle[i]];
AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize);
/* Extra bits. */
if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize);
else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize);
else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize);
}
}
result_size += 14; /* hlit, hdist, hclen bits */
result_size += (hclen + 4) * 3; /* clcl bits */
for(i = 0; i < 19; i++) {
result_size += clcl[i] * clcounts[i];
}
/* Extra bits. */
result_size += clcounts[16] * 2;
result_size += clcounts[17] * 3;
result_size += clcounts[18] * 7;
/* Note: in case of "size_only" these are null pointers so no effect. */
free(rle);
free(rle_bits);
return result_size;
}
static void AddBit(int bit,
unsigned char* bp, unsigned char** out, size_t* outsize) {
if (((*bp) & 7) == 0) ZOPFLI_APPEND_DATA(0, out, outsize);
(*out)[*outsize - 1] |= bit << ((*bp) & 7);
(*bp)++;
}
static void AddDynamicTree(const unsigned* ll_lengths,
const unsigned* d_lengths,
unsigned char* bp,
unsigned char** out, size_t* outsize) {
unsigned* lld_lengths = 0; /* All litlen and dist lengthts with ending zeros
trimmed together in one array. */
unsigned lld_total; /* Size of lld_lengths. */
unsigned* rle = 0; /* Runlength encoded version of lengths of litlen and dist
trees. */
unsigned* rle_bits = 0; /* Extra bits for rle values 16, 17 and 18. */
size_t rle_size = 0; /* Size of rle array. */
size_t rle_bits_size = 0; /* Should have same value as rle_size. */
unsigned hlit = 29; /* 286 - 257 */
unsigned hdist = 29; /* 32 - 1, but gzip does not like hdist > 29.*/
unsigned hclen;
size_t i, j;
size_t clcounts[19];
unsigned clcl[19]; /* Code length code lengths. */
unsigned clsymbols[19];
/* The order in which code length code lengths are encoded as per deflate. */
unsigned order[19] = {
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
};
/* Trim zeros. */
while (hlit > 0 && ll_lengths[257 + hlit - 1] == 0) hlit--;
while (hdist > 0 && d_lengths[1 + hdist - 1] == 0) hdist--;
lld_total = hlit + 257 + hdist + 1;
lld_lengths = (unsigned*)malloc(sizeof(*lld_lengths) * lld_total);
if (!lld_lengths) exit(-1); /* Allocation failed. */
for (i = 0; i < lld_total; i++) {
lld_lengths[i] = i < 257 + hlit
? ll_lengths[i] : d_lengths[i - 257 - hlit];
assert(lld_lengths[i] < 16);
}
for (i = 0; i < lld_total; i++) {
size_t count = 0;
for (j = i; j < lld_total && lld_lengths[i] == lld_lengths[j]; j++) {
count++;
}
if (count >= 4 || (count >= 3 && lld_lengths[i] == 0)) {
if (lld_lengths[i] == 0) {
if (count > 10) {
if (count > 138) count = 138;
ZOPFLI_APPEND_DATA(18, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count - 11, &rle_bits, &rle_bits_size);
} else {
ZOPFLI_APPEND_DATA(17, &rle, &rle_size);
ZOPFLI_APPEND_DATA(count - 3, &rle_bits, &rle_bits_size);
}
} else {
unsigned repeat = count - 1; /* Since the first one is hardcoded. */
ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
while (repeat >= 6) {
ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
ZOPFLI_APPEND_DATA(6 - 3, &rle_bits, &rle_bits_size);
repeat -= 6;
}
if (repeat >= 3) {
ZOPFLI_APPEND_DATA(16, &rle, &rle_size);
ZOPFLI_APPEND_DATA(3 - 3, &rle_bits, &rle_bits_size);
repeat -= 3;
}
while (repeat != 0) {
ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
repeat--;
}
}
i += count - 1;
} else {
ZOPFLI_APPEND_DATA(lld_lengths[i], &rle, &rle_size);
ZOPFLI_APPEND_DATA(0, &rle_bits, &rle_bits_size);
}
assert(rle[rle_size - 1] <= 18);
}
for (i = 0; i < 19; i++) {
clcounts[i] = 0;
}
for (i = 0; i < rle_size; i++) {
clcounts[rle[i]]++;
}
ZopfliCalculateBitLengths(clcounts, 19, 7, clcl);
ZopfliLengthsToSymbols(clcl, 19, 7, clsymbols);
hclen = 15;
/* Trim zeros. */
while (hclen > 0 && clcounts[order[hclen + 4 - 1]] == 0) hclen--;
AddBits(hlit, 5, bp, out, outsize);
AddBits(hdist, 5, bp, out, outsize);
AddBits(hclen, 4, bp, out, outsize);
for (i = 0; i < hclen + 4; i++) {
AddBits(clcl[order[i]], 3, bp, out, outsize);
}
for (i = 0; i < rle_size; i++) {
unsigned symbol = clsymbols[rle[i]];
AddHuffmanBits(symbol, clcl[rle[i]], bp, out, outsize);
/* Extra bits. */
if (rle[i] == 16) AddBits(rle_bits[i], 2, bp, out, outsize);
else if (rle[i] == 17) AddBits(rle_bits[i], 3, bp, out, outsize);
else if (rle[i] == 18) AddBits(rle_bits[i], 7, bp, out, outsize);
}
free(lld_lengths);
free(rle);
free(rle_bits);
}
/*
Appends the length and distance to the LZ77 arrays of the ZopfliLZ77Store.
context must be a ZopfliLZ77Store*.
*/
void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist,
ZopfliLZ77Store* store) {
size_t size2 = store->size; /* Needed for using ZOPFLI_APPEND_DATA twice. */
ZOPFLI_APPEND_DATA(length, &store->litlens, &store->size);
ZOPFLI_APPEND_DATA(dist, &store->dists, &size2);
}
/*
Compresses the data according to the gzip specification.
*/
void ZopfliGzipCompress(const ZopfliOptions* options,
const unsigned char* in, size_t insize,
unsigned char** out, size_t* outsize) {
unsigned long crcvalue = lodepng_crc32(in, insize);
unsigned char bp = 0;
ZOPFLI_APPEND_DATA(31, out, outsize); /* ID1 */
ZOPFLI_APPEND_DATA(139, out, outsize); /* ID2 */
ZOPFLI_APPEND_DATA(8, out, outsize); /* CM */
ZOPFLI_APPEND_DATA(0, out, outsize); /* FLG */
/* MTIME */
ZOPFLI_APPEND_DATA(0, out, outsize);
ZOPFLI_APPEND_DATA(0, out, outsize);
ZOPFLI_APPEND_DATA(0, out, outsize);
ZOPFLI_APPEND_DATA(0, out, outsize);
ZOPFLI_APPEND_DATA(2, out, outsize); /* XFL, 2 indicates best compression. */
ZOPFLI_APPEND_DATA(3, out, outsize); /* OS follows Unix conventions. */
ZopfliDeflate(options, 2 /* Dynamic block */, 1,
in, insize, &bp, out, outsize);
/* CRC */
ZOPFLI_APPEND_DATA(crcvalue % 256, out, outsize);
ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, out, outsize);
ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, out, outsize);
ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, out, outsize);
/* ISIZE */
ZOPFLI_APPEND_DATA(insize % 256, out, outsize);
ZOPFLI_APPEND_DATA((insize >> 8) % 256, out, outsize);
ZOPFLI_APPEND_DATA((insize >> 16) % 256, out, outsize);
ZOPFLI_APPEND_DATA((insize >> 24) % 256, out, outsize);
if (options->verbose) {
ZopfliPrintSizeVerbose(insize, *outsize, "Gzip");
}
}