int
lz4_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
int level, uchar_t chdr, void *data)
{
int rv;
struct lz4_params *lzdat = (struct lz4_params *)data;
int _srclen = srclen;
uchar_t *dst2;
if (lzdat->level == 1) {
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
} else if (lzdat->level == 2) {
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
if (rv == 0 || rv > *dstlen) {
return (-1);
}
dst2 = (uchar_t *)slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv));
*((int *)dst2) = htonl(rv);
rv = LZ4_compressHC((const char *)dst, (char *)(dst2 + sizeof (int)), rv);
if (rv != 0) {
rv += sizeof (int);
memcpy(dst, dst2, rv);
}
slab_free(NULL, dst2);
} else {
rv = LZ4_compressHC((const char *)src, (char *)dst, _srclen);
}
if (rv == 0) {
return (-1);
}
*dstlen = rv;
return (0);
}
bool CompressStream(Serializer& dest, Deserializer& src)
{
unsigned srcSize = src.GetSize() - src.GetPosition();
// Prepend the source and dest. data size in the stream so that we know to buffer & uncompress the right amount
if (!srcSize)
{
dest.WriteUInt(0);
dest.WriteUInt(0);
return true;
}
unsigned maxDestSize = LZ4_compressBound(srcSize);
SharedArrayPtr<unsigned char> srcBuffer(new unsigned char[srcSize]);
SharedArrayPtr<unsigned char> destBuffer(new unsigned char[maxDestSize]);
if (src.Read(srcBuffer, srcSize) != srcSize)
return false;
unsigned destSize = LZ4_compressHC((const char*)srcBuffer.Get(), (char*)destBuffer.Get(), srcSize);
bool success = true;
success &= dest.WriteUInt(srcSize);
success &= dest.WriteUInt(destSize);
success &= dest.Write(destBuffer, destSize) == destSize;
return success;
}
unsigned CompressData(void* dest, const void* src, unsigned srcSize)
{
if (!dest || !src || !srcSize)
return 0;
else
return LZ4_compressHC((const char*)src, (char*)dest, srcSize);
}
std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {
std::unique_ptr<IOBuf> clone;
if (data->isChained()) {
// LZ4 doesn't support streaming, so we have to coalesce
clone = data->clone();
clone->coalesce();
data = clone.get();
}
uint32_t extraSize = encodeSize() ? kMaxVarintLength64 : 0;
auto out = IOBuf::create(extraSize + LZ4_compressBound(data->length()));
if (encodeSize()) {
encodeVarintToIOBuf(data->length(), out.get());
}
int n;
if (highCompression_) {
n = LZ4_compressHC(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
} else {
n = LZ4_compress(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
}
CHECK_GE(n, 0);
CHECK_LE(n, out->capacity());
out->append(n);
return out;
}
static bool compress( T2 &buffer_out, const T1 &buffer_in, bool highest_compression = true )
{
static const bool verbose = false;
bool ret = false;
if( 1 )
{
// resize to worst case
buffer_out.resize( LZ4_compressBound((int)(buffer_in.size())) );
// compress
size_t compressed_size = highest_compression ?
LZ4_compressHC( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() )
:
LZ4_compress( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() );
ret = ( compressed_size > 0 && compressed_size < buffer_in.size() );
if( ret )
{
// if ok, resize properly to unused space
buffer_out.resize( compressed_size );
}
if( verbose )
{
// std::cout << moon9::echo( ret, compressed_size, buffer_in.size() );
}
}
return ret;
}
static ZEND_FUNCTION(lz4_compress)
{
zval *data;
char *output;
int output_len, data_len;
zend_bool high = 0;
char *extra = NULL;
int extra_len = -1;
int offset = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|bs", &data, &high,
&extra, &extra_len) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"lz4_compress : expects parameter to be string.");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
offset = extra_len;
} else {
offset = sizeof(int);
}
data_len = Z_STRLEN_P(data);
output = (char *)emalloc(LZ4_compressBound(data_len) + offset);
if (!output) {
zend_error(E_WARNING, "lz4_compress : memory error");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
memcpy(output, extra, offset);
} else {
/* Set the data length */
memcpy(output, &data_len, offset);
}
if (high) {
output_len = LZ4_compressHC(Z_STRVAL_P(data), output + offset, data_len);
} else {
output_len = LZ4_compress(Z_STRVAL_P(data), output + offset, data_len);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
RETVAL_STRINGL(output, output_len + offset, 1);
}
efree(output);
}
/* compression backend for LZ4 */
static int lz4_backend_compress(int level, const void* input, int length,
void* output) {
if (level == Z_BEST_COMPRESSION) {
return LZ4_compressHC(input, output, length);
}
else {
return LZ4_compress(input, output, length);
}
}
static inline void compress(I begin,I end,O out)
{
unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
char *buf = new char[bufLen * 2];
char *buf2 = buf + bufLen;
try {
I inp(begin);
for(;;) {
unsigned int readLen = 0;
while ((readLen < ZT_COMPRESSION_BLOCK_SIZE)&&(inp != end)) {
buf[readLen++] = (char)*inp;
++inp;
}
if (!readLen)
break;
uint32_t l = hton((uint32_t)readLen);
out((const void *)&l,4); // original size
if (readLen < 32) { // don't bother compressing itty bitty blocks
l = 0; // stored
out((const void *)&l,4);
out((const void *)buf,readLen);
continue;
}
int lz4CompressedLen = LZ4_compressHC(buf,buf2,(int)readLen);
if ((lz4CompressedLen <= 0)||(lz4CompressedLen >= (int)readLen)) {
l = 0; // stored
out((const void *)&l,4);
out((const void *)buf,readLen);
continue;
}
l = hton((uint32_t)lz4CompressedLen); // lz4 only
out((const void *)&l,4);
out((const void *)buf2,(unsigned int)lz4CompressedLen);
}
delete [] buf;
} catch ( ... ) {
delete [] buf;
throw;
}
}
void hh_save(value out_filename) {
CAMLparam1(out_filename);
FILE* fp = fopen(String_val(out_filename), "wb");
fwrite_no_fail(&MAGIC_CONSTANT, sizeof MAGIC_CONSTANT, 1, fp);
size_t revlen = strlen(BuildInfo_kRevision);
fwrite_no_fail(&revlen, sizeof revlen, 1, fp);
fwrite_no_fail(BuildInfo_kRevision, sizeof(char), revlen, fp);
fwrite_no_fail(&heap_init_size, sizeof heap_init_size, 1, fp);
/*
* Format of the compressed shared memory:
* LZ4 can only work in chunks of 2GB, so we compress each chunk individually,
* and write out each one as
* [compressed size of chunk][uncompressed size of chunk][chunk]
* A compressed size of zero indicates the end of the compressed section.
*/
char* chunk_start = save_start();
int compressed_size = 0;
while (chunk_start < *heap) {
uintptr_t remaining = *heap - chunk_start;
uintptr_t chunk_size = LZ4_MAX_INPUT_SIZE < remaining ?
LZ4_MAX_INPUT_SIZE : remaining;
char* compressed = malloc(chunk_size * sizeof(char));
assert(compressed != NULL);
compressed_size = LZ4_compressHC(chunk_start, compressed,
chunk_size);
assert(compressed_size > 0);
fwrite_no_fail(&compressed_size, sizeof compressed_size, 1, fp);
fwrite_no_fail(&chunk_size, sizeof chunk_size, 1, fp);
fwrite_no_fail((void*)compressed, 1, compressed_size, fp);
chunk_start += chunk_size;
free(compressed);
}
compressed_size = 0;
fwrite_no_fail(&compressed_size, sizeof compressed_size, 1, fp);
fclose(fp);
CAMLreturn0;
}
static ZEND_FUNCTION(horde_lz4_compress)
{
zval *data;
char *output;
int header_offset = (sizeof(headerid) + sizeof(int));
int output_len, data_len;
zend_bool high = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|b", &data, &high) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"horde_lz4_compress: uncompressed data must be a string.");
RETURN_FALSE;
}
data_len = Z_STRLEN_P(data);
output = (char *)emalloc(LZ4_compressBound(data_len) + header_offset);
if (!output) {
zend_error(E_WARNING, "horde_lz4_compress: memory error");
RETURN_FALSE;
}
*output = headerid;
memcpy(output + sizeof(headerid), &data_len, sizeof(int));
if (high) {
output_len = LZ4_compressHC(Z_STRVAL_P(data), output + header_offset, data_len);
} else {
output_len = LZ4_compress(Z_STRVAL_P(data), output + header_offset, data_len);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
RETVAL_STRINGL(output, output_len + header_offset, 1);
}
efree(output);
}
void hh_save_dep_table(value out_filename) {
CAMLparam1(out_filename);
FILE* fp = fopen(String_val(out_filename), "wb");
fwrite_header(fp);
int compressed_size = 0;
assert(LZ4_MAX_INPUT_SIZE >= DEP_SIZE_B);
char* compressed = malloc(DEP_SIZE_B);
assert(compressed != NULL);
compressed_size = LZ4_compressHC((char*)deptbl, compressed, DEP_SIZE_B);
assert(compressed_size > 0);
fwrite_no_fail(&compressed_size, sizeof compressed_size, 1, fp);
fwrite_no_fail((void*)compressed, 1, compressed_size, fp);
free(compressed);
fclose(fp);
CAMLreturn0;
}
/*
* Class: net_jpountz_lz4_LZ4
* Method: LZ4_compressHC
* Signature: ([BII[BI)I
*/
JNIEXPORT jint JNICALL Java_net_jpountz_lz4_LZ4JNI_LZ4_1compressHC
(JNIEnv *env, jclass cls, jbyteArray src, jint srcOff, jint srcLen, jbyteArray dest, jint destOff) {
char* in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0);
if (in == NULL) {
throw_OOM(env);
return 0;
}
char* out = (char*) (*env)->GetPrimitiveArrayCritical(env, dest, 0);
if (out == NULL) {
throw_OOM(env);
return 0;
}
jint compressed = LZ4_compressHC(in + srcOff, out + destOff, srcLen);
(*env)->ReleasePrimitiveArrayCritical(env, src, in, 0);
(*env)->ReleasePrimitiveArrayCritical(env, src, out, 0);
return compressed;
}
bool compress(Compression compression, char *&data, std::uint32_t in_size, std::uint32_t &out_size)
{
if (data == nullptr)
{
return false;
}
switch (compression)
{
case COMPRESS_NONE:
{
out_size = in_size;
return true;
}
#ifdef ZAP_COMPRESS_LZ4
case COMPRESS_LZ4:
{
char *out_data = new char[LZ4_compressBound(in_size)];
out_size = LZ4_compressHC(data, out_data, in_size);
if (out_size > 0)
{
delete[] data;
data = out_data;
return true;
}
else
{
return false;
}
}
#endif
default:
{
return false;
}
}
}
int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
unsigned long long hcbytes = 0;
unsigned long long ccbytes = 0;
void* CNBuffer;
char* compressedBuffer;
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
unsigned int randState=seed;
int ret, cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
U32 crcOrig, crcCheck;
int displayRefresh;
// init
memset(&LZ4dict, 0, sizeof(LZ4dict));
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// display refresh rate
switch(displayLevel)
{
case 0: displayRefresh = nbCycles+1; break;
case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
case 2: displayRefresh=89; break;
default : displayRefresh=1;
}
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
// synd rand & dict
int dictSize, blockSize, blockStart;
char* dict;
char* block;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
int testNb = 0;
char* dict;
char* block;
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
if ((cycleNb%displayRefresh) == 0)
{
printf("\r%7i /%7i - ", cycleNb, nbCycles);
fflush(stdout);
}
// Select block to test
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
/* Compression tests */
// Test compression HC
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
// Test compression HC using external state
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
compressedSize = ret;
/* Decompression tests */
crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with more than enough output size => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
decodedBuffer[blockSize+1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with output size being one byte too short => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
// Test decoding with output size being 10 bytes too short => must fail
FUZ_DISPLAYTEST;
if (blockSize>10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
}
// Test decoding with input size being one byte too short => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
// Test decoding with input size being one byte too large => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
// Test partial decoding with target output size being max/2 => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test partial decoding with target output size being just below max => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
/* Test Compression with limited output size */
// Test compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
// Test compression with output size being exactly what's necessary and external state (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
// Test compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
/* Dictionary tests */
// Compress using dictionary
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
LZ4_free (LZ4continue);
// Decompress with dictionary as prefix
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
// Compress using External dictionary
FUZ_DISPLAYTEST;
dict -= 9; // Separation, so it is an ExtDict
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
if (blockSize > 10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
}
// Fill stats
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
printf("\r%7i /%7i - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
// unalloc
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 0;
_output_error:
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 1;
}
bool ResourcePackager::WritePackageFile(const String& destFilePath)
{
buildBase_->BuildLog("Writing package");
SharedPtr<File> dest(new File(context_, destFilePath, FILE_WRITE));
if (!dest->IsOpen())
{
buildBase_->BuildError("Could not open output file " + destFilePath);
return false;
}
// Write ID, number of files & placeholder for checksum
WriteHeader(dest);
for (unsigned i = 0; i < resourceEntries_.Size(); i++)
{
BuildResourceEntry* entry = resourceEntries_[i];
// Write entry (correct offset is still unknown, will be filled in later)
dest->WriteString(entry->packagePath_);
dest->WriteUInt(entry->offset_);
dest->WriteUInt(entry->size_);
dest->WriteUInt(entry->checksum_);
}
unsigned totalDataSize = 0;
// Write file data, calculate checksums & correct offsets
for (unsigned i = 0; i < resourceEntries_.Size(); i++)
{
BuildResourceEntry* entry = resourceEntries_[i];
entry->offset_ = dest->GetSize();
File srcFile(context_, entry->absolutePath_);
if (!srcFile.IsOpen())
{
buildBase_->BuildError("Could not open input file " + entry->absolutePath_);
return false;
}
unsigned dataSize = entry->size_;
totalDataSize += dataSize;
SharedArrayPtr<unsigned char> buffer(new unsigned char[dataSize]);
if (srcFile.Read(&buffer[0], dataSize) != dataSize)
{
buildBase_->BuildError("Could not read input file " + entry->absolutePath_);
return false;
}
srcFile.Close();
for (unsigned j = 0; j < dataSize; ++j)
{
checksum_ = SDBMHash(checksum_, buffer[j]);
entry->checksum_ = SDBMHash(entry->checksum_, buffer[j]);
}
// might not want to compress for WebGL
//if (!compress_)
//{
// PrintLine(entries_[i].name_ + " size " + String(dataSize));
// dest.Write(&buffer[0], entries_[i].size_);
//}
//else
//{
unsigned compressedBlockSize_ = 32768;
SharedArrayPtr<unsigned char> compressBuffer(new unsigned char[LZ4_compressBound(compressedBlockSize_)]);
unsigned pos = 0;
unsigned totalPackedBytes = 0;
while (pos < dataSize)
{
unsigned unpackedSize = compressedBlockSize_;
if (pos + unpackedSize > dataSize)
unpackedSize = dataSize - pos;
unsigned packedSize = LZ4_compressHC((const char*)&buffer[pos], (char*)compressBuffer.Get(), unpackedSize);
if (!packedSize)
{
buildBase_->BuildError("LZ4 compression failed for file " + entry->absolutePath_ + " at offset " + pos);
return false;
}
dest->WriteUShort(unpackedSize);
dest->WriteUShort(packedSize);
dest->Write(compressBuffer.Get(), packedSize);
totalPackedBytes += 6 + packedSize;
pos += unpackedSize;
}
buildBase_->BuildLog(entry->absolutePath_ + " in " + String(dataSize) + " out " + String(totalPackedBytes));
}
//}
// Write package size to the end of file to allow finding it linked to an executable file
unsigned currentSize = dest->GetSize();
dest->WriteUInt(currentSize + sizeof(unsigned));
// Write header again with correct offsets & checksums
dest->Seek(0);
WriteHeader(dest);
for (unsigned i = 0; i < resourceEntries_.Size(); i++)
{
BuildResourceEntry* entry = resourceEntries_[i];
dest->WriteString(entry->packagePath_);
dest->WriteUInt(entry->offset_);
dest->WriteUInt(entry->size_);
dest->WriteUInt(entry->checksum_);
}
buildBase_->BuildLog("Number of files " + String(resourceEntries_.Size()));
buildBase_->BuildLog("File data size " + String(totalDataSize));
buildBase_->BuildLog("Package size " + String(dest->GetSize()));
return true;
}
static int
_save_direct_tgv(RGBA_Image *im, const char *file, int compress)
{
// TODO: Add block by block compression.
int image_width, image_height, planes = 1;
uint32_t width, height;
uint8_t header[8] = "TGV1";
int etc_block_size, etc_data_size, buffer_size, data_size, remain;
uint8_t *buffer = NULL;
uint8_t *data, *ptr;
FILE *f;
if (!im->cache_entry.borders.l || !im->cache_entry.borders.t ||
!im->cache_entry.borders.r || !im->cache_entry.borders.b)
WRN("No im->cache_entry.borders on ETC image. Final image may have wrong dimensions.");
image_width = im->cache_entry.w + im->cache_entry.borders.l + im->cache_entry.borders.r;
image_height = im->cache_entry.h + im->cache_entry.borders.t + im->cache_entry.borders.b;
data = im->image.data8;
width = htonl(image_width);
height = htonl(image_height);
compress = !!compress;
if ((image_width & 0x3) || (image_height & 0x3))
return 0;
// header[4]: block size info, unused
header[4] = 0;
// header[5]: 0 for ETC1
switch (im->cache_entry.space)
{
case EVAS_COLORSPACE_ETC1:
etc_block_size = 8;
header[5] = 0;
break;
case EVAS_COLORSPACE_RGB8_ETC2:
etc_block_size = 8;
header[5] = 1;
break;
case EVAS_COLORSPACE_RGBA8_ETC2_EAC:
etc_block_size = 16;
header[5] = 2;
break;
case EVAS_COLORSPACE_ETC1_ALPHA:
// FIXME: Properly handle premul vs. unpremul data
etc_block_size = 8;
header[5] = 3;
planes = 2;
break;
default:
return 0;
}
// header[6]: 0 for raw, 1, for LZ4 compressed, 2 for block-less mode
header[6] = compress | 0x2;
// header[7]: options (unused)
header[7] = 0;
f = fopen(file, "wb");
if (!f) return 0;
// Write header
if (fwrite(header, sizeof (uint8_t), 8, f) != 8) goto on_error;
if (fwrite(&width, sizeof (uint32_t), 1, f) != 1) goto on_error;
if (fwrite(&height, sizeof (uint32_t), 1, f) != 1) goto on_error;
etc_data_size = image_width * image_height * etc_block_size * planes / 16;
if (compress)
{
buffer_size = LZ4_compressBound(etc_data_size);
buffer = malloc(buffer_size);
if (!buffer) goto on_error;
data_size = LZ4_compressHC((char *) data, (char *) buffer, etc_data_size);
}
else
{
data_size = buffer_size = etc_data_size;
buffer = data;
}
// Write block length header -- We keep this even in block-less mode
if (data_size > 0)
{
unsigned int blen = data_size;
while (blen)
{
unsigned char plen;
plen = blen & 0x7F;
blen = blen >> 7;
if (blen) plen = 0x80 | plen;
if (fwrite(&plen, 1, 1, f) != 1) goto on_error;
}
}
extern __declspec(dllexport) int dll_LZ4_compressHC (const char* source, char* dest, int isize)
{
return LZ4_compressHC (source, dest, isize);
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** Формат сжатого блока - см. CompressedStream.h
*/
switch (method)
{
case CompressionMethod::QuickLZ:
{
#ifdef USE_QUICKLZ
compressed_buffer.resize(uncompressed_size + QUICKLZ_ADDITIONAL_SPACE);
compressed_size = qlz_compress(
working_buffer.begin(),
&compressed_buffer[0],
uncompressed_size,
qlz_state.get());
compressed_buffer[0] &= 3;
compressed_buffer_ptr = &compressed_buffer[0];
break;
#else
throw Exception("QuickLZ compression method is disabled", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
#endif
}
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
else
compressed_size = header_size + LZ4_compressHC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
uint128 checksum = CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
