UInt32 CompressionCodecZSTD::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
size_t compressed_size = ZSTD_compress(dest, ZSTD_compressBound(source_size), source, source_size, level);
if (ZSTD_isError(compressed_size))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(compressed_size)), ErrorCodes::CANNOT_COMPRESS);
return compressed_size;
}
void SL::Remote_Access_Library::Network::Packet::compress()
{
if (_PacketHeader.UnCompressedlen > 0) return;//allready compressed
auto maxsize = ZSTD_compressBound(_PacketHeader.PayloadLen);
auto buf = Remote_Access_Library::INTERNAL::_PacketBuffer.AquireBuffer(maxsize);
_PacketHeader.UnCompressedlen = _PacketHeader.PayloadLen;
_PacketHeader.PayloadLen = static_cast<unsigned int>(ZSTD_compress(buf.data, maxsize, data(), _PacketHeader.PayloadLen));
memcpy(data(), buf.data, _PacketHeader.PayloadLen);
Remote_Access_Library::INTERNAL::_PacketBuffer.ReleaseBuffer(buf);
}
std::unique_ptr<IOBuf> ZSTDCodec::doCompress(const IOBuf* data) {
// Support earlier versions of the codec (working with a single IOBuf,
// and using ZSTD_decompress which requires ZSTD frame to contain size,
// which isn't populated by streaming API).
if (!data->isChained()) {
auto out = IOBuf::createCombined(ZSTD_compressBound(data->length()));
const auto rc = ZSTD_compress(
out->writableData(),
out->capacity(),
data->data(),
data->length(),
level_);
zstdThrowIfError(rc);
out->append(rc);
return out;
}
auto zcs = ZSTD_createCStream();
SCOPE_EXIT {
ZSTD_freeCStream(zcs);
};
auto rc = ZSTD_initCStream(zcs, level_);
zstdThrowIfError(rc);
Cursor cursor(data);
auto result = IOBuf::createCombined(ZSTD_compressBound(cursor.totalLength()));
ZSTD_outBuffer out;
out.dst = result->writableTail();
out.size = result->capacity();
out.pos = 0;
for (auto buffer = cursor.peekBytes(); !buffer.empty();) {
ZSTD_inBuffer in;
in.src = buffer.data();
in.size = buffer.size();
for (in.pos = 0; in.pos != in.size;) {
rc = ZSTD_compressStream(zcs, &out, &in);
zstdThrowIfError(rc);
}
cursor.skip(in.size);
buffer = cursor.peekBytes();
}
rc = ZSTD_endStream(zcs, &out);
zstdThrowIfError(rc);
CHECK_EQ(rc, 0);
result->append(out.pos);
return result;
}
static PyObject *py_zstd_compress(PyObject* self, PyObject *args) {
PyObject *result;
const char *source;
uint32_t source_size;
char *dest;
uint32_t dest_size;
uint32_t header_size;
size_t cSize;
uint32_t level = 0;
#if PY_MAJOR_VERSION >= 3
if (!PyArg_ParseTuple(args, "y#|i", &source, &source_size, &level))
return NULL;
#else
if (!PyArg_ParseTuple(args, "s#|i", &source, &source_size, &level))
return NULL;
#endif
if (level <= 0) level=1;
if (level > 20) level=20;
header_size = sizeof(source_size);
dest_size = ZSTD_compressBound(source_size);
result = PyBytes_FromStringAndSize(NULL, header_size + dest_size);
if (result == NULL) {
return NULL;
}
dest = PyBytes_AS_STRING(result);
memcpy(dest, &source_size, header_size);
dest += header_size;
if (source_size > 0) {
// Low level == old version
cSize = ZSTD_compress(dest, dest_size, source, source_size, level);
if (ZSTD_isError(cSize))
PyErr_Format(ZstdError, "Compression error: %s", ZSTD_getErrorName(cSize));
Py_SIZE(result) = cSize + sizeof(source_size);
}
return result;
}
ByteArray Compress(const ByteArrayView view, const int32 compressionLevel)
{
const size_t bufferSize = ZSTD_compressBound(view.size());
Array<Byte> buffer(bufferSize);
const size_t result = ZSTD_compress(buffer.data(), buffer.size(), view.data(), view.size(), compressionLevel);
if (ZSTD_isError(result))
{
return ByteArray();
}
buffer.resize(result);
buffer.shrink_to_fit();
return ByteArray(std::move(buffer));
}
static ezResult CompressZStd(ezArrayPtr<const ezUInt8> pUncompressedData, ezDynamicArray<ezUInt8>& out_Data)
{
size_t uiSizeBound = ZSTD_compressBound(pUncompressedData.GetCount());
if(uiSizeBound > std::numeric_limits<ezUInt32>::max())
{
ezLog::Error("Can't compress since the output container can't hold enough elements ({0})", static_cast<ezUInt64>(uiSizeBound));
return EZ_FAILURE;
}
out_Data.SetCountUninitialized(static_cast<ezUInt32>(uiSizeBound));
size_t const cSize = ZSTD_compress(out_Data.GetData(), uiSizeBound, pUncompressedData.GetPtr(), pUncompressedData.GetCount(), 1);
if (ZSTD_isError(cSize))
{
ezLog::Error("Compression failed with error: '{0}'.", ZSTD_getErrorName(cSize));
return EZ_FAILURE;
}
out_Data.SetCount(static_cast<ezUInt32>(cSize));
return EZ_SUCCESS;
}
std::unique_ptr<IOBuf> ZSTDCodec::doCompress(const IOBuf* data) {
size_t rc;
size_t maxCompressedLength = ZSTD_compressBound(data->length());
auto out = IOBuf::createCombined(maxCompressedLength);
CHECK_EQ(out->length(), 0);
rc = ZSTD_compress(out->writableTail(),
out->capacity(),
data->data(),
data->length(),
level_);
if (ZSTD_isError(rc)) {
throw std::runtime_error(to<std::string>(
"ZSTD compression returned an error: ",
ZSTD_getErrorName(rc)));
}
out->append(rc);
CHECK_EQ(out->length(), rc);
return out;
}
int64_t lzbench_zstd_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*)
{
return ZSTD_compress(outbuf, outsize, inbuf, insize, level);
}
/*
* Finish off an encoded strip by flushing it.
*/
static int
LERCPostEncode(TIFF* tif)
{
lerc_status lerc_ret;
static const char module[] = "LERCPostEncode";
LERCState *sp = EncoderState(tif);
unsigned int numBytes = 0;
unsigned int numBytesWritten = 0;
TIFFDirectory *td = &tif->tif_dir;
int use_mask = 0;
unsigned dst_nbands = td->td_samplesperpixel;
if( sp->uncompressed_offset != sp->uncompressed_size )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected number of bytes in the buffer");
return 0;
}
/* Extract alpha mask (if containing only 0 and 255 values, */
/* and compact array of regular bands */
if( td->td_planarconfig == PLANARCONFIG_CONTIG &&
td->td_extrasamples > 0 &&
td->td_sampleinfo[td->td_extrasamples-1] == EXTRASAMPLE_UNASSALPHA &&
GetLercDataType(tif) == 1 )
{
unsigned dst_stride = (td->td_samplesperpixel - 1) *
(td->td_bitspersample / 8);
unsigned src_stride = td->td_samplesperpixel *
(td->td_bitspersample / 8);
unsigned i = 0;
unsigned nb_pixels = sp->segment_width * sp->segment_height;
use_mask = 1;
for( i = 0 ; i < nb_pixels; i++)
{
int v = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
if( v != 0 && v != 255 )
{
use_mask = 0;
break;
}
}
if( use_mask )
{
dst_nbands --;
/* First pixels must use memmove due to overlapping areas */
for( i = 0 ;i < dst_nbands && i < nb_pixels; i++)
{
memmove( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
dst_stride );
sp->mask_buffer[i] = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
}
for(; i < nb_pixels; i++)
{
memcpy( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
dst_stride );
sp->mask_buffer[i] = sp->uncompressed_buffer[
i * src_stride + td->td_samplesperpixel - 1];
}
}
}
#if 0
lerc_ret = lerc_computeCompressedSize(
sp->uncompressed_buffer,
sp->lerc_version,
GetLercDataType(tif),
td->td_planarconfig == PLANARCONFIG_CONTIG ?
dst_nbands : 1,
sp->segment_width,
sp->segment_height,
1,
use_mask ? sp->mask_buffer : NULL,
sp->maxzerror,
&numBytes);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_computeCompressedSize() failed");
return 0;
}
#else
numBytes = sp->uncompressed_alloc;
#endif
if( sp->compressed_size < numBytes )
{
_TIFFfree(sp->compressed_buffer);
sp->compressed_buffer = _TIFFmalloc(numBytes);
if( !sp->compressed_buffer )
{
sp->compressed_size = 0;
return 0;
}
sp->compressed_size = numBytes;
}
lerc_ret = lerc_encodeForVersion(
sp->uncompressed_buffer,
sp->lerc_version,
GetLercDataType(tif),
td->td_planarconfig == PLANARCONFIG_CONTIG ?
dst_nbands : 1,
sp->segment_width,
sp->segment_height,
1,
use_mask ? sp->mask_buffer : NULL,
sp->maxzerror,
sp->compressed_buffer,
sp->compressed_size,
&numBytesWritten);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_encode() failed");
return 0;
}
assert( numBytesWritten < numBytes );
if( sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE )
{
z_stream strm;
int zlib_ret;
memset(&strm, 0, sizeof(strm));
strm.zalloc = NULL;
strm.zfree = NULL;
strm.opaque = NULL;
zlib_ret = deflateInit(&strm, sp->zipquality);
if( zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"deflateInit() failed");
return 0;
}
strm.avail_in = numBytesWritten;
strm.next_in = sp->compressed_buffer;
strm.avail_out = sp->uncompressed_alloc;
strm.next_out = sp->uncompressed_buffer;
zlib_ret = deflate(&strm, Z_FINISH);
if( zlib_ret != Z_STREAM_END )
{
TIFFErrorExt(tif->tif_clientdata, module,
"deflate() failed");
deflateEnd(&strm);
return 0;
}
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->uncompressed_buffer;
tif->tif_rawcc = sp->uncompressed_alloc - strm.avail_out;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
{
deflateEnd(&strm);
return 0;
}
}
deflateEnd(&strm);
}
else if( sp->additional_compression == LERC_ADD_COMPRESSION_ZSTD )
{
#ifdef ZSTD_SUPPORT
size_t zstd_ret = ZSTD_compress( sp->uncompressed_buffer,
sp->uncompressed_alloc,
sp->compressed_buffer,
numBytesWritten,
sp->zstd_compress_level );
if( ZSTD_isError(zstd_ret) ) {
TIFFErrorExt(tif->tif_clientdata, module,
"Error in ZSTD_compress(): %s",
ZSTD_getErrorName(zstd_ret));
return 0;
}
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->uncompressed_buffer;
tif->tif_rawcc = zstd_ret;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
{
return 0;
}
}
#else
TIFFErrorExt(tif->tif_clientdata, module, "ZSTD support missing");
return 0;
#endif
}
else if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unhandled additional compression");
return 0;
}
else
{
int ret;
uint8* tif_rawdata_backup = tif->tif_rawdata;
tif->tif_rawdata = sp->compressed_buffer;
tif->tif_rawcc = numBytesWritten;
ret = TIFFFlushData1(tif);
tif->tif_rawdata = tif_rawdata_backup;
if( !ret )
return 0;
}
return 1;
}
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
size_t result, cSize;
U32 testNb=0;
/* Create compressible test buffer */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
if (!CNBuffer || !compressedBuffer || !decodedBuffer)
{
DISPLAY("Not enough memory, aborting\n");
testResult = 1;
goto _end;
}
RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState);
/* Basic tests */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (ZSTD_isError(result)) goto _output_error;
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
{
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++)
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Dictionary and Duplication tests */
{
ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
const size_t dictSize = 500;
size_t cSizeOrig;
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
result = ZSTD_compressBegin(ctxOrig, 2);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
cSize = 0;
result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
cSizeOrig = cSize;
cSize = 0;
result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
ZSTD_freeCCtx(ctxDuplicated);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeDCtx(dctx);
DISPLAYLEVEL(4, "OK \n");
}
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* block API tests */
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t blockSize = 100 KB;
const size_t dictSize = 16 KB;
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* dictionary block compression */
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(cctx, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize);
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
/* long rle test */
{
size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error;
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result!=sampleSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
}
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** The format of compressed block - see CompressedStream.h
*/
switch (method)
{
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress_default(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size));
else
compressed_size = header_size + LZ4_compress_HC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size),
0);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
(void)startTest; (void)compressibility;
/* allocation */
srcBuffer = malloc (srcBufferSize);
dstBuffer = malloc (dstBufferSize);
cBuffer = malloc (cBufferSize);
CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled");
/* Create initial sample */
FUZ_generateSynthetic(srcBuffer, srcBufferSize, 0.50, &coreSeed);
/* catch up testNb */
for (testNb=0; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for (testNb=startTest; testNb < nbTests; testNb++)
{
size_t sampleSize, sampleStart;
size_t cSize, dSize, dSupSize;
U32 sampleSizeLog;
U64 crcOrig, crcDest;
/* init */
DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1<<sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
/* compression tests*/
cSize = ZSTD_compress(cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize);
CHECK(ZSTD_isError(cSize), "ZSTD_compress failed");
/* decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s)", ZSTD_getErrorName(dSize));
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
}
DISPLAY("\rAll fuzzer tests completed \n");
_cleanup:
free(srcBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
size_t result, cSize;
U32 testNb=0;
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_generateSynthetic(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
// Basic tests
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH);
if (ZSTD_isError(result)) goto _output_error;
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
{
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++)
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** Формат сжатого блока - см. CompressedStream.h
*/
switch (method)
{
case CompressionMethod::QuickLZ:
{
#ifdef USE_QUICKLZ
compressed_buffer.resize(uncompressed_size + QUICKLZ_ADDITIONAL_SPACE);
compressed_size = qlz_compress(
working_buffer.begin(),
&compressed_buffer[0],
uncompressed_size,
qlz_state.get());
compressed_buffer[0] &= 3;
compressed_buffer_ptr = &compressed_buffer[0];
break;
#else
throw Exception("QuickLZ compression method is disabled", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
#endif
}
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
else
compressed_size = header_size + LZ4_compressHC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
uint128 checksum = CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
// You should have received a copy of the GNU Lesser General Public License
// along with SCIL. If not, see <http://www.gnu.org/licenses/>.
#include <algo/zstd-11.h>
#include <string.h>
#include <zstd/zstd.h>
#pragma GCC diagnostic ignored "-Wunused-parameter"
int scil_zstd11_compress(const scil_context_t* ctx, byte* restrict dest, size_t * restrict out_size, const byte*restrict source, const size_t source_size){
size_t size;
// store the size of the data
*((size_t*) dest) = source_size;
// normal compression, not fast
//size = LZ4_compress_fast((const char *) (source), (char *) dest + 4, source_size, 2*source_size, 4);
size = ZSTD_compress (dest, 2*source_size, source, source_size, 11);
*out_size = size + 4;
if (size == 0){
return -1;
}
return 0;
}
#pragma GCC diagnostic ignored "-Wunused-parameter"
int scil_zstd11_decompress(byte*restrict dest, size_t buff_size, const byte*restrict src, const size_t in_size, size_t * uncomp_size_out){
size_t size;
// retrieve the size of the uncompressed data
size = *((size_t *) src);
*uncomp_size_out = size;
