static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) {
PyObject *result;
const char *source;
uint32_t source_size;
uint32_t dest_size;
uint32_t header_size;
size_t cSize;
#if PY_MAJOR_VERSION >= 3
if (!PyArg_ParseTuple(args, "y#", &source, &source_size))
return NULL;
#else
if (!PyArg_ParseTuple(args, "s#", &source, &source_size))
return NULL;
#endif
header_size = sizeof(dest_size);
memcpy(&dest_size, source, header_size);
result = PyBytes_FromStringAndSize(NULL, dest_size);
source += header_size;
if (result != NULL && dest_size > 0) {
char *dest = PyBytes_AS_STRING(result);
cSize = ZSTD_decompress(dest, dest_size, source, source_size - header_size);
if (ZSTD_isError(cSize))
PyErr_Format(ZstdError, "Decompression error: %s", ZSTD_getErrorName(cSize));
}
return result;
}
int testSimpleAPI(void)
{
size_t const size = strlen(EXPECTED);
char* const output = malloc(size);
if (!output) {
DISPLAY("ERROR: Not enough memory!\n");
return 1;
}
{
size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE);
if (ZSTD_isError(ret)) {
if (ret == ZSTD_error_prefix_unknown) {
DISPLAY("ERROR: Invalid frame magic number, was this compiled "
"without legacy support?\n");
} else {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
}
return 1;
}
if (ret != size) {
DISPLAY("ERROR: Wrong decoded size\n");
}
}
if (memcmp(EXPECTED, output, size) != 0) {
DISPLAY("ERROR: Wrong decoded output produced\n");
return 1;
}
free(output);
DISPLAY("Simple API OK\n");
return 0;
}
void CompressionCodecZSTD::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
size_t res = ZSTD_decompress(dest, uncompressed_size, source, source_size);
if (ZSTD_isError(res))
throw Exception("Cannot ZSTD_decompress: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_DECOMPRESS);
}
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==ZSTD_CONTENTSIZE_ERROR) {
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
exit(5);
} else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
fprintf(stderr,
"%s : original size unknown. Use streaming decompression instead.\n", fname);
exit(6);
}
void* const rBuff = malloc_orDie((size_t)rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
if (dSize != rSize) {
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
free(rBuff);
free(cBuff);
}
static ezResult DecompressZStd(ezArrayPtr<const ezUInt8> pCompressedData, ezDynamicArray<ezUInt8>& out_Data)
{
size_t uiSize = ZSTD_findDecompressedSize(pCompressedData.GetPtr(), pCompressedData.GetCount());
if (uiSize == ZSTD_CONTENTSIZE_ERROR)
{
ezLog::Error("Can't decompress since it wasn't compressed with ZStd");
return EZ_FAILURE;
}
else if (uiSize == ZSTD_CONTENTSIZE_UNKNOWN)
{
ezLog::Error("Can't decompress since the original size can't be determined, was the data compressed using the streaming variant?");
return EZ_FAILURE;
}
if (uiSize > std::numeric_limits<ezUInt32>::max())
{
ezLog::Error("Can't compress since the output container can't hold enough elements ({0})", static_cast<ezUInt64>(uiSize));
return EZ_FAILURE;
}
out_Data.SetCountUninitialized(static_cast<ezUInt32>(uiSize));
size_t const uiActualSize = ZSTD_decompress(out_Data.GetData(), uiSize, pCompressedData.GetPtr(), pCompressedData.GetCount());
if (uiActualSize != uiSize)
{
ezLog::Error("Error during ZStd decompression: '{0}'.", ZSTD_getErrorName(uiActualSize));
return EZ_FAILURE;
}
return EZ_SUCCESS;
}
void SL::Remote_Access_Library::Network::Packet::decompress()
{
if (_PacketHeader.UnCompressedlen <= 0) return;//allready decompressed
auto buf = Remote_Access_Library::INTERNAL::_PacketBuffer.AquireBuffer(_PacketHeader.UnCompressedlen);
auto dstsize = ZSTD_decompress(buf.data, _PacketHeader.UnCompressedlen, data(), _PacketHeader.PayloadLen);
memcpy(data(), buf.data, dstsize);
_PacketHeader.PayloadLen = static_cast<unsigned int>(dstsize);
_PacketHeader.UnCompressedlen = 0;
Remote_Access_Library::INTERNAL::_PacketBuffer.ReleaseBuffer(buf);
}
static int zstd_uncompress(void *dest, void *src, int size, int outsize,
int *error)
{
const size_t res = ZSTD_decompress(dest, outsize, src, size);
if (ZSTD_isError(res)) {
fprintf(stderr, "\t%d %d\n", outsize, size);
*error = (int)ZSTD_getErrorCode(res);
return -1;
}
return (int)res;
}
ByteArray Decompress(const ByteArrayView view)
{
const uint64 originalSize = ZSTD_getDecompressedSize(view.data(), view.size());
if (originalSize == 0)
{
return ByteArray();
}
Array<Byte> outputBuffer(static_cast<size_t>(originalSize));
const size_t decompressedSize = ZSTD_decompress(outputBuffer.data(), outputBuffer.size(), view.data(), view.size());
if (originalSize != decompressedSize)
{
return ByteArray();
}
return ByteArray(std::move(outputBuffer));
}
std::unique_ptr<IOBuf> ZSTDCodec::doUncompress(const IOBuf* data,
uint64_t uncompressedLength) {
size_t rc;
auto out = IOBuf::createCombined(uncompressedLength);
CHECK_GE(out->capacity(), uncompressedLength);
CHECK_EQ(out->length(), 0);
rc = ZSTD_decompress(
out->writableTail(), out->capacity(), data->data(), data->length());
if (ZSTD_isError(rc)) {
throw std::runtime_error(to<std::string>(
"ZSTD decompression returned an error: ",
ZSTD_getErrorName(rc)));
}
out->append(rc);
CHECK_EQ(out->length(), rc);
return out;
}
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = loadFile_X(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown \n", fname);
exit(5);
}
void* const rBuff = malloc_X(rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
if (dSize != rSize) {
printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
free(rBuff);
free(cBuff);
}
int64_t lzbench_zstd_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*)
{
return ZSTD_decompress(outbuf, outsize, inbuf, insize);
}
/*
* Setup state for decoding a strip.
*/
static int
LERCPreDecode(TIFF* tif, uint16 s)
{
static const char module[] = "LERCPreDecode";
lerc_status lerc_ret;
TIFFDirectory *td = &tif->tif_dir;
LERCState* sp = DecoderState(tif);
int lerc_data_type;
unsigned int infoArray[8];
unsigned nomask_bands = td->td_samplesperpixel;
int ndims;
int use_mask = 0;
uint8* lerc_data = tif->tif_rawcp;
unsigned int lerc_data_size = (unsigned int)tif->tif_rawcc;
(void) s;
assert(sp != NULL);
lerc_data_type = GetLercDataType(tif);
if( lerc_data_type < 0 )
return 0;
if( !SetupUncompressedBuffer(tif, sp, module) )
return 0;
if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
if( sp->compressed_size < sp->uncompressed_alloc )
{
_TIFFfree(sp->compressed_buffer);
sp->compressed_buffer = _TIFFmalloc(sp->uncompressed_alloc);
if( !sp->compressed_buffer )
{
sp->compressed_size = 0;
return 0;
}
sp->compressed_size = sp->uncompressed_alloc;
}
}
if( sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE )
{
z_stream strm;
int zlib_ret;
memset(&strm, 0, sizeof(strm));
strm.zalloc = NULL;
strm.zfree = NULL;
strm.opaque = NULL;
zlib_ret = inflateInit(&strm);
if( zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"inflateInit() failed");
inflateEnd(&strm);
return 0;
}
strm.avail_in = (uInt)tif->tif_rawcc;
strm.next_in = tif->tif_rawcp;
strm.avail_out = sp->compressed_size;
strm.next_out = sp->compressed_buffer;
zlib_ret = inflate(&strm, Z_FINISH);
if( zlib_ret != Z_STREAM_END && zlib_ret != Z_OK )
{
TIFFErrorExt(tif->tif_clientdata, module,
"inflate() failed");
inflateEnd(&strm);
return 0;
}
lerc_data = sp->compressed_buffer;
lerc_data_size = sp->compressed_size - strm.avail_out;
inflateEnd(&strm);
}
else if( sp->additional_compression == LERC_ADD_COMPRESSION_ZSTD )
{
#ifdef ZSTD_SUPPORT
size_t zstd_ret;
zstd_ret = ZSTD_decompress(sp->compressed_buffer,
sp->compressed_size,
tif->tif_rawcp,
tif->tif_rawcc);
if( ZSTD_isError(zstd_ret) ) {
TIFFErrorExt(tif->tif_clientdata, module,
"Error in ZSTD_decompress(): %s",
ZSTD_getErrorName(zstd_ret));
return 0;
}
lerc_data = sp->compressed_buffer;
lerc_data_size = (unsigned int)zstd_ret;
#else
TIFFErrorExt(tif->tif_clientdata, module, "ZSTD support missing");
return 0;
#endif
}
else if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unhandled additional compression");
return 0;
}
lerc_ret = lerc_getBlobInfo(
lerc_data,
lerc_data_size,
infoArray,
NULL,
8,
0);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_getBlobInfo() failed");
return 0;
}
/* If the configuration is compatible of a LERC mask, and that the */
/* LERC info has dim == samplesperpixel - 1, then there is a LERC */
/* mask. */
if( td->td_planarconfig == PLANARCONFIG_CONTIG &&
td->td_extrasamples > 0 &&
td->td_sampleinfo[td->td_extrasamples-1] == EXTRASAMPLE_UNASSALPHA &&
GetLercDataType(tif) == 1 &&
infoArray[2] == td->td_samplesperpixel - 1U )
{
use_mask = 1;
nomask_bands --;
}
ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ?
nomask_bands : 1;
/* Info returned in infoArray is { version, dataType, nDim, nCols,
nRows, nBands, nValidPixels, blobSize } */
if( infoArray[0] != (unsigned)sp->lerc_version )
{
TIFFWarningExt(tif->tif_clientdata, module,
"Unexpected version number: %d. Expected: %d",
infoArray[0], sp->lerc_version);
}
if( infoArray[1] != (unsigned)lerc_data_type )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected dataType: %d. Expected: %d",
infoArray[1], lerc_data_type);
return 0;
}
if( infoArray[2] != (unsigned)ndims )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nDim: %d. Expected: %d",
infoArray[2], ndims);
return 0;
}
if( infoArray[3] != sp->segment_width )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nCols: %d. Expected: %du",
infoArray[3], sp->segment_width);
return 0;
}
if( infoArray[4] != sp->segment_height )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nRows: %d. Expected: %u",
infoArray[4], sp->segment_height);
return 0;
}
if( infoArray[5] != 1 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected nBands: %d. Expected: %d",
infoArray[5], 1);
return 0;
}
if( infoArray[7] != lerc_data_size )
{
TIFFErrorExt(tif->tif_clientdata, module,
"Unexpected blobSize: %d. Expected: %u",
infoArray[7],
lerc_data_size);
return 0;
}
lerc_ret = lerc_decode(
lerc_data,
lerc_data_size,
use_mask ? sp->mask_buffer : NULL,
ndims,
sp->segment_width,
sp->segment_height,
1,
lerc_data_type,
sp->uncompressed_buffer);
if( lerc_ret != 0 )
{
TIFFErrorExt(tif->tif_clientdata, module,
"lerc_decode() failed");
return 0;
}
/* Interleave alpha mask with other samples. */
if( use_mask )
{
unsigned src_stride =
(td->td_samplesperpixel - 1) * (td->td_bitspersample / 8);
unsigned dst_stride =
td->td_samplesperpixel * (td->td_bitspersample / 8);
unsigned i = sp->segment_width * sp->segment_height;
/* Operate from end to begin to be able to move in place */
while( i > 0 && i > nomask_bands )
{
i --;
sp->uncompressed_buffer[
i * dst_stride + td->td_samplesperpixel - 1] =
255 * sp->mask_buffer[i];
memcpy( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
src_stride );
}
/* First pixels must use memmove due to overlapping areas */
while( i > 0 )
{
i --;
sp->uncompressed_buffer[
i * dst_stride + td->td_samplesperpixel - 1] =
255 * sp->mask_buffer[i];
memmove( sp->uncompressed_buffer + i * dst_stride,
sp->uncompressed_buffer + i * src_stride,
src_stride );
}
}
return 1;
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* cNoiseBuffer[5];
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
BYTE* mirrorBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* refCtx;
ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
refCtx = ZSTD_createCCtx();
ctx = ZSTD_createCCtx();
dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
srcBuffer = cNoiseBuffer[2];
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig, crcDest;
int cLevel;
BYTE* sampleBuffer;
const BYTE* dict;
size_t dictSize;
/* init */
if (nbTests >= testNb)
{ DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2;
else
{
buffNb >>= 3;
if (buffNb & 7)
{
const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3];
}
else
{
const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3];
}
}
srcBuffer = cNoiseBuffer[buffNb];
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
/* create sample buffer (to catch read error with valgrind & sanitizers) */
sampleBuffer = (BYTE*)malloc(sampleSize);
CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression test */
cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
/* compression failure test : too small dest buffer */
if (cSize > 3)
{
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
static const U32 endMark = 0x4DC2B1A9;
U32 endCheck;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
}
/* successfull decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
free(sampleBuffer); /* no longer useful after this point */
/* truncated src decompression test */
{
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)");
free(cBufferTooSmall);
}
/* too small dst decompression test */
if (sampleSize > 3)
{
const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = sampleSize - missing;
static const BYTE token = 0xA9;
dstBuffer[tooSmallSize] = token;
errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
}
/* noisy src decompression test */
if (cSize > 6)
{
const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4));
size_t pos = 4; /* preserve magic number (too easy to detect) */
U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
size_t mask = (1<<nbBits) - 1;
size_t skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
while (pos < cSize)
{
/* add noise */
size_t noiseStart, noiseLength;
nbBits = FUZ_rand(&lseed) % maxNbBits;
if (nbBits>0) nbBits--;
mask = (1<<nbBits) - 1;
noiseLength = (FUZ_rand(&lseed) & mask) + 1;
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength;
/* keep some original src */
nbBits = FUZ_rand(&lseed) % maxNbBits;
mask = (1<<nbBits) - 1;
skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
}
/* decompress noisy source */
{
U32 noiseSrc = FUZ_rand(&lseed) % 5;
const U32 endMark = 0xA9B1C3D6;
U32 endCheck;
srcBuffer = cNoiseBuffer[noiseSrc];
memcpy(dstBuffer+sampleSize, &endMark, 4);
errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
/* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
memcpy(&endCheck, dstBuffer+sampleSize, 4);
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
}
}
/* Streaming compression of scattered segments test */
XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_duplicateCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0; cSize = 0;
for (n=0; n<nbChunks; n++)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
/* avoid invalid dstBufferTooSmall */
break;
if (totalTestSize+sampleSize > maxTestSize) break;
errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize;
}
errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
crcOrig = XXH64_digest(xxh64);
/* streaming decompression test */
errorCode = ZSTD_resetDCtx(dctx);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
totalGenSize += genSize;
totalCSize += inSize;
}
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig)
errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
}
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
size_t result, cSize;
U32 testNb=0;
/* Create compressible test buffer */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
if (!CNBuffer || !compressedBuffer || !decodedBuffer)
{
DISPLAY("Not enough memory, aborting\n");
testResult = 1;
goto _end;
}
RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState);
/* Basic tests */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (ZSTD_isError(result)) goto _output_error;
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
{
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++)
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Dictionary and Duplication tests */
{
ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
const size_t dictSize = 500;
size_t cSizeOrig;
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
result = ZSTD_compressBegin(ctxOrig, 2);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
cSize = 0;
result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
cSizeOrig = cSize;
cSize = 0;
result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
ZSTD_freeCCtx(ctxDuplicated);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeDCtx(dctx);
DISPLAYLEVEL(4, "OK \n");
}
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* block API tests */
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t blockSize = 100 KB;
const size_t dictSize = 16 KB;
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* dictionary block compression */
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(cctx, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize);
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
/* long rle test */
{
size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error;
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result!=sampleSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
}
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
(void)startTest; (void)compressibility;
/* allocation */
srcBuffer = malloc (srcBufferSize);
dstBuffer = malloc (dstBufferSize);
cBuffer = malloc (cBufferSize);
CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled");
/* Create initial sample */
FUZ_generateSynthetic(srcBuffer, srcBufferSize, 0.50, &coreSeed);
/* catch up testNb */
for (testNb=0; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for (testNb=startTest; testNb < nbTests; testNb++)
{
size_t sampleSize, sampleStart;
size_t cSize, dSize, dSupSize;
U32 sampleSizeLog;
U64 crcOrig, crcDest;
/* init */
DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1<<sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
/* compression tests*/
cSize = ZSTD_compress(cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize);
CHECK(ZSTD_isError(cSize), "ZSTD_compress failed");
/* decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s)", ZSTD_getErrorName(dSize));
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
}
DISPLAY("\rAll fuzzer tests completed \n");
_cleanup:
free(srcBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
size_t result, cSize;
U32 testNb=0;
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_generateSynthetic(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
// Basic tests
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH);
if (ZSTD_isError(result)) goto _output_error;
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
{
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++)
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}