static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args) { PyObject *result; const char *source; uint32_t source_size; uint32_t dest_size; uint32_t header_size; size_t cSize; #if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTuple(args, "y#", &source, &source_size)) return NULL; #else if (!PyArg_ParseTuple(args, "s#", &source, &source_size)) return NULL; #endif header_size = sizeof(dest_size); memcpy(&dest_size, source, header_size); result = PyBytes_FromStringAndSize(NULL, dest_size); source += header_size; if (result != NULL && dest_size > 0) { char *dest = PyBytes_AS_STRING(result); cSize = ZSTD_decompress(dest, dest_size, source, source_size - header_size); if (ZSTD_isError(cSize)) PyErr_Format(ZstdError, "Decompression error: %s", ZSTD_getErrorName(cSize)); } return result; }
int testSimpleAPI(void) { size_t const size = strlen(EXPECTED); char* const output = malloc(size); if (!output) { DISPLAY("ERROR: Not enough memory!\n"); return 1; } { size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE); if (ZSTD_isError(ret)) { if (ret == ZSTD_error_prefix_unknown) { DISPLAY("ERROR: Invalid frame magic number, was this compiled " "without legacy support?\n"); } else { DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); } return 1; } if (ret != size) { DISPLAY("ERROR: Wrong decoded size\n"); } } if (memcmp(EXPECTED, output, size) != 0) { DISPLAY("ERROR: Wrong decoded output produced\n"); return 1; } free(output); DISPLAY("Simple API OK\n"); return 0; }
void CompressionCodecZSTD::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { size_t res = ZSTD_decompress(dest, uncompressed_size, source, source_size); if (ZSTD_isError(res)) throw Exception("Cannot ZSTD_decompress: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_DECOMPRESS); }
static void decompress(const char* fname) { size_t cSize; void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==ZSTD_CONTENTSIZE_ERROR) { fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); exit(5); } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { fprintf(stderr, "%s : original size unknown. Use streaming decompression instead.\n", fname); exit(6); } void* const rBuff = malloc_orDie((size_t)rSize); size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); if (dSize != rSize) { fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); exit(7); } /* success */ printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); free(rBuff); free(cBuff); }
static ezResult DecompressZStd(ezArrayPtr<const ezUInt8> pCompressedData, ezDynamicArray<ezUInt8>& out_Data) { size_t uiSize = ZSTD_findDecompressedSize(pCompressedData.GetPtr(), pCompressedData.GetCount()); if (uiSize == ZSTD_CONTENTSIZE_ERROR) { ezLog::Error("Can't decompress since it wasn't compressed with ZStd"); return EZ_FAILURE; } else if (uiSize == ZSTD_CONTENTSIZE_UNKNOWN) { ezLog::Error("Can't decompress since the original size can't be determined, was the data compressed using the streaming variant?"); return EZ_FAILURE; } if (uiSize > std::numeric_limits<ezUInt32>::max()) { ezLog::Error("Can't compress since the output container can't hold enough elements ({0})", static_cast<ezUInt64>(uiSize)); return EZ_FAILURE; } out_Data.SetCountUninitialized(static_cast<ezUInt32>(uiSize)); size_t const uiActualSize = ZSTD_decompress(out_Data.GetData(), uiSize, pCompressedData.GetPtr(), pCompressedData.GetCount()); if (uiActualSize != uiSize) { ezLog::Error("Error during ZStd decompression: '{0}'.", ZSTD_getErrorName(uiActualSize)); return EZ_FAILURE; } return EZ_SUCCESS; }
void SL::Remote_Access_Library::Network::Packet::decompress() { if (_PacketHeader.UnCompressedlen <= 0) return;//allready decompressed auto buf = Remote_Access_Library::INTERNAL::_PacketBuffer.AquireBuffer(_PacketHeader.UnCompressedlen); auto dstsize = ZSTD_decompress(buf.data, _PacketHeader.UnCompressedlen, data(), _PacketHeader.PayloadLen); memcpy(data(), buf.data, dstsize); _PacketHeader.PayloadLen = static_cast<unsigned int>(dstsize); _PacketHeader.UnCompressedlen = 0; Remote_Access_Library::INTERNAL::_PacketBuffer.ReleaseBuffer(buf); }
static int zstd_uncompress(void *dest, void *src, int size, int outsize, int *error) { const size_t res = ZSTD_decompress(dest, outsize, src, size); if (ZSTD_isError(res)) { fprintf(stderr, "\t%d %d\n", outsize, size); *error = (int)ZSTD_getErrorCode(res); return -1; } return (int)res; }
ByteArray Decompress(const ByteArrayView view) { const uint64 originalSize = ZSTD_getDecompressedSize(view.data(), view.size()); if (originalSize == 0) { return ByteArray(); } Array<Byte> outputBuffer(static_cast<size_t>(originalSize)); const size_t decompressedSize = ZSTD_decompress(outputBuffer.data(), outputBuffer.size(), view.data(), view.size()); if (originalSize != decompressedSize) { return ByteArray(); } return ByteArray(std::move(outputBuffer)); }
std::unique_ptr<IOBuf> ZSTDCodec::doUncompress(const IOBuf* data, uint64_t uncompressedLength) { size_t rc; auto out = IOBuf::createCombined(uncompressedLength); CHECK_GE(out->capacity(), uncompressedLength); CHECK_EQ(out->length(), 0); rc = ZSTD_decompress( out->writableTail(), out->capacity(), data->data(), data->length()); if (ZSTD_isError(rc)) { throw std::runtime_error(to<std::string>( "ZSTD decompression returned an error: ", ZSTD_getErrorName(rc))); } out->append(rc); CHECK_EQ(out->length(), rc); return out; }
static void decompress(const char* fname) { size_t cSize; void* const cBuff = loadFile_X(fname, &cSize); unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); if (rSize==0) { printf("%s : original size unknown \n", fname); exit(5); } void* const rBuff = malloc_X(rSize); size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); if (dSize != rSize) { printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); exit(7); } /* success */ printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); free(rBuff); free(cBuff); }
int64_t lzbench_zstd_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*) { return ZSTD_decompress(outbuf, outsize, inbuf, insize); }
/* * Setup state for decoding a strip. */ static int LERCPreDecode(TIFF* tif, uint16 s) { static const char module[] = "LERCPreDecode"; lerc_status lerc_ret; TIFFDirectory *td = &tif->tif_dir; LERCState* sp = DecoderState(tif); int lerc_data_type; unsigned int infoArray[8]; unsigned nomask_bands = td->td_samplesperpixel; int ndims; int use_mask = 0; uint8* lerc_data = tif->tif_rawcp; unsigned int lerc_data_size = (unsigned int)tif->tif_rawcc; (void) s; assert(sp != NULL); lerc_data_type = GetLercDataType(tif); if( lerc_data_type < 0 ) return 0; if( !SetupUncompressedBuffer(tif, sp, module) ) return 0; if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE ) { if( sp->compressed_size < sp->uncompressed_alloc ) { _TIFFfree(sp->compressed_buffer); sp->compressed_buffer = _TIFFmalloc(sp->uncompressed_alloc); if( !sp->compressed_buffer ) { sp->compressed_size = 0; return 0; } sp->compressed_size = sp->uncompressed_alloc; } } if( sp->additional_compression == LERC_ADD_COMPRESSION_DEFLATE ) { z_stream strm; int zlib_ret; memset(&strm, 0, sizeof(strm)); strm.zalloc = NULL; strm.zfree = NULL; strm.opaque = NULL; zlib_ret = inflateInit(&strm); if( zlib_ret != Z_OK ) { TIFFErrorExt(tif->tif_clientdata, module, "inflateInit() failed"); inflateEnd(&strm); return 0; } strm.avail_in = (uInt)tif->tif_rawcc; strm.next_in = tif->tif_rawcp; strm.avail_out = sp->compressed_size; strm.next_out = sp->compressed_buffer; zlib_ret = inflate(&strm, Z_FINISH); if( zlib_ret != Z_STREAM_END && zlib_ret != Z_OK ) { TIFFErrorExt(tif->tif_clientdata, module, "inflate() failed"); inflateEnd(&strm); return 0; } lerc_data = sp->compressed_buffer; lerc_data_size = sp->compressed_size - strm.avail_out; inflateEnd(&strm); } else if( sp->additional_compression == LERC_ADD_COMPRESSION_ZSTD ) { #ifdef ZSTD_SUPPORT size_t zstd_ret; zstd_ret = ZSTD_decompress(sp->compressed_buffer, sp->compressed_size, tif->tif_rawcp, tif->tif_rawcc); if( ZSTD_isError(zstd_ret) ) { TIFFErrorExt(tif->tif_clientdata, module, "Error in ZSTD_decompress(): %s", ZSTD_getErrorName(zstd_ret)); return 0; } lerc_data = sp->compressed_buffer; lerc_data_size = (unsigned int)zstd_ret; #else TIFFErrorExt(tif->tif_clientdata, module, "ZSTD support missing"); return 0; #endif } else if( sp->additional_compression != LERC_ADD_COMPRESSION_NONE ) { TIFFErrorExt(tif->tif_clientdata, module, "Unhandled additional compression"); return 0; } lerc_ret = lerc_getBlobInfo( lerc_data, lerc_data_size, infoArray, NULL, 8, 0); if( lerc_ret != 0 ) { TIFFErrorExt(tif->tif_clientdata, module, "lerc_getBlobInfo() failed"); return 0; } /* If the configuration is compatible of a LERC mask, and that the */ /* LERC info has dim == samplesperpixel - 1, then there is a LERC */ /* mask. */ if( td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_extrasamples > 0 && td->td_sampleinfo[td->td_extrasamples-1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1 && infoArray[2] == td->td_samplesperpixel - 1U ) { use_mask = 1; nomask_bands --; } ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ? nomask_bands : 1; /* Info returned in infoArray is { version, dataType, nDim, nCols, nRows, nBands, nValidPixels, blobSize } */ if( infoArray[0] != (unsigned)sp->lerc_version ) { TIFFWarningExt(tif->tif_clientdata, module, "Unexpected version number: %d. Expected: %d", infoArray[0], sp->lerc_version); } if( infoArray[1] != (unsigned)lerc_data_type ) { TIFFErrorExt(tif->tif_clientdata, module, "Unexpected dataType: %d. Expected: %d", infoArray[1], lerc_data_type); return 0; } if( infoArray[2] != (unsigned)ndims ) { TIFFErrorExt(tif->tif_clientdata, module, "Unexpected nDim: %d. Expected: %d", infoArray[2], ndims); return 0; } if( infoArray[3] != sp->segment_width ) { TIFFErrorExt(tif->tif_clientdata, module, "Unexpected nCols: %d. Expected: %du", infoArray[3], sp->segment_width); return 0; } if( infoArray[4] != sp->segment_height ) { TIFFErrorExt(tif->tif_clientdata, module, "Unexpected nRows: %d. Expected: %u", infoArray[4], sp->segment_height); return 0; } if( infoArray[5] != 1 ) { TIFFErrorExt(tif->tif_clientdata, module, "Unexpected nBands: %d. Expected: %d", infoArray[5], 1); return 0; } if( infoArray[7] != lerc_data_size ) { TIFFErrorExt(tif->tif_clientdata, module, "Unexpected blobSize: %d. Expected: %u", infoArray[7], lerc_data_size); return 0; } lerc_ret = lerc_decode( lerc_data, lerc_data_size, use_mask ? sp->mask_buffer : NULL, ndims, sp->segment_width, sp->segment_height, 1, lerc_data_type, sp->uncompressed_buffer); if( lerc_ret != 0 ) { TIFFErrorExt(tif->tif_clientdata, module, "lerc_decode() failed"); return 0; } /* Interleave alpha mask with other samples. */ if( use_mask ) { unsigned src_stride = (td->td_samplesperpixel - 1) * (td->td_bitspersample / 8); unsigned dst_stride = td->td_samplesperpixel * (td->td_bitspersample / 8); unsigned i = sp->segment_width * sp->segment_height; /* Operate from end to begin to be able to move in place */ while( i > 0 && i > nomask_bands ) { i --; sp->uncompressed_buffer[ i * dst_stride + td->td_samplesperpixel - 1] = 255 * sp->mask_buffer[i]; memcpy( sp->uncompressed_buffer + i * dst_stride, sp->uncompressed_buffer + i * src_stride, src_stride ); } /* First pixels must use memmove due to overlapping areas */ while( i > 0 ) { i --; sp->uncompressed_buffer[ i * dst_stride + td->td_samplesperpixel - 1] = 255 * sp->mask_buffer[i]; memmove( sp->uncompressed_buffer + i * dst_stride, sp->uncompressed_buffer + i * src_stride, src_stride ); } } return 1; }
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* cNoiseBuffer[5]; BYTE* srcBuffer; BYTE* cBuffer; BYTE* dstBuffer; BYTE* mirrorBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; size_t dstBufferSize = (size_t)1<<maxSampleLog; size_t cBufferSize = ZSTD_compressBound(dstBufferSize); U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; ZSTD_CCtx* refCtx; ZSTD_CCtx* ctx; ZSTD_DCtx* dctx; U32 startTime = FUZ_GetMilliStart(); /* allocation */ refCtx = ZSTD_createCCtx(); ctx = ZSTD_createCCtx(); dctx= ZSTD_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize); mirrorBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ srcBuffer = cNoiseBuffer[2]; /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize; U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n; XXH64_CREATESTATE_STATIC(xxh64); U64 crcOrig, crcDest; int cLevel; BYTE* sampleBuffer; const BYTE* dict; size_t dictSize; /* init */ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; buffNb = FUZ_rand(&lseed) & 127; if (buffNb & 7) buffNb=2; else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); /* create sample buffer (to catch read error with valgrind & sanitizers) */ sampleBuffer = (BYTE*)malloc(sampleSize); CHECK (sampleBuffer==NULL, "not enough memory for sample buffer"); memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression test */ cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */ cLevel = (FUZ_rand(&lseed) % cLevelMod) +1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); /* compression failure test : too small dest buffer */ if (cSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; static const U32 endMark = 0x4DC2B1A9; U32 endCheck; memcpy(dstBuffer+tooSmallSize, &endMark, 4); errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); memcpy(&endCheck, dstBuffer+tooSmallSize, 4); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } /* successfull decompression tests*/ dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); crcDest = XXH64(dstBuffer, sampleSize, 0); CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); free(sampleBuffer); /* no longer useful after this point */ /* truncated src decompression test */ { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ CHECK(cBufferTooSmall == NULL, "not enough memory !"); memcpy(cBufferTooSmall, cBuffer, tooSmallSize); errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); free(cBufferTooSmall); } /* too small dst decompression test */ if (sampleSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = sampleSize - missing; static const BYTE token = 0xA9; dstBuffer[tooSmallSize] = token; errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); } /* noisy src decompression test */ if (cSize > 6) { const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4)); size_t pos = 4; /* preserve magic number (too easy to detect) */ U32 nbBits = FUZ_rand(&lseed) % maxNbBits; size_t mask = (1<<nbBits) - 1; size_t skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; while (pos < cSize) { /* add noise */ size_t noiseStart, noiseLength; nbBits = FUZ_rand(&lseed) % maxNbBits; if (nbBits>0) nbBits--; mask = (1<<nbBits) - 1; noiseLength = (FUZ_rand(&lseed) & mask) + 1; if ( pos+noiseLength > cSize ) noiseLength = cSize-pos; noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); pos += noiseLength; /* keep some original src */ nbBits = FUZ_rand(&lseed) % maxNbBits; mask = (1<<nbBits) - 1; skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; } /* decompress noisy source */ { U32 noiseSrc = FUZ_rand(&lseed) % 5; const U32 endMark = 0xA9B1C3D6; U32 endCheck; srcBuffer = cNoiseBuffer[noiseSrc]; memcpy(dstBuffer+sampleSize, &endMark, 4); errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */ CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize), "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize); memcpy(&endCheck, dstBuffer+sampleSize, 4); CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); } } /* Streaming compression of scattered segments test */ XXH64_reset(xxh64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); dict = srcBuffer + sampleStart; dictSize = sampleSize; errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize); CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_duplicateCCtx(ctx, refCtx); CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode)); totalTestSize = 0; cSize = 0; for (n=0; n<nbChunks; n++) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) /* avoid invalid dstBufferTooSmall */ break; if (totalTestSize+sampleSize > maxTestSize) break; errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize); CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); totalTestSize += sampleSize; } errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; crcOrig = XXH64_digest(xxh64); /* streaming decompression test */ errorCode = ZSTD_resetDCtx(dctx); CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); ZSTD_decompress_insertDictionary(dctx, dict, dictSize); totalCSize = 0; totalGenSize = 0; while (totalCSize < cSize) { size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx); size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize)); totalGenSize += genSize; totalCSize += inSize; } CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]); }
static int basicUnitTests(U32 seed, double compressibility) { int testResult = 0; void* CNBuffer; void* compressedBuffer; void* decodedBuffer; U32 randState = seed; size_t result, cSize; U32 testNb=0; /* Create compressible test buffer */ CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); if (!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("Not enough memory, aborting\n"); testResult = 1; goto _end; } RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState); /* Basic tests */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); if (ZSTD_isError(result)) goto _output_error; cSize = result; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); { size_t i; DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) { if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; } DISPLAYLEVEL(4, "OK \n"); } DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; DISPLAYLEVEL(4, "OK \n"); /* Dictionary and Duplication tests */ { ZSTD_CCtx* ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); const size_t dictSize = 500; size_t cSizeOrig; DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++); result = ZSTD_compressBegin(ctxOrig, 2); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig); if (ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); cSize = 0; result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); result = ZSTD_decompress_usingDict(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */ DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++); cSizeOrig = cSize; cSize = 0; result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */ ZSTD_freeCCtx(ctxDuplicated); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); result = ZSTD_decompress_usingDict(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeDCtx(dctx); DISPLAYLEVEL(4, "OK \n"); } /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); ((char*)(CNBuffer))[0] = 1; result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); if (!ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); const size_t blockSize = 100 KB; const size_t dictSize = 16 KB; /* basic block compression */ DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); result = ZSTD_compressBegin(cctx, 5); if (ZSTD_isError(result)) goto _output_error; cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); result = ZSTD_resetDCtx(dctx); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; if (result != blockSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); /* dictionary block compression */ DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++); result = ZSTD_compressBegin(cctx, 5); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_compress_insertDictionary(cctx, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); result = ZSTD_resetDCtx(dctx); if (ZSTD_isError(result)) goto _output_error; ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize); result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; if (result != blockSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); ZSTD_freeCCtx(cctx); ZSTD_freeDCtx(dctx); } /* long rle test */ { size_t sampleSize = 0; DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); sampleSize += 256 KB - 1; RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState); sampleSize += 96 KB; cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); if (ZSTD_isError(cSize)) goto _output_error; result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; if (result!=sampleSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); } _end: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); return testResult; _output_error: testResult = 1; DISPLAY("Error detected in Unit tests ! \n"); goto _end; }
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* srcBuffer; BYTE* cBuffer; BYTE* dstBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; size_t dstBufferSize = (size_t)1<<maxSampleLog; size_t cBufferSize = ZSTD_compressBound(dstBufferSize); U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; (void)startTest; (void)compressibility; /* allocation */ srcBuffer = malloc (srcBufferSize); dstBuffer = malloc (dstBufferSize); cBuffer = malloc (cBufferSize); CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled"); /* Create initial sample */ FUZ_generateSynthetic(srcBuffer, srcBufferSize, 0.50, &coreSeed); /* catch up testNb */ for (testNb=0; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for (testNb=startTest; testNb < nbTests; testNb++) { size_t sampleSize, sampleStart; size_t cSize, dSize, dSupSize; U32 sampleSizeLog; U64 crcOrig, crcDest; /* init */ DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1<<sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0); /* compression tests*/ cSize = ZSTD_compress(cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize); CHECK(ZSTD_isError(cSize), "ZSTD_compress failed"); /* decompression tests*/ dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s)", ZSTD_getErrorName(dSize)); crcDest = XXH64(dstBuffer, sampleSize, 0); CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize); } DISPLAY("\rAll fuzzer tests completed \n"); _cleanup: free(srcBuffer); free(cBuffer); free(dstBuffer); return result; _output_error: result = 1; goto _cleanup; }
static int basicUnitTests(U32 seed, double compressibility) { int testResult = 0; void* CNBuffer; void* compressedBuffer; void* decodedBuffer; U32 randState = seed; size_t result, cSize; U32 testNb=0; // Create compressible test buffer CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); FUZ_generateSynthetic(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); // Basic tests DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH); if (ZSTD_isError(result)) goto _output_error; cSize = result; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); { size_t i; DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) { if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; } DISPLAYLEVEL(4, "OK \n"); } DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); ((char*)(CNBuffer))[0] = 1; result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error; DISPLAYLEVEL(4, "OK \n"); _end: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); return testResult; _output_error: testResult = 1; DISPLAY("Error detected in Unit tests ! \n"); goto _end; }