size_t up_pack_step( void* outbuf_ptr, size_t outbuf_size, size_t* data_left ) {
size_t result, done_size, read_size, chunk_left;
done_size = outbuf_size;
if (lib_pack_left_sz > 0) {
read_size = lib_pack_chunk_sz;
result = ZBUFF_compressContinue( lib_pack_ctx,
outbuf_ptr, &done_size, lib_pack_chunk, &read_size );
} else {
if (lib_pack_flush_sz == 0) {
result = ZBUFF_compressEnd( lib_pack_ctx, outbuf_ptr, &done_size );
} else {
result = ZBUFF_compressFlush( lib_pack_ctx, outbuf_ptr, &done_size );
}
}
if ( !ZBUFF_isError( result ) ) {
if (lib_pack_left_sz > 0) {
lib_pack_chunk += read_size;
lib_pack_chunk_sz -= read_size;
lib_pack_left_sz -= read_size;
chunk_left = lib_pack_chunk_sz;
} else {
lib_pack_data_end = true;
lib_pack_flush_sz = result;
chunk_left = 0;
}
if (data_left != NULL) { *data_left = chunk_left; }
} else {
lib_error = result;
}
return done_size;
}
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + *maxDstSizePtr;
size_t outSize = *maxDstSizePtr;
size_t epilogueSize, remaining;
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
op += outSize;
epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
zbc->outBuffContentSize += epilogueSize;
outSize = oend-op;
zbc->stage = ZBUFFcs_flush;
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
op += outSize;
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
*maxDstSizePtr = op-ostart; /* tells how many bytes were written */
return remaining;
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* cNoiseBuffer[5];
BYTE* srcBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
size_t dstBufferSize = srcBufferSize;
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZBUFF_CCtx* zc;
ZBUFF_DCtx* zd;
XXH64_state_t crc64;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
zc = ZBUFF_createCCtx();
zd = ZBUFF_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
copyBuffer= (BYTE*)malloc (copyBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
srcBuffer = cNoiseBuffer[2];
memset(copyBuffer, 0x65, copyBufferSize);
memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize)); /* make copyBuffer considered initialized */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart;
size_t cSize;
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks;
U64 crcOrig, crcDest;
/* init */
DISPLAYUPDATE(2, "\r%6u", testNb);
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2; /* select buffer */
else
{
buffNb >>= 3;
if (buffNb & 7)
{
const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3];
}
else
{
const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3];
}
}
srcBuffer = cNoiseBuffer[buffNb];
/* Multi - segments compression test */
XXH64_reset(&crc64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
totalTestSize = 0;
cSize = 0;
for (n=0; n<nbChunks; n++)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
readSize = sampleSize;
/* random size output buffer */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
XXH64_update(&crc64, srcBuffer+sampleStart, readSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
cSize += genSize;
totalTestSize += readSize;
if ((FUZ_rand(&lseed) & 15) == 0)
{
/* add a few random flushes operations, to mess around */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
cSize += genSize;
}
if (totalTestSize > maxTestSize) break;
}
genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
CHECK (errorCode != 0, "frame epilogue not fully consumed");
cSize += genSize;
crcOrig = XXH64_digest(&crc64);
/* multi - fragments decompression test */
ZBUFF_decompressInit(zd);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalGenSize += genSize;
totalCSize += readSize;
}
CHECK (errorCode != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
/* noisy/erroneous src decompression test */
/* add some noise */
nbChunks = (FUZ_rand(&lseed) & 7) + 2;
for (n=0; n<nbChunks; n++)
{
size_t cStart;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
if (sampleSize > cSize/3) sampleSize = cSize/3;
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
cStart = FUZ_rand(&lseed) % (cSize - sampleSize);
memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize);
}
/* try decompression on noisy data */
ZBUFF_decompressInit(zd);
totalCSize = 0;
totalGenSize = 0;
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) )
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
if (ZBUFF_isError(errorCode)) break; /* error correctly detected */
totalGenSize += genSize;
totalCSize += readSize;
}
}
