int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* cNoiseBuffer[5]; BYTE* srcBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; BYTE* copyBuffer; size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog); BYTE* cBuffer; size_t cBufferSize = ZSTD_compressBound(srcBufferSize); BYTE* dstBuffer; size_t dstBufferSize = srcBufferSize; U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; ZBUFF_CCtx* zc; ZBUFF_DCtx* zd; XXH64_state_t crc64; U32 startTime = FUZ_GetMilliStart(); /* allocation */ zc = ZBUFF_createCCtx(); zd = ZBUFF_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); copyBuffer= (BYTE*)malloc (copyBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ srcBuffer = cNoiseBuffer[2]; memset(copyBuffer, 0x65, copyBufferSize); memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize)); /* make copyBuffer considered initialized */ /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart; size_t cSize; size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; U64 crcOrig, crcDest; /* init */ DISPLAYUPDATE(2, "\r%6u", testNb); if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; buffNb = FUZ_rand(&lseed) & 127; if (buffNb & 7) buffNb=2; /* select buffer */ else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; /* Multi - segments compression test */ XXH64_reset(&crc64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); totalTestSize = 0; cSize = 0; for (n=0; n<nbChunks; n++) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); readSize = sampleSize; /* random size output buffer */ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN (cBufferSize - cSize, sampleSize); errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); XXH64_update(&crc64, srcBuffer+sampleStart, readSize); memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize); cSize += genSize; totalTestSize += readSize; if ((FUZ_rand(&lseed) & 15) == 0) { /* add a few random flushes operations, to mess around */ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN (cBufferSize - cSize, sampleSize); errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode)); cSize += genSize; } if (totalTestSize > maxTestSize) break; } genSize = cBufferSize - cSize; errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (errorCode != 0, "frame epilogue not fully consumed"); cSize += genSize; crcOrig = XXH64_digest(&crc64); /* multi - fragments decompression test */ ZBUFF_decompressInit(zd); totalCSize = 0; totalGenSize = 0; while (totalCSize < cSize) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); readSize = sampleSize; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN(sampleSize, dstBufferSize - totalGenSize); errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize); CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); totalGenSize += genSize; totalCSize += readSize; } CHECK (errorCode != 0, "frame not fully decoded"); CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); /* noisy/erroneous src decompression test */ /* add some noise */ nbChunks = (FUZ_rand(&lseed) & 7) + 2; for (n=0; n<nbChunks; n++) { size_t cStart; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); if (sampleSize > cSize/3) sampleSize = cSize/3; sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); cStart = FUZ_rand(&lseed) % (cSize - sampleSize); memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize); } /* try decompression on noisy data */ ZBUFF_decompressInit(zd); totalCSize = 0; totalGenSize = 0; while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); readSize = sampleSize; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN(sampleSize, dstBufferSize - totalGenSize); errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize); if (ZBUFF_isError(errorCode)) break; /* error correctly detected */ totalGenSize += genSize; totalCSize += readSize; } }
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* cNoiseBuffer[5]; BYTE* srcBuffer; BYTE* cBuffer; BYTE* dstBuffer; BYTE* mirrorBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; size_t dstBufferSize = (size_t)1<<maxSampleLog; size_t cBufferSize = ZSTD_compressBound(dstBufferSize); U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; ZSTD_CCtx* refCtx; ZSTD_CCtx* ctx; ZSTD_DCtx* dctx; U32 startTime = FUZ_GetMilliStart(); /* allocation */ refCtx = ZSTD_createCCtx(); ctx = ZSTD_createCCtx(); dctx= ZSTD_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize); mirrorBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ srcBuffer = cNoiseBuffer[2]; /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize; U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n; XXH64_CREATESTATE_STATIC(xxh64); U64 crcOrig, crcDest; int cLevel; BYTE* sampleBuffer; const BYTE* dict; size_t dictSize; /* init */ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; buffNb = FUZ_rand(&lseed) & 127; if (buffNb & 7) buffNb=2; else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); /* create sample buffer (to catch read error with valgrind & sanitizers) */ sampleBuffer = (BYTE*)malloc(sampleSize); CHECK (sampleBuffer==NULL, "not enough memory for sample buffer"); memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression test */ cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */ cLevel = (FUZ_rand(&lseed) % cLevelMod) +1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); /* compression failure test : too small dest buffer */ if (cSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; static const U32 endMark = 0x4DC2B1A9; U32 endCheck; memcpy(dstBuffer+tooSmallSize, &endMark, 4); errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); memcpy(&endCheck, dstBuffer+tooSmallSize, 4); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } /* successfull decompression tests*/ dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); crcDest = XXH64(dstBuffer, sampleSize, 0); CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); free(sampleBuffer); /* no longer useful after this point */ /* truncated src decompression test */ { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ CHECK(cBufferTooSmall == NULL, "not enough memory !"); memcpy(cBufferTooSmall, cBuffer, tooSmallSize); errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); free(cBufferTooSmall); } /* too small dst decompression test */ if (sampleSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = sampleSize - missing; static const BYTE token = 0xA9; dstBuffer[tooSmallSize] = token; errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); } /* noisy src decompression test */ if (cSize > 6) { const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4)); size_t pos = 4; /* preserve magic number (too easy to detect) */ U32 nbBits = FUZ_rand(&lseed) % maxNbBits; size_t mask = (1<<nbBits) - 1; size_t skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; while (pos < cSize) { /* add noise */ size_t noiseStart, noiseLength; nbBits = FUZ_rand(&lseed) % maxNbBits; if (nbBits>0) nbBits--; mask = (1<<nbBits) - 1; noiseLength = (FUZ_rand(&lseed) & mask) + 1; if ( pos+noiseLength > cSize ) noiseLength = cSize-pos; noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); pos += noiseLength; /* keep some original src */ nbBits = FUZ_rand(&lseed) % maxNbBits; mask = (1<<nbBits) - 1; skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; } /* decompress noisy source */ { U32 noiseSrc = FUZ_rand(&lseed) % 5; const U32 endMark = 0xA9B1C3D6; U32 endCheck; srcBuffer = cNoiseBuffer[noiseSrc]; memcpy(dstBuffer+sampleSize, &endMark, 4); errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */ CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize), "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize); memcpy(&endCheck, dstBuffer+sampleSize, 4); CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); } } /* Streaming compression of scattered segments test */ XXH64_reset(xxh64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); dict = srcBuffer + sampleStart; dictSize = sampleSize; errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize); CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_duplicateCCtx(ctx, refCtx); CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode)); totalTestSize = 0; cSize = 0; for (n=0; n<nbChunks; n++) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) /* avoid invalid dstBufferTooSmall */ break; if (totalTestSize+sampleSize > maxTestSize) break; errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize); CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); totalTestSize += sampleSize; } errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; crcOrig = XXH64_digest(xxh64); /* streaming decompression test */ errorCode = ZSTD_resetDCtx(dctx); CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); ZSTD_decompress_insertDictionary(dctx, dict, dictSize); totalCSize = 0; totalGenSize = 0; while (totalCSize < cSize) { size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx); size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize)); totalGenSize += genSize; totalCSize += inSize; } CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]); }
static int basicUnitTests(U32 seed, double compressibility) { int testResult = 0; void* CNBuffer; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* compressedBuffer; size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); void* decodedBuffer; size_t decodedBufferSize = CNBufferSize; U32 randState = seed; size_t result, cSize, readSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx(); ZBUFF_DCtx* zd = ZBUFF_createDCtx(); /* Create compressible test buffer */ CNBuffer = malloc(CNBufferSize); compressedBuffer = malloc(compressedBufferSize); decodedBuffer = malloc(decodedBufferSize); if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { DISPLAY("Not enough memory, aborting\n"); goto _output_error; } RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState); /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_compressInit(zc, 1); readSize = CNBufferSize; genSize = compressedBufferSize; result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ cSize = genSize; genSize = compressedBufferSize - cSize; result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); if (result != 0) goto _output_error; /* error, or some data not flushed */ cSize += genSize; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInit(zd); readSize = cSize; genSize = CNBufferSize; result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize != cSize) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ { size_t i; DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); for (i=0; i<CNBufferSize; i++) { if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; } DISPLAYLEVEL(4, "OK \n"); } _end: ZBUFF_freeCCtx(zc); ZBUFF_freeDCtx(zd); free(CNBuffer); free(compressedBuffer); free(decodedBuffer); return testResult; _output_error: testResult = 1; DISPLAY("Error detected in Unit tests ! \n"); goto _end; }
static int basicUnitTests(U32 seed, double compressibility) { int testResult = 0; void* CNBuffer; void* compressedBuffer; void* decodedBuffer; U32 randState = seed; size_t result, cSize; U32 testNb=0; /* Create compressible test buffer */ CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); if (!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("Not enough memory, aborting\n"); testResult = 1; goto _end; } RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState); /* Basic tests */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); if (ZSTD_isError(result)) goto _output_error; cSize = result; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); { size_t i; DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) { if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; } DISPLAYLEVEL(4, "OK \n"); } DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; DISPLAYLEVEL(4, "OK \n"); /* Dictionary and Duplication tests */ { ZSTD_CCtx* ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); const size_t dictSize = 500; size_t cSizeOrig; DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++); result = ZSTD_compressBegin(ctxOrig, 2); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig); if (ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); cSize = 0; result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); result = ZSTD_decompress_usingDict(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */ DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++); cSizeOrig = cSize; cSize = 0; result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize); if (ZSTD_isError(result)) goto _output_error; cSize += result; if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */ ZSTD_freeCCtx(ctxDuplicated); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); result = ZSTD_decompress_usingDict(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error; ZSTD_freeDCtx(dctx); DISPLAYLEVEL(4, "OK \n"); } /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3); if (!ZSTD_isError(result)) goto _output_error; if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); ((char*)(CNBuffer))[0] = 1; result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4); if (!ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); const size_t blockSize = 100 KB; const size_t dictSize = 16 KB; /* basic block compression */ DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++); result = ZSTD_compressBegin(cctx, 5); if (ZSTD_isError(result)) goto _output_error; cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); result = ZSTD_resetDCtx(dctx); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; if (result != blockSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); /* dictionary block compression */ DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++); result = ZSTD_compressBegin(cctx, 5); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_compress_insertDictionary(cctx, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); if (ZSTD_isError(cSize)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); result = ZSTD_resetDCtx(dctx); if (ZSTD_isError(result)) goto _output_error; ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize); result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; if (result != blockSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); ZSTD_freeCCtx(cctx); ZSTD_freeDCtx(dctx); } /* long rle test */ { size_t sampleSize = 0; DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); sampleSize += 256 KB - 1; RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState); sampleSize += 96 KB; cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); if (ZSTD_isError(cSize)) goto _output_error; result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; if (result!=sampleSize) goto _output_error; DISPLAYLEVEL(4, "OK \n"); } _end: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); return testResult; _output_error: testResult = 1; DISPLAY("Error detected in Unit tests ! \n"); goto _end; }