unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) { #if 0 // Simple version, good for code maintenance, but unfortunately slow for small inputs XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, input, len); return XXH64_digest(&state); #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); else return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); } # endif if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); else return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); #endif }
size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output) { size_t const prevOutPos = output->pos; /* end the frame */ size_t ret = ZSTD_endStream(zcs->cstream, output); zcs->frameCSize += output->pos - prevOutPos; /* need to flush before doing the rest */ if (ret) return ret; /* frame done */ /* store the frame data for later */ ret = ZSTD_seekable_logFrame( &zcs->framelog, zcs->frameCSize, zcs->frameDSize, zcs->framelog.checksumFlag ? XXH64_digest(&zcs->xxhState) & 0xFFFFFFFFU : 0); if (ret) return ret; /* reset for the next frame */ zcs->frameCSize = 0; zcs->frameDSize = 0; ZSTD_resetCStream(zcs->cstream, 0); if (zcs->framelog.checksumFlag) XXH64_reset(&zcs->xxhState, 0); return 0; }
/* ZSTDMT_flushNextJob() : * output : will be updated with amount of data flushed . * blockToFlush : if >0, the function will block and wait if there is no data available to flush . * @return : amount of data remaining within internal buffer, 1 if unknown but > 0, 0 if no more, or an error code */ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush) { unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask; if (zcs->doneJobID == zcs->nextJobID) return 0; /* all flushed ! */ PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex); while (zcs->jobs[wJobID].jobCompleted==0) { DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID); if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; } /* nothing ready to be flushed => skip */ pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); /* block when nothing available to flush */ } pthread_mutex_unlock(&zcs->jobCompleted_mutex); /* compression job completed : output can be flushed */ { ZSTDMT_jobDescription job = zcs->jobs[wJobID]; if (!job.jobScanned) { if (ZSTD_isError(job.cSize)) { DEBUGLOG(5, "compression error detected "); ZSTDMT_waitForAllJobsCompleted(zcs); ZSTDMT_releaseAllJobResources(zcs); return job.cSize; } ZSTDMT_releaseCCtx(zcs->cctxPool, job.cctx); zcs->jobs[wJobID].cctx = NULL; DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag); if (zcs->params.fParams.checksumFlag) { XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize); if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) { /* write checksum at end of last section */ U32 const checksum = (U32)XXH64_digest(&zcs->xxhState); DEBUGLOG(4, "writing checksum : %08X \n", checksum); MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum); job.cSize += 4; zcs->jobs[wJobID].cSize += 4; } } ZSTDMT_releaseBuffer(zcs->buffPool, job.src); zcs->jobs[wJobID].srcStart = NULL; zcs->jobs[wJobID].src = g_nullBuffer; zcs->jobs[wJobID].jobScanned = 1; } { size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos); DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID); memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite); output->pos += toWrite; job.dstFlushed += toWrite; } if (job.dstFlushed == job.cSize) { /* output buffer fully flushed => move to next one */ ZSTDMT_releaseBuffer(zcs->buffPool, job.dstBuff); zcs->jobs[wJobID].dstBuff = g_nullBuffer; zcs->jobs[wJobID].jobCompleted = 0; zcs->doneJobID++; } else { zcs->jobs[wJobID].dstFlushed = job.dstFlushed; } /* return value : how many bytes left in buffer ; fake it to 1 if unknown but >0 */ if (job.cSize > job.dstFlushed) return (job.cSize - job.dstFlushed); if (zcs->doneJobID < zcs->nextJobID) return 1; /* still some buffer to flush */ zcs->allJobsCompleted = zcs->frameEnded; /* frame completed and entirely flushed */ return 0; /* everything flushed */ } }
static void xxHash64_test(const void *key, int len, uint32_t seed, void *out) { #if 0 if (! state) state = XXH64_createState (); XXH64_reset (state, seed); XXH64_update (state, key, len); *(uint64_t*)out = XXH64_digest (state); #else *(uint64_t*)out = XXH64 (key, len, seed); #endif }
static guint64 rspamd_re_cache_class_id (enum rspamd_re_type type, gpointer type_data, gsize datalen) { XXH64_state_t st; XXH64_reset (&st, 0xdeadbabe); XXH64_update (&st, &type, sizeof (type)); if (datalen > 0) { XXH64_update (&st, type_data, datalen); } return XXH64_digest (&st); }
STDMETHODIMP_(void) CXXH64Hasher::Final(Byte *digest) throw() { UInt64 val = XXH64_digest(_ctx); SetUi64(digest, val); }
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* cNoiseBuffer[5]; BYTE* srcBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; BYTE* copyBuffer; size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog); BYTE* cBuffer; size_t cBufferSize = ZSTD_compressBound(srcBufferSize); BYTE* dstBuffer; size_t dstBufferSize = srcBufferSize; U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; ZBUFF_CCtx* zc; ZBUFF_DCtx* zd; XXH64_state_t crc64; U32 startTime = FUZ_GetMilliStart(); /* allocation */ zc = ZBUFF_createCCtx(); zd = ZBUFF_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); copyBuffer= (BYTE*)malloc (copyBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ srcBuffer = cNoiseBuffer[2]; memset(copyBuffer, 0x65, copyBufferSize); memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize)); /* make copyBuffer considered initialized */ /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart; size_t cSize; size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; U64 crcOrig, crcDest; /* init */ DISPLAYUPDATE(2, "\r%6u", testNb); if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; buffNb = FUZ_rand(&lseed) & 127; if (buffNb & 7) buffNb=2; /* select buffer */ else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; /* Multi - segments compression test */ XXH64_reset(&crc64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); totalTestSize = 0; cSize = 0; for (n=0; n<nbChunks; n++) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); readSize = sampleSize; /* random size output buffer */ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN (cBufferSize - cSize, sampleSize); errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); XXH64_update(&crc64, srcBuffer+sampleStart, readSize); memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize); cSize += genSize; totalTestSize += readSize; if ((FUZ_rand(&lseed) & 15) == 0) { /* add a few random flushes operations, to mess around */ sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN (cBufferSize - cSize, sampleSize); errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode)); cSize += genSize; } if (totalTestSize > maxTestSize) break; } genSize = cBufferSize - cSize; errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (errorCode != 0, "frame epilogue not fully consumed"); cSize += genSize; crcOrig = XXH64_digest(&crc64); /* multi - fragments decompression test */ ZBUFF_decompressInit(zd); totalCSize = 0; totalGenSize = 0; while (totalCSize < cSize) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); readSize = sampleSize; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN(sampleSize, dstBufferSize - totalGenSize); errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize); CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); totalGenSize += genSize; totalCSize += readSize; } CHECK (errorCode != 0, "frame not fully decoded"); CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); /* noisy/erroneous src decompression test */ /* add some noise */ nbChunks = (FUZ_rand(&lseed) & 7) + 2; for (n=0; n<nbChunks; n++) { size_t cStart; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); if (sampleSize > cSize/3) sampleSize = cSize/3; sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); cStart = FUZ_rand(&lseed) % (cSize - sampleSize); memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize); } /* try decompression on noisy data */ ZBUFF_decompressInit(zd); totalCSize = 0; totalGenSize = 0; while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); readSize = sampleSize; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); genSize = MIN(sampleSize, dstBufferSize - totalGenSize); errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize); if (ZBUFF_isError(errorCode)) break; /* error correctly detected */ totalGenSize += genSize; totalCSize += readSize; } }
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { BYTE* cNoiseBuffer[5]; BYTE* srcBuffer; BYTE* cBuffer; BYTE* dstBuffer; BYTE* mirrorBuffer; size_t srcBufferSize = (size_t)1<<maxSrcLog; size_t dstBufferSize = (size_t)1<<maxSampleLog; size_t cBufferSize = ZSTD_compressBound(dstBufferSize); U32 result = 0; U32 testNb = 0; U32 coreSeed = seed, lseed = 0; ZSTD_CCtx* refCtx; ZSTD_CCtx* ctx; ZSTD_DCtx* dctx; U32 startTime = FUZ_GetMilliStart(); /* allocation */ refCtx = ZSTD_createCCtx(); ctx = ZSTD_createCCtx(); dctx= ZSTD_createDCtx(); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize); mirrorBuffer = (BYTE*)malloc (dstBufferSize); cBuffer = (BYTE*)malloc (cBufferSize); CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, "Not enough memory, fuzzer tests cancelled"); /* Create initial samples */ RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ srcBuffer = cNoiseBuffer[2]; /* catch up testNb */ for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); /* test loop */ for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize; U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n; XXH64_CREATESTATE_STATIC(xxh64); U64 crcOrig, crcDest; int cLevel; BYTE* sampleBuffer; const BYTE* dict; size_t dictSize; /* init */ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; buffNb = FUZ_rand(&lseed) & 127; if (buffNb & 7) buffNb=2; else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); /* create sample buffer (to catch read error with valgrind & sanitizers) */ sampleBuffer = (BYTE*)malloc(sampleSize); CHECK (sampleBuffer==NULL, "not enough memory for sample buffer"); memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression test */ cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */ cLevel = (FUZ_rand(&lseed) % cLevelMod) +1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed"); /* compression failure test : too small dest buffer */ if (cSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; static const U32 endMark = 0x4DC2B1A9; U32 endCheck; memcpy(dstBuffer+tooSmallSize, &endMark, 4); errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); memcpy(&endCheck, dstBuffer+tooSmallSize, 4); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } /* successfull decompression tests*/ dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize); CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize); crcDest = XXH64(dstBuffer, sampleSize, 0); CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize); free(sampleBuffer); /* no longer useful after this point */ /* truncated src decompression test */ { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ CHECK(cBufferTooSmall == NULL, "not enough memory !"); memcpy(cBufferTooSmall, cBuffer, tooSmallSize); errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); free(cBufferTooSmall); } /* too small dst decompression test */ if (sampleSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = sampleSize - missing; static const BYTE token = 0xA9; dstBuffer[tooSmallSize] = token; errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); } /* noisy src decompression test */ if (cSize > 6) { const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4)); size_t pos = 4; /* preserve magic number (too easy to detect) */ U32 nbBits = FUZ_rand(&lseed) % maxNbBits; size_t mask = (1<<nbBits) - 1; size_t skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; while (pos < cSize) { /* add noise */ size_t noiseStart, noiseLength; nbBits = FUZ_rand(&lseed) % maxNbBits; if (nbBits>0) nbBits--; mask = (1<<nbBits) - 1; noiseLength = (FUZ_rand(&lseed) & mask) + 1; if ( pos+noiseLength > cSize ) noiseLength = cSize-pos; noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); pos += noiseLength; /* keep some original src */ nbBits = FUZ_rand(&lseed) % maxNbBits; mask = (1<<nbBits) - 1; skipLength = FUZ_rand(&lseed) & mask; pos += skipLength; } /* decompress noisy source */ { U32 noiseSrc = FUZ_rand(&lseed) % 5; const U32 endMark = 0xA9B1C3D6; U32 endCheck; srcBuffer = cNoiseBuffer[noiseSrc]; memcpy(dstBuffer+sampleSize, &endMark, 4); errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */ CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize), "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize); memcpy(&endCheck, dstBuffer+sampleSize, 4); CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); } } /* Streaming compression of scattered segments test */ XXH64_reset(xxh64, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); dict = srcBuffer + sampleStart; dictSize = sampleSize; errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize); CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_duplicateCCtx(ctx, refCtx); CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode)); totalTestSize = 0; cSize = 0; for (n=0; n<nbChunks; n++) { sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSize = (size_t)1 << sampleSizeLog; sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) /* avoid invalid dstBufferTooSmall */ break; if (totalTestSize+sampleSize > maxTestSize) break; errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize); CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); totalTestSize += sampleSize; } errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode)); cSize += errorCode; crcOrig = XXH64_digest(xxh64); /* streaming decompression test */ errorCode = ZSTD_resetDCtx(dctx); CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); ZSTD_decompress_insertDictionary(dctx, dict, dictSize); totalCSize = 0; totalGenSize = 0; while (totalCSize < cSize) { size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx); size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize)); totalGenSize += genSize; totalCSize += inSize; } CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]); }
GArray * rspamd_tokenize_text (gchar *text, gsize len, gboolean is_utf, struct rspamd_config *cfg, GList *exceptions, gboolean compat, guint64 *hash) { rspamd_ftok_t token, buf; const gchar *pos = NULL; gsize l; GArray *res; GList *cur = exceptions; token_get_function func; guint min_len = 0, max_len = 0, word_decay = 0, initial_size = 128; guint64 hv = 0; XXH64_state_t *st; gboolean decay = FALSE; guint64 prob; if (text == NULL) { return NULL; } buf.begin = text; buf.len = len; token.begin = NULL; token.len = 0; if (compat || !is_utf) { func = rspamd_tokenizer_get_word_compat; } else { func = rspamd_tokenizer_get_word; } if (cfg != NULL) { min_len = cfg->min_word_len; max_len = cfg->max_word_len; word_decay = cfg->words_decay; initial_size = word_decay * 2; } res = g_array_sized_new (FALSE, FALSE, sizeof (rspamd_ftok_t), initial_size); st = XXH64_createState (); XXH64_reset (st, 0); while (func (&buf, &pos, &token, &cur, is_utf, &l, FALSE)) { if (l == 0 || (min_len > 0 && l < min_len) || (max_len > 0 && l > max_len)) { token.begin = pos; continue; } if (!decay) { XXH64_update (st, token.begin, token.len); /* Check for decay */ if (word_decay > 0 && res->len > word_decay && pos - text < (gssize)len) { /* Start decay */ gdouble decay_prob; decay = TRUE; hv = XXH64_digest (st); /* We assume that word is 6 symbols length in average */ decay_prob = (gdouble)word_decay / ((len - (pos - text)) / 6.0); if (decay_prob >= 1.0) { prob = G_MAXUINT64; } else { prob = decay_prob * G_MAXUINT64; } } } else { /* Decaying probability */ /* LCG64 x[n] = a x[n - 1] + b mod 2^64 */ hv = 2862933555777941757ULL * hv + 3037000493ULL; if (hv > prob) { token.begin = pos; continue; } } g_array_append_val (res, token); token.begin = pos; } if (!decay) { hv = XXH64_digest (st); } if (hash) { *hash = hv; } XXH64_freeState (st); return res; }
int64 xxHash64Stream::Finish() { Flush(); return XXH64_digest((XXH64_state_t *)context); }
/* * Class: net_jpountz_xxhash_XXHashJNI * Method: XXH64_digest * Signature: (J)J */ JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1digest (JNIEnv *env, jclass cls, jlong state) { return XXH64_digest((XXH64_state_t*) state); }
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration) { unsigned testResult = 0; unsigned testNb = 0; void* srcBuffer = NULL; void* compressedBuffer = NULL; void* decodedBuffer = NULL; U32 coreRand = seed; LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cCtx = NULL; size_t result; const U32 startTime = FUZ_GetMilliStart(); XXH64_state_t xxh64; # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } // backup all allocated addresses, from which we will later select buffers const size_t max_buf_size = 131 KB; size_t num_buf_size_distribution_deviations = 0; LZ4SG_in_t sg_in_buf_potential [2*MAX_SG_BUFFERS]; LZ4SG_out_t sg_out_buf_potential[2*MAX_SG_BUFFERS]; LZ4SG_in_t sg_cin [MAX_SG_BUFFERS]; LZ4SG_out_t sg_cout[MAX_SG_BUFFERS]; LZ4SG_in_t sg_din [MAX_SG_BUFFERS]; LZ4SG_out_t sg_dout[MAX_SG_BUFFERS]; size_t sg_cin_len, sg_cout_len, sg_din_len, sg_dout_len; const size_t maxDstSize = LZ4_SG_compressBound(srcDataLength, NELEMS(sg_cin), NELEMS(sg_cout)); unsigned int i; for (i = 0; i < NELEMS(sg_in_buf_potential); i++) { sg_in_buf_potential [i].sg_base = malloc(max_buf_size); sg_in_buf_potential [i].sg_len = max_buf_size; sg_out_buf_potential[i].sg_base = malloc(max_buf_size); sg_out_buf_potential[i].sg_len = max_buf_size; } /* Init */ duration *= 1000; /* Create buffers */ result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); srcBuffer = malloc(srcDataLength); CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); const size_t compressedBufferLength = maxDstSize; compressedBuffer = malloc(compressedBufferLength); CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); /* jump to requested testNb */ for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer /* main fuzzer test loop */ for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++) { U32 randState = coreRand ^ prime1; (void)FUZ_rand(&coreRand); /* update seed */ srand48(FUZ_rand(&randState)); DISPLAYUPDATE(2, "\r%5u ", testNb); const size_t max_src_buf_size = (4 MB > srcDataLength) ? srcDataLength : 4 MB; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(max_src_buf_size-1) - 1)) + 1; const size_t min_src_size = 20; const size_t min_first_dest_buf_size = 21; const size_t min_src_buf_size = 1; const size_t min_dst_buf_size = 10; size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + min_src_size; size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize); size_t cSize; size_t dstSize; size_t dstSizeBound; U64 crcOrig, crcDecoded; unsigned int test_selection = FUZ_rand(&randState); //TODO: enable lz4f_compress_compatibility_test with LZ4_SG_decompress int lz4f_compress_compatibility_test = 0;//(test_selection % 4) == 0; if (!lz4f_compress_compatibility_test) { // SGL compress unsigned int buffer_selection = FUZ_rand(&randState); if ((buffer_selection & 0xF) == 1) { // SG compress single source and single target buffers sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart; sg_cin[0].sg_len = srcSize; sg_cin_len = 1; sg_cout[0].sg_base = compressedBuffer; sg_cout[0].sg_len = compressedBufferLength; sg_cout_len = 1; dstSizeBound = dstSize = compressedBufferLength; } else { // SG compress random number and size source and target buffers sg_cin_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS); sg_cout_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS); // single source buffer if (1 == sg_cin_len) { sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart; sg_cin[0].sg_len = srcSize; DISPLAYUPDATE(4, "INFO: single source buf size %i\n", (int)srcSize); } else { // multiple source buffers if (srcSize > sg_cin_len*max_buf_size/2) { srcSize = sg_cin_len*max_buf_size/2; num_buf_size_distribution_deviations++; DISPLAYUPDATE(4, "NOTE: source buffer total size deviation %i\n", (int)num_buf_size_distribution_deviations); } size_t exact_src_size = 0; unsigned int buf_size_mean = srcSize / sg_cin_len; for (i = 0; i < sg_cin_len; i++) { size_t buf_size = rnd_exponential(buf_size_mean, min_src_buf_size, max_buf_size); DISPLAYUPDATE(4, "INFO: source buf %i size %i\n", i, (int)buf_size); if (srcStart+exact_src_size+buf_size > srcDataLength) { buf_size = srcDataLength-(srcStart+exact_src_size); } sg_cin[i].sg_base = sg_in_buf_potential[i*2+1].sg_base; sg_cin[i].sg_len = buf_size; memcpy((void *)sg_cin[i].sg_base, (BYTE*)srcBuffer+srcStart+exact_src_size, buf_size); exact_src_size += buf_size; if (srcStart+exact_src_size == srcDataLength) { num_buf_size_distribution_deviations++; sg_cin_len = i+1; DISPLAYUPDATE(4, "NOTE: final source buffer size deviation %i (buffers number limited to %i)\n", (int)num_buf_size_distribution_deviations, (int)sg_cin_len); } } srcSize = exact_src_size; } // we can now derive the required limit for output dstSizeBound = LZ4_SG_compressBound(srcSize, sg_cin_len, sg_cout_len); // single target buffer if (1 == sg_cout_len) { sg_cout[0].sg_base = compressedBuffer; sg_cout[0].sg_len = compressedBufferLength; } else { // multiple target buffers int finalBufferTruncated = 0; dstSize = 0; unsigned int buf_size_mean = dstSizeBound / sg_cout_len; for (i = 0; i < sg_cout_len; i++) { const size_t min_buf_size = (i == 0) ? min_first_dest_buf_size : min_dst_buf_size; size_t buf_size = rnd_exponential(buf_size_mean, min_buf_size, max_buf_size); DISPLAYUPDATE(4, "INFO: target buf %i size %i\n", (int)i, (int)buf_size); if (dstSize+buf_size > dstSizeBound) { buf_size = dstSizeBound-dstSize; finalBufferTruncated = 1; } dstSize += buf_size; sg_cout[i].sg_base = sg_out_buf_potential[i*2+1].sg_base; sg_cout[i].sg_len = buf_size; if (finalBufferTruncated) { num_buf_size_distribution_deviations++; if (buf_size < min_buf_size) { // merge truncated with previous? if (i > 0) { sg_cout[i-1].sg_len += buf_size; if (sg_cout[i-1].sg_len > max_buf_size) { // skip, too much hassle DISPLAYUPDATE(4, "NOTE: unable to truncate final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations); sg_cout_len = 0; break; } } else { // can this happen? DISPLAYUPDATE(4, "NOTE: unable to truncate first and final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations); sg_cout_len = 0; break; } sg_cout_len = i; } else { sg_cout_len = i+1; } DISPLAYUPDATE(4, "NOTE: final target buffer size truncated (%i), buffers number limited to %i, final's size is now %i (deviations %i)\n", (int)buf_size, (int)sg_cout_len, (int)sg_cout[sg_cout_len-1].sg_len, (int)num_buf_size_distribution_deviations); } } // skip/abort condition if (0 == sg_cout_len) continue; } if ((buffer_selection & 0xF) == 0) { //TODO: select a random input and output buffer and split it in two, // feeding consecutive addresses as consecutive entries in SGL } } crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); size_t sourceSizeOut = srcSize; result = LZ4_SG_compress(&sg_cin[0], sg_cin_len, &sg_cout[0], sg_cout_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL); if (((result == 0) || (sourceSizeOut != srcSize)) && (dstSize < dstSizeBound)) { // forgive compression failure when output total size is lower than bound num_buf_size_distribution_deviations++; DISPLAYUPDATE(4, "NOTE: dstSize %i < %i dstSizeBound, compression attempt failed, not totally unexpected (deviations %i), skipping\n", (int)dstSize, (int)dstSizeBound, (int)num_buf_size_distribution_deviations); continue; } CHECK(result <= 0, "Compression failed (error %i)", (int)result); CHECK(sourceSizeOut != srcSize, "Compression stopped at %i out of %i", (int)sourceSizeOut, (int)srcSize); cSize = result; } else { // LZ4F compression - use it in order to verify SGL decompress compatibility with it DISPLAYUPDATE(4, "INFO: LZ4F compression\n"); // alternative // size_t dstMaxSize = LZ4F_compressFrameBound(srcSize, prefsPtr); // DISPLAYLEVEL(3, "compressFrame srcSize %zu dstMaxSize %zu\n", // srcSize, dstMaxSize); // cSize = LZ4F_compressFrame(compressedBuffer, dstMaxSize, (char*)srcBuffer + srcStart, srcSize, prefsPtr); // CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); unsigned BSId = 4 + (FUZ_rand(&randState) & 3); unsigned BMId = FUZ_rand(&randState) & 1; unsigned CCflag = FUZ_rand(&randState) & 1; unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; LZ4F_preferences_t prefs; LZ4F_compressOptions_t cOptions; LZ4F_preferences_t* prefsPtr = &prefs; memset(&prefs, 0, sizeof(prefs)); memset(&cOptions, 0, sizeof(cOptions)); prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId; prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId; prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag; prefs.frameInfo.contentSize = frameContentSize; prefs.autoFlush = autoflush; prefs.compressionLevel = FUZ_rand(&randState) % 5; if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; const BYTE* ip = (const BYTE*)srcBuffer + srcStart; const BYTE* const iend = ip + srcSize; BYTE* op = (BYTE*)compressedBuffer; BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL); unsigned maxBits = FUZ_highbit((U32)srcSize); result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); op += result; while (ip < iend) { unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; size_t oSize = LZ4F_compressBound(iSize, prefsPtr); unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1); if (iSize > (size_t)(iend-ip)) iSize = iend-ip; cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); DISPLAYLEVEL(3, "compressUpdate ip %d iSize %zu oSize %zu forceFlush %d\n", (int)(ip-((const BYTE*)srcBuffer + srcStart)), iSize, oSize, forceFlush); result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; ip += iSize; if (forceFlush) { result = LZ4F_flush(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; } } result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result); op += result; cSize = op-(BYTE*)compressedBuffer; } //DECOMPRESS test_selection = FUZ_rand(&randState); if (lz4f_compress_compatibility_test || ((test_selection % 2) == 0)) { //TODO: SGL decompress with random buffer sizes // SGL decompress with same buffer sizes used for compression // prepare din with cout's data sg_din_len = sg_cout_len; for (i = 0; i < sg_din_len; i++) { sg_din[i].sg_len = sg_cout[i].sg_len; if (sg_cout[i].sg_len <= max_buf_size) { // enough room to copy - do it sg_din[i].sg_base = sg_in_buf_potential[i*2+0].sg_base; if (sg_din[i].sg_base != sg_cout[i].sg_base) { memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, sg_cout[i].sg_len); } } else { // this is probably single output buffer - skip copy, use directly sg_din[i].sg_base = sg_cout[i].sg_base; } } // prepare dout to receive decompressed data sg_dout_len = sg_cin_len; for (i = 0; i < sg_dout_len; i++) { sg_dout[i].sg_len = sg_cin[i].sg_len; if (sg_cin[i].sg_len <= max_buf_size) { // enough room to decompress into independent buffer sg_dout[i].sg_base = sg_out_buf_potential[i*2+0].sg_base; } else { // this is probably single input buffer, use an external output buffer sg_dout[i].sg_base = decodedBuffer; } } size_t sourceSizeOut = cSize; size_t maxOutputSize = srcSize; int decomp_result = LZ4_SG_decompress(&sg_din[0], sg_din_len, &sg_dout[0], sg_dout_len, &sourceSizeOut, maxOutputSize); CHECK(decomp_result <= 0, "SG decompression failed (error %i)", (int)decomp_result); CHECK(decomp_result != (int)srcSize, "SG decompression stopped at %i", (int)decomp_result); // verify result checksum size_t total_checked = 0; XXH64_reset(&xxh64, 1); for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) { size_t cur_size = sg_dout[i].sg_len; size_t rem = decomp_result - total_checked; if (rem < cur_size) cur_size = rem; total_checked += cur_size; XXH64_update(&xxh64, sg_dout[i].sg_base, cur_size); } crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) { DISPLAYLEVEL(1, "checked %i out of %i (crcDecoded %08x, crcOrig %08x)\n", (int)total_checked, decomp_result, (unsigned)crcDecoded, (unsigned)crcOrig); // locate error if any total_checked = 0; for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) { size_t cur_size = sg_dout[i].sg_len; size_t rem = decomp_result - total_checked; if (rem < cur_size) cur_size = rem; total_checked += cur_size; U64 crc_in = XXH64(sg_cin [i].sg_base, cur_size, 1); U64 crc_out = XXH64(sg_dout[i].sg_base, cur_size, 1); if (crc_in != crc_out) { locateBuffDiff(sg_cin[i].sg_base, sg_dout[i].sg_base, cur_size); break; } } DISPLAYLEVEL(1, "checked %i out of %i\n", (int)total_checked, decomp_result); } CHECK(crcDecoded != crcOrig, "Decompression corruption"); } else { // prepare compressedBuffer from SGL size_t total_copied = 0; for (i = 0; i < sg_cout_len; i++) { size_t buf_size_bytes = cSize - total_copied; if (buf_size_bytes == 0) break; if (buf_size_bytes > sg_cout[i].sg_len) buf_size_bytes = sg_cout[i].sg_len; if (((char *)compressedBuffer)+total_copied != sg_cout[i].sg_base) { memcpy(((char *)compressedBuffer)+total_copied, sg_cout[i].sg_base, buf_size_bytes); } total_copied += buf_size_bytes; } LZ4F_decompressOptions_t dOptions; memset(&dOptions, 0, sizeof(dOptions)); const BYTE* ip = (const BYTE*)compressedBuffer; const BYTE* const iend = ip + cSize; BYTE* op = (BYTE*)decodedBuffer; BYTE* const oend = op + srcDataLength; size_t totalOut = 0; unsigned maxBits = FUZ_highbit((U32)cSize); XXH64_reset(&xxh64, 1); if (maxBits < 3) maxBits = 3; while (ip < iend) { unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; if (oSize > (size_t)(oend-op)) oSize = oend-op; dOptions.stableDst = FUZ_rand(&randState) & 1; result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize); CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s ip %d)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result), (int)(ip-(const BYTE*)compressedBuffer)); XXH64_update(&xxh64, op, (U32)oSize); totalOut += oSize; op += oSize; ip += iSize; } CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); if (totalOut) /* otherwise, it's a skippable frame */ { crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize); CHECK(crcDecoded != crcOrig, "Decompression corruption"); } } } DISPLAYLEVEL(2, "\rAll tests completed \n"); _end: LZ4F_freeDecompressionContext(dCtx); LZ4F_freeCompressionContext(cCtx); free(srcBuffer); free(compressedBuffer); free(decodedBuffer); for (i = 0; i < NELEMS(sg_in_buf_potential); i++) { free((void *)(sg_in_buf_potential [i].sg_base)); free( sg_out_buf_potential[i].sg_base); } if (num_buf_size_distribution_deviations > 0) { DISPLAYLEVEL(2, "NOTE: %i buffer size deviations \n", (int)num_buf_size_distribution_deviations); } if (pause) { DISPLAY("press enter to finish \n"); (void)getchar(); } return testResult; _output_error: testResult = 1; goto _end; // unreachable return -1; #undef CHECK }
/* Returns the xxhash of the data produced so far */ XXH64_hash_t SEQ_digest(SEQ_stream const* stream) { return XXH64_digest(&stream->xxh); }