Exemplo n.º 1
0
static int zstd_compress(void *strm, void *dest, void *src, int size,
			 int block_size, int *error)
{
	const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,
					     src, size, compression_level);

	if (ZSTD_isError(res)) {
		/* FIXME:
		 * zstd does not expose stable error codes. The error enum may
		 * change between versions. Until upstream zstd stablizes the
		 * error codes, we have no way of knowing why the error occurs.
		 * zstd shouldn't fail to compress any input unless there isn't
		 * enough output space. We assume that is the cause and return
		 * the special error code for not enough output space.
		 */
		return 0;
	}

	return (int)res;
}
Exemplo n.º 2
0
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
    BYTE* cNoiseBuffer[5];
    BYTE* srcBuffer;
    BYTE* cBuffer;
    BYTE* dstBuffer;
    BYTE* mirrorBuffer;
    size_t srcBufferSize = (size_t)1<<maxSrcLog;
    size_t dstBufferSize = (size_t)1<<maxSampleLog;
    size_t cBufferSize   = ZSTD_compressBound(dstBufferSize);
    U32 result = 0;
    U32 testNb = 0;
    U32 coreSeed = seed, lseed = 0;
    ZSTD_CCtx* refCtx;
    ZSTD_CCtx* ctx;
    ZSTD_DCtx* dctx;
    U32 startTime = FUZ_GetMilliStart();

    /* allocation */
    refCtx = ZSTD_createCCtx();
    ctx = ZSTD_createCCtx();
    dctx= ZSTD_createDCtx();
    cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
    dstBuffer = (BYTE*)malloc (dstBufferSize);
    mirrorBuffer = (BYTE*)malloc (dstBufferSize);
    cBuffer   = (BYTE*)malloc (cBufferSize);
    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
           || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
           "Not enough memory, fuzzer tests cancelled");

    /* Create initial samples */
    RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
    RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
    RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
    RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
    RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
    srcBuffer = cNoiseBuffer[2];

    /* catch up testNb */
    for (testNb=1; testNb < startTest; testNb++)
        FUZ_rand(&coreSeed);

    /* test loop */
    for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
    {
        size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
        size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
        U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
        XXH64_CREATESTATE_STATIC(xxh64);
        U64 crcOrig, crcDest;
        int cLevel;
        BYTE* sampleBuffer;
        const BYTE* dict;
        size_t dictSize;

        /* init */
        if (nbTests >= testNb)
             { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
        else { DISPLAYUPDATE(2, "\r%6u      ", testNb); }

        FUZ_rand(&coreSeed);
        lseed = coreSeed ^ prime1;
        buffNb = FUZ_rand(&lseed) & 127;
        if (buffNb & 7) buffNb=2;
        else
        {
            buffNb >>= 3;
            if (buffNb & 7)
            {
                const U32 tnb[2] = { 1, 3 };
                buffNb = tnb[buffNb >> 3];
            }
            else
            {
                const U32 tnb[2] = { 0, 4 };
                buffNb = tnb[buffNb >> 3];
            }
        }
        srcBuffer = cNoiseBuffer[buffNb];
        sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
        sampleSize = (size_t)1 << sampleSizeLog;
        sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
        sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);

        /* create sample buffer (to catch read error with valgrind & sanitizers)  */
        sampleBuffer = (BYTE*)malloc(sampleSize);
        CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
        memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
        crcOrig = XXH64(sampleBuffer, sampleSize, 0);

        /* compression test */
        cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2));   /* use high compression levels with small samples, for speed */
        cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
        cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
        CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");

        /* compression failure test : too small dest buffer */
        if (cSize > 3)
        {
            const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;   /* no problem, as cSize > 4 (frameHeaderSizer) */
            const size_t tooSmallSize = cSize - missing;
            static const U32 endMark = 0x4DC2B1A9;
            U32 endCheck;
            memcpy(dstBuffer+tooSmallSize, &endMark, 4);
            errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
            CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
            memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
            CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
        }

        /* successfull decompression tests*/
        dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
        dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
        CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
        crcDest = XXH64(dstBuffer, sampleSize, 0);
        CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);

        free(sampleBuffer);   /* no longer useful after this point */

        /* truncated src decompression test */
        {
            const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;   /* no problem, as cSize > 4 (frameHeaderSizer) */
            const size_t tooSmallSize = cSize - missing;
            void* cBufferTooSmall = malloc(tooSmallSize);   /* valgrind will catch overflows */
            CHECK(cBufferTooSmall == NULL, "not enough memory !");
            memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
            errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
            CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)");
            free(cBufferTooSmall);
        }

        /* too small dst decompression test */
        if (sampleSize > 3)
        {
            const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1;   /* no problem, as cSize > 4 (frameHeaderSizer) */
            const size_t tooSmallSize = sampleSize - missing;
            static const BYTE token = 0xA9;
            dstBuffer[tooSmallSize] = token;
            errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
            CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
            CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
        }

        /* noisy src decompression test */
        if (cSize > 6)
        {
            const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4));
            size_t pos = 4;   /* preserve magic number (too easy to detect) */
            U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
            size_t mask = (1<<nbBits) - 1;
            size_t skipLength = FUZ_rand(&lseed) & mask;
            pos += skipLength;

            while (pos < cSize)
            {
                /* add noise */
                size_t noiseStart, noiseLength;
                nbBits = FUZ_rand(&lseed) % maxNbBits;
                if (nbBits>0) nbBits--;
                mask = (1<<nbBits) - 1;
                noiseLength = (FUZ_rand(&lseed) & mask) + 1;
                if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
                noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
                memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
                pos += noiseLength;

                /* keep some original src */
                nbBits = FUZ_rand(&lseed) % maxNbBits;
                mask = (1<<nbBits) - 1;
                skipLength = FUZ_rand(&lseed) & mask;
                pos += skipLength;
            }

            /* decompress noisy source */
            {
                U32 noiseSrc = FUZ_rand(&lseed) % 5;
                const U32 endMark = 0xA9B1C3D6;
                U32 endCheck;
                srcBuffer = cNoiseBuffer[noiseSrc];
                memcpy(dstBuffer+sampleSize, &endMark, 4);
                errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
                /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
                CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
                      "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
                memcpy(&endCheck, dstBuffer+sampleSize, 4);
                CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
            }
        }

        /* Streaming compression of scattered segments test */
        XXH64_reset(xxh64, 0);
        nbChunks = (FUZ_rand(&lseed) & 127) + 2;
        sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
        maxTestSize = (size_t)1 << sampleSizeLog;
        maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
        if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;

        sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
        sampleSize = (size_t)1 << sampleSizeLog;
        sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
        sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
        dict = srcBuffer + sampleStart;
        dictSize = sampleSize;

        errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
        CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode));
        errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize);
        CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
        errorCode = ZSTD_duplicateCCtx(ctx, refCtx);
        CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode));
        totalTestSize = 0; cSize = 0;
        for (n=0; n<nbChunks; n++)
        {
            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);

            if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
                /* avoid invalid dstBufferTooSmall */
                break;
            if (totalTestSize+sampleSize > maxTestSize) break;

            errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
            CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
            cSize += errorCode;

            XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
            memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
            totalTestSize += sampleSize;
        }
        errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
        CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
        cSize += errorCode;
        crcOrig = XXH64_digest(xxh64);

        /* streaming decompression test */
        errorCode = ZSTD_resetDCtx(dctx);
        CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
        ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
        totalCSize = 0;
        totalGenSize = 0;
        while (totalCSize < cSize)
        {
            size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
            size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
            CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
            totalGenSize += genSize;
            totalCSize += inSize;
        }
        CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
        CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
        CHECK (totalCSize != cSize, "compressed data should be fully read")
        crcDest = XXH64(dstBuffer, totalTestSize, 0);
        if (crcDest!=crcOrig)
            errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
        CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u  (%02X!=%02X)",
               (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);

    }
Exemplo n.º 3
0
size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
                           void* dst, size_t dstCapacity,
                     const void* src, size_t srcSize,
                           int compressionLevel)
{
    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
    U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 0 : 3;
    size_t const overlapSize = (size_t)1 << (params.cParams.windowLog - overlapLog);
    size_t const chunkTargetSize = (size_t)1 << (params.cParams.windowLog + 2);
    unsigned const nbChunksMax = (unsigned)(srcSize / chunkTargetSize) + 1;
    unsigned nbChunks = MIN(nbChunksMax, mtctx->nbThreads);
    size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks;
    size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0xFFFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize;   /* avoid too small last block */
    size_t remainingSrcSize = srcSize;
    const char* const srcStart = (const char*)src;
    unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize));  /* presumes avgChunkSize >= 256 KB, which should be the case */
    size_t frameStartPos = 0, dstBufferPos = 0;

    DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes  ", params.cParams.windowLog, (U32)chunkTargetSize);
    DEBUGLOG(2, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
    params.fParams.contentSizeFlag = 1;

    if (nbChunks==1) {   /* fallback to single-thread mode */
        ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
        return ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel);
    }

    {   unsigned u;
        for (u=0; u<nbChunks; u++) {
            size_t const chunkSize = MIN(remainingSrcSize, avgChunkSize);
            size_t const dstBufferCapacity = ZSTD_compressBound(chunkSize);
            buffer_t const dstAsBuffer = { (char*)dst + dstBufferPos, dstBufferCapacity };
            buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity);
            ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(mtctx->cctxPool);
            size_t dictSize = u ? overlapSize : 0;

            if ((cctx==NULL) || (dstBuffer.start==NULL)) {
                mtctx->jobs[u].cSize = ERROR(memory_allocation);   /* job result */
                mtctx->jobs[u].jobCompleted = 1;
                nbChunks = u+1;
                break;   /* let's wait for previous jobs to complete, but don't start new ones */
            }

            mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
            mtctx->jobs[u].dictSize = dictSize;
            mtctx->jobs[u].srcSize = chunkSize;
            mtctx->jobs[u].fullFrameSize = srcSize;
            mtctx->jobs[u].params = params;
            mtctx->jobs[u].dstBuff = dstBuffer;
            mtctx->jobs[u].cctx = cctx;
            mtctx->jobs[u].firstChunk = (u==0);
            mtctx->jobs[u].lastChunk = (u==nbChunks-1);
            mtctx->jobs[u].jobCompleted = 0;
            mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex;
            mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond;

            DEBUGLOG(3, "posting job %u   (%u bytes)", u, (U32)chunkSize);
            DEBUG_PRINTHEX(3, mtctx->jobs[u].srcStart, 12);
            POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);

            frameStartPos += chunkSize;
            dstBufferPos += dstBufferCapacity;
            remainingSrcSize -= chunkSize;
    }   }
    /* note : since nbChunks <= nbThreads, all jobs should be running immediately in parallel */

    {   unsigned chunkID;
        size_t error = 0, dstPos = 0;
        for (chunkID=0; chunkID<nbChunks; chunkID++) {
            DEBUGLOG(3, "waiting for chunk %u ", chunkID);
            PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
            while (mtctx->jobs[chunkID].jobCompleted==0) {
                DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", chunkID);
                pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
            }
            pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
            DEBUGLOG(3, "ready to write chunk %u ", chunkID);

            ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx);
            mtctx->jobs[chunkID].cctx = NULL;
            mtctx->jobs[chunkID].srcStart = NULL;
            {   size_t const cSize = mtctx->jobs[chunkID].cSize;
                if (ZSTD_isError(cSize)) error = cSize;
                if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall);
                if (chunkID) {   /* note : chunk 0 is already written directly into dst */
                    if (!error)
                        memmove((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize);  /* may overlap if chunk decompressed within dst */
                    if (chunkID >= compressWithinDst)   /* otherwise, it decompresses within dst */
                        ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff);
                    mtctx->jobs[chunkID].dstBuff = g_nullBuffer;
                }
                dstPos += cSize ;
            }
        }
        if (!error) DEBUGLOG(3, "compressed size : %u  ", (U32)dstPos);
        return error ? error : dstPos;
    }

}