Exemplo n.º 1
0
int main(int argc, char** argv) {
    U32 timestamp = FUZ_GetMilliStart();
    U32 seed=0;
    int seedset=0;
    int argNb;
    int nbTests = NB_ATTEMPTS;
    int testNb = 0;
    int proba = FUZ_COMPRESSIBILITY_DEFAULT;

    // Check command line
    programName = argv[0];
    for(argNb=1; argNb<argc; argNb++)
    {
        char* argument = argv[argNb];

        if(!argument) continue;   // Protection if argument empty

        // Decode command (note : aggregated commands are allowed)
        if (argument[0]=='-')
        {
            if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; }

            while (argument[1]!=0)
            {
                argument++;
                switch(*argument)
                {
                case 'h':
                    return FUZ_usage();
                case 'v':
                    argument++;
                    displayLevel=4;
                    break;
                case 'i':
                    argument++;
                    nbTests=0;
                    while ((*argument>='0') && (*argument<='9'))
                    {
                        nbTests *= 10;
                        nbTests += *argument - '0';
                        argument++;
                    }
                    break;
                case 's':
                    argument++;
                    seed=0; seedset=1;
                    while ((*argument>='0') && (*argument<='9'))
                    {
                        seed *= 10;
                        seed += *argument - '0';
                        argument++;
                    }
                    break;
                case 't':
                    argument++;
                    testNb=0;
                    while ((*argument>='0') && (*argument<='9'))
                    {
                        testNb *= 10;
                        testNb += *argument - '0';
                        argument++;
                    }
                    break;
                case 'p':
                    argument++;
                    proba=0;
                    while ((*argument>='0') && (*argument<='9'))
                    {
                        proba *= 10;
                        proba += *argument - '0';
                        argument++;
                    }
                    if (proba<0) proba=0;
                    if (proba>100) proba=100;
                    break;
                default: ;
                }
            }
        }
    }

    // Get Seed
    printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);

    if (!seedset)
    {
        char userInput[50] = {0};
        printf("Select an Initialisation number (default : random) : ");
        fflush(stdout);
        if ( no_prompt || fgets(userInput, sizeof userInput, stdin) )
        {
            if ( sscanf(userInput, "%u", &seed) == 1 ) {}
            else seed = FUZ_GetMilliSpan(timestamp);
        }
    }
    printf("Seed = %u\n", seed);
    if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);

    FUZ_Issue52();
    FUZ_Issue134();

    if (nbTests<=0) nbTests=1;

    return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100);
}
Exemplo n.º 2
0
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
    BYTE* cNoiseBuffer[5];
    BYTE* srcBuffer;
    size_t srcBufferSize = (size_t)1<<maxSrcLog;
    BYTE* copyBuffer;
    size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
    BYTE* cBuffer;
    size_t cBufferSize   = ZSTD_compressBound(srcBufferSize);
    BYTE* dstBuffer;
    size_t dstBufferSize = srcBufferSize;
    U32 result = 0;
    U32 testNb = 0;
    U32 coreSeed = seed, lseed = 0;
    ZBUFF_CCtx* zc;
    ZBUFF_DCtx* zd;
    XXH64_state_t crc64;
    U32 startTime = FUZ_GetMilliStart();

    /* allocation */
    zc = ZBUFF_createCCtx();
    zd = ZBUFF_createDCtx();
    cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
    copyBuffer= (BYTE*)malloc (copyBufferSize);
    dstBuffer = (BYTE*)malloc (dstBufferSize);
    cBuffer   = (BYTE*)malloc (cBufferSize);
    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
           !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
           "Not enough memory, fuzzer tests cancelled");

    /* Create initial samples */
    RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
    RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
    RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
    RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
    RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
    srcBuffer = cNoiseBuffer[2];
    memset(copyBuffer, 0x65, copyBufferSize);
    memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize));   /* make copyBuffer considered initialized */

    /* catch up testNb */
    for (testNb=1; testNb < startTest; testNb++)
        FUZ_rand(&coreSeed);

    /* test loop */
    for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
    {
        size_t sampleSize, sampleStart;
        size_t cSize;
        size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
        size_t errorCode;
        U32 sampleSizeLog, buffNb, n, nbChunks;
        U64 crcOrig, crcDest;

        /* init */
        DISPLAYUPDATE(2, "\r%6u", testNb);
        if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u   ", nbTests);
        FUZ_rand(&coreSeed);
        lseed = coreSeed ^ prime1;
        buffNb = FUZ_rand(&lseed) & 127;
        if (buffNb & 7) buffNb=2;   /* select buffer */
        else
        {
            buffNb >>= 3;
            if (buffNb & 7)
            {
                const U32 tnb[2] = { 1, 3 };
                buffNb = tnb[buffNb >> 3];
            }
            else
            {
                const U32 tnb[2] = { 0, 4 };
                buffNb = tnb[buffNb >> 3];
            }
        }
        srcBuffer = cNoiseBuffer[buffNb];

        /* Multi - segments compression test */
        XXH64_reset(&crc64, 0);
        nbChunks = (FUZ_rand(&lseed) & 127) + 2;
        sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
        maxTestSize = (size_t)1 << sampleSizeLog;
        maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
        ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
        totalTestSize = 0;
        cSize = 0;
        for (n=0; n<nbChunks; n++)
        {
            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
            readSize = sampleSize;

            /* random size output buffer */
            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            genSize = MIN (cBufferSize - cSize, sampleSize);

            errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
            CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));

            XXH64_update(&crc64, srcBuffer+sampleStart, readSize);
            memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
            cSize += genSize;
            totalTestSize += readSize;

            if ((FUZ_rand(&lseed) & 15) == 0)
            {
                /* add a few random flushes operations, to mess around */
                sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
                sampleSize = (size_t)1 << sampleSizeLog;
                sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
                genSize = MIN (cBufferSize - cSize, sampleSize);
                errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
                CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
                cSize += genSize;
            }

            if (totalTestSize > maxTestSize) break;
        }
        genSize = cBufferSize - cSize;
        errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
        CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
        CHECK (errorCode != 0, "frame epilogue not fully consumed");
        cSize += genSize;
        crcOrig = XXH64_digest(&crc64);

        /* multi - fragments decompression test */
        ZBUFF_decompressInit(zd);
        totalCSize = 0;
        totalGenSize = 0;
        while (totalCSize < cSize)
        {
            sampleSizeLog  = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize  = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            readSize = sampleSize;
            sampleSizeLog  = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize  = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
            errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
            CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
            totalGenSize += genSize;
            totalCSize += readSize;
        }
        CHECK (errorCode != 0, "frame not fully decoded");
        CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
        CHECK (totalCSize != cSize, "compressed data should be fully read")
        crcDest = XXH64(dstBuffer, totalTestSize, 0);
        if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
        CHECK (crcDest!=crcOrig, "decompressed data corrupted");

        /* noisy/erroneous src decompression test */
        /* add some noise */
        nbChunks = (FUZ_rand(&lseed) & 7) + 2;
        for (n=0; n<nbChunks; n++)
        {
            size_t cStart;

            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            if (sampleSize > cSize/3) sampleSize = cSize/3;
            sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
            cStart = FUZ_rand(&lseed) % (cSize - sampleSize);

            memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize);
        }

        /* try decompression on noisy data */
        ZBUFF_decompressInit(zd);
        totalCSize = 0;
        totalGenSize = 0;
        while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) )
        {
            sampleSizeLog  = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize  = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            readSize = sampleSize;
            sampleSizeLog  = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize  = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
            errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
            if (ZBUFF_isError(errorCode)) break;   /* error correctly detected */
            totalGenSize += genSize;
            totalCSize += readSize;
        }
    }
Exemplo n.º 3
0
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
    BYTE* cNoiseBuffer[5];
    BYTE* srcBuffer;
    BYTE* cBuffer;
    BYTE* dstBuffer;
    BYTE* mirrorBuffer;
    size_t srcBufferSize = (size_t)1<<maxSrcLog;
    size_t dstBufferSize = (size_t)1<<maxSampleLog;
    size_t cBufferSize   = ZSTD_compressBound(dstBufferSize);
    U32 result = 0;
    U32 testNb = 0;
    U32 coreSeed = seed, lseed = 0;
    ZSTD_CCtx* refCtx;
    ZSTD_CCtx* ctx;
    ZSTD_DCtx* dctx;
    U32 startTime = FUZ_GetMilliStart();

    /* allocation */
    refCtx = ZSTD_createCCtx();
    ctx = ZSTD_createCCtx();
    dctx= ZSTD_createDCtx();
    cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
    cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
    dstBuffer = (BYTE*)malloc (dstBufferSize);
    mirrorBuffer = (BYTE*)malloc (dstBufferSize);
    cBuffer   = (BYTE*)malloc (cBufferSize);
    CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
           || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
           "Not enough memory, fuzzer tests cancelled");

    /* Create initial samples */
    RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
    RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
    RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
    RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
    RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
    srcBuffer = cNoiseBuffer[2];

    /* catch up testNb */
    for (testNb=1; testNb < startTest; testNb++)
        FUZ_rand(&coreSeed);

    /* test loop */
    for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
    {
        size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
        size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
        U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
        XXH64_CREATESTATE_STATIC(xxh64);
        U64 crcOrig, crcDest;
        int cLevel;
        BYTE* sampleBuffer;
        const BYTE* dict;
        size_t dictSize;

        /* init */
        if (nbTests >= testNb)
             { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
        else { DISPLAYUPDATE(2, "\r%6u      ", testNb); }

        FUZ_rand(&coreSeed);
        lseed = coreSeed ^ prime1;
        buffNb = FUZ_rand(&lseed) & 127;
        if (buffNb & 7) buffNb=2;
        else
        {
            buffNb >>= 3;
            if (buffNb & 7)
            {
                const U32 tnb[2] = { 1, 3 };
                buffNb = tnb[buffNb >> 3];
            }
            else
            {
                const U32 tnb[2] = { 0, 4 };
                buffNb = tnb[buffNb >> 3];
            }
        }
        srcBuffer = cNoiseBuffer[buffNb];
        sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
        sampleSize = (size_t)1 << sampleSizeLog;
        sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
        sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);

        /* create sample buffer (to catch read error with valgrind & sanitizers)  */
        sampleBuffer = (BYTE*)malloc(sampleSize);
        CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
        memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
        crcOrig = XXH64(sampleBuffer, sampleSize, 0);

        /* compression test */
        cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2));   /* use high compression levels with small samples, for speed */
        cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
        cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
        CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");

        /* compression failure test : too small dest buffer */
        if (cSize > 3)
        {
            const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;   /* no problem, as cSize > 4 (frameHeaderSizer) */
            const size_t tooSmallSize = cSize - missing;
            static const U32 endMark = 0x4DC2B1A9;
            U32 endCheck;
            memcpy(dstBuffer+tooSmallSize, &endMark, 4);
            errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
            CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
            memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
            CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
        }

        /* successfull decompression tests*/
        dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
        dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
        CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
        crcDest = XXH64(dstBuffer, sampleSize, 0);
        CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);

        free(sampleBuffer);   /* no longer useful after this point */

        /* truncated src decompression test */
        {
            const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;   /* no problem, as cSize > 4 (frameHeaderSizer) */
            const size_t tooSmallSize = cSize - missing;
            void* cBufferTooSmall = malloc(tooSmallSize);   /* valgrind will catch overflows */
            CHECK(cBufferTooSmall == NULL, "not enough memory !");
            memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
            errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
            CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)");
            free(cBufferTooSmall);
        }

        /* too small dst decompression test */
        if (sampleSize > 3)
        {
            const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1;   /* no problem, as cSize > 4 (frameHeaderSizer) */
            const size_t tooSmallSize = sampleSize - missing;
            static const BYTE token = 0xA9;
            dstBuffer[tooSmallSize] = token;
            errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
            CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
            CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
        }

        /* noisy src decompression test */
        if (cSize > 6)
        {
            const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4));
            size_t pos = 4;   /* preserve magic number (too easy to detect) */
            U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
            size_t mask = (1<<nbBits) - 1;
            size_t skipLength = FUZ_rand(&lseed) & mask;
            pos += skipLength;

            while (pos < cSize)
            {
                /* add noise */
                size_t noiseStart, noiseLength;
                nbBits = FUZ_rand(&lseed) % maxNbBits;
                if (nbBits>0) nbBits--;
                mask = (1<<nbBits) - 1;
                noiseLength = (FUZ_rand(&lseed) & mask) + 1;
                if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
                noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
                memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
                pos += noiseLength;

                /* keep some original src */
                nbBits = FUZ_rand(&lseed) % maxNbBits;
                mask = (1<<nbBits) - 1;
                skipLength = FUZ_rand(&lseed) & mask;
                pos += skipLength;
            }

            /* decompress noisy source */
            {
                U32 noiseSrc = FUZ_rand(&lseed) % 5;
                const U32 endMark = 0xA9B1C3D6;
                U32 endCheck;
                srcBuffer = cNoiseBuffer[noiseSrc];
                memcpy(dstBuffer+sampleSize, &endMark, 4);
                errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
                /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
                CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
                      "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
                memcpy(&endCheck, dstBuffer+sampleSize, 4);
                CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
            }
        }

        /* Streaming compression of scattered segments test */
        XXH64_reset(xxh64, 0);
        nbChunks = (FUZ_rand(&lseed) & 127) + 2;
        sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
        maxTestSize = (size_t)1 << sampleSizeLog;
        maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
        if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;

        sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
        sampleSize = (size_t)1 << sampleSizeLog;
        sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
        sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
        dict = srcBuffer + sampleStart;
        dictSize = sampleSize;

        errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
        CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode));
        errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize);
        CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
        errorCode = ZSTD_duplicateCCtx(ctx, refCtx);
        CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode));
        totalTestSize = 0; cSize = 0;
        for (n=0; n<nbChunks; n++)
        {
            sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
            sampleSize = (size_t)1 << sampleSizeLog;
            sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
            sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);

            if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
                /* avoid invalid dstBufferTooSmall */
                break;
            if (totalTestSize+sampleSize > maxTestSize) break;

            errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
            CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
            cSize += errorCode;

            XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
            memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
            totalTestSize += sampleSize;
        }
        errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
        CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
        cSize += errorCode;
        crcOrig = XXH64_digest(xxh64);

        /* streaming decompression test */
        errorCode = ZSTD_resetDCtx(dctx);
        CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
        ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
        totalCSize = 0;
        totalGenSize = 0;
        while (totalCSize < cSize)
        {
            size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
            size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
            CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
            totalGenSize += genSize;
            totalCSize += inSize;
        }
        CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
        CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
        CHECK (totalCSize != cSize, "compressed data should be fully read")
        crcDest = XXH64(dstBuffer, totalTestSize, 0);
        if (crcDest!=crcOrig)
            errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
        CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u  (%02X!=%02X)",
               (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);

    }
Exemplo n.º 4
0
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
    BYTE* bufferP0    = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP1    = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP15   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP90   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP100  = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferDst   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
    size_t bufferDstSize = BUFFERSIZE+64;
    unsigned testNb, maxSV, tableLog;
    const size_t maxTestSizeMask = 0x1FFFF;
    U32 rootSeed = seed;
    U32 time = FUZ_GetMilliStart();

    generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
    generate (bufferP1  , BUFFERSIZE, 0.01, &rootSeed);
    generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
    generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
    memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);

    if (startTestNb)
    {
        U32 i;
        for (i=0; i<startTestNb; i++)
            FUZ_rand (&rootSeed);
    }

    for (testNb=startTestNb; testNb<totalTest; testNb++)
    {
        BYTE* bufferTest;
        int tag=0;
        U32 roundSeed = rootSeed ^ 0xEDA5B371;
        FUZ_rand(&rootSeed);

        DISPLAYLEVEL (4, "\r test %5u  ", testNb);
        if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
        {
            DISPLAY ("\r test %5u  ", testNb);
            time = FUZ_GetMilliStart();
        }

        /* Compression / Decompression tests */
        {
            /* determine test sample */
            size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
            size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
            size_t sizeCompressed;
            U32 hashOrig;

            if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
            else
            {
                switch(FUZ_rand(&roundSeed) & 3)
                {
                    case 0: bufferTest = bufferP0 + offset; break;
                    case 1: bufferTest = bufferP1 + offset; break;
                    case 2: bufferTest = bufferP90 + offset; break;
                    default : bufferTest = bufferP100 + offset; break;
                }
            }
            DISPLAYLEVEL (4,"%3i ", tag++);;
            hashOrig = XXH32 (bufferTest, sizeOrig, 0);

            /* compress test */
            sizeCompressed = FSE_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
            CHECK(FSE_isError(sizeCompressed), "Compression failed !");

            if (sizeCompressed > 1)   /* don't check uncompressed & rle corner cases */
            {
                /* failed compression test*/
                {
                    size_t errorCode;
                    void* tooSmallDBuffer = malloc(sizeCompressed-1);   /* overflows detected with Valgrind */
                    CHECK(tooSmallDBuffer==NULL, "Not enough memory for tooSmallDBuffer test");
                    errorCode = FSE_compress (tooSmallDBuffer, sizeCompressed-1, bufferTest, sizeOrig);
                    CHECK(errorCode!=0, "Compression should have failed : destination buffer too small");
                    free(tooSmallDBuffer);
                }

                /* decompression test */
                {
                    U32 hashEnd;
                    BYTE saved = (bufferVerif[sizeOrig] = 254);
                    size_t result = FSE_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
                    CHECK(bufferVerif[sizeOrig] != saved, "Output buffer overrun (bufferVerif) : write beyond specified end");
                    CHECK(FSE_isError(result), "Decompression failed");
                    hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
                    CHECK(hashEnd != hashOrig, "Decompressed data corrupted");
                }
            }
        }

        /* Attempt header decoding on bogus data */
        {
            short count[256];
            size_t result;
            DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
            maxSV = 255;
            result = FSE_readNCount (count, &maxSV, &tableLog, bufferTest, FSE_NCOUNTBOUND);
            if (!FSE_isError(result))   /* an error would be normal */
            {
                int checkCount;
                CHECK(result > FSE_NCOUNTBOUND, "FSE_readHeader() reads too far (buffer overflow)");
                CHECK(maxSV > 255, "count table overflow (%u)", maxSV+1);
                checkCount = FUZ_checkCount(count, tableLog, maxSV);
                CHECK(checkCount==-1, "symbol distribution corrupted");
            }
        }

        /* Attempt decompression on bogus data */
        {
            size_t maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
            size_t sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
            BYTE saved = (bufferDst[maxDstSize] = 253);
            size_t result;
            DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
            result = FSE_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
            CHECK(!FSE_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
            CHECK(bufferDst[maxDstSize] != saved, "Output buffer bufferDst corrupted");
        }
    }

    /* exit */
    free (bufferP0);
    free (bufferP1);
    free (bufferP15);
    free (bufferP90);
    free (bufferP100);
    free (bufferDst);
    free (bufferVerif);
}
Exemplo n.º 5
0
Arquivo: sgtest.c Projeto: infidob/lz4
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
{
    unsigned testResult = 0;
    unsigned testNb = 0;
    void* srcBuffer = NULL;
    void* compressedBuffer = NULL;
    void* decodedBuffer = NULL;
    U32 coreRand = seed;
    LZ4F_decompressionContext_t dCtx = NULL;
    LZ4F_compressionContext_t cCtx = NULL;
    size_t result;
    const U32 startTime = FUZ_GetMilliStart();
    XXH64_state_t xxh64;
#   define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
                            DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }

    // backup all allocated addresses, from which we will later select buffers
    const size_t max_buf_size = 131 KB;
    size_t num_buf_size_distribution_deviations = 0;

    LZ4SG_in_t  sg_in_buf_potential [2*MAX_SG_BUFFERS];
    LZ4SG_out_t sg_out_buf_potential[2*MAX_SG_BUFFERS];

    LZ4SG_in_t  sg_cin [MAX_SG_BUFFERS];
    LZ4SG_out_t sg_cout[MAX_SG_BUFFERS];
    LZ4SG_in_t  sg_din [MAX_SG_BUFFERS];
    LZ4SG_out_t sg_dout[MAX_SG_BUFFERS];
    size_t sg_cin_len, sg_cout_len, sg_din_len, sg_dout_len;
    const size_t maxDstSize = LZ4_SG_compressBound(srcDataLength, NELEMS(sg_cin), NELEMS(sg_cout));

    unsigned int i;
    for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
        sg_in_buf_potential [i].sg_base = malloc(max_buf_size);
        sg_in_buf_potential [i].sg_len  = max_buf_size;
        sg_out_buf_potential[i].sg_base = malloc(max_buf_size);
        sg_out_buf_potential[i].sg_len  = max_buf_size;
    }

    /* Init */
    duration *= 1000;

    /* Create buffers */
    result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
    CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
    result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
    CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
    srcBuffer = malloc(srcDataLength);
    CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
    const size_t compressedBufferLength = maxDstSize;
    compressedBuffer = malloc(compressedBufferLength);
    CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
    decodedBuffer = calloc(1, srcDataLength);   /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
    CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
    FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);

    /* jump to requested testNb */
    for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand);   // sync randomizer

    /* main fuzzer test loop */
    for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
    {
        U32 randState = coreRand ^ prime1;
        (void)FUZ_rand(&coreRand);   /* update seed */

        srand48(FUZ_rand(&randState));

        DISPLAYUPDATE(2, "\r%5u   ", testNb);

        const size_t max_src_buf_size = (4 MB > srcDataLength) ? srcDataLength : 4 MB;
        unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(max_src_buf_size-1) - 1)) + 1;
        const size_t min_src_size = 20;
        const size_t min_first_dest_buf_size = 21;
        const size_t min_src_buf_size = 1;
        const size_t min_dst_buf_size = 10;
        size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + min_src_size;
        size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
        size_t cSize;
        size_t dstSize;
        size_t dstSizeBound;
        U64 crcOrig, crcDecoded;

        unsigned int test_selection = FUZ_rand(&randState);
        //TODO: enable lz4f_compress_compatibility_test with LZ4_SG_decompress
        int lz4f_compress_compatibility_test = 0;//(test_selection % 4) == 0;

        if (!lz4f_compress_compatibility_test)
        {
            // SGL compress
            unsigned int buffer_selection = FUZ_rand(&randState);

            if ((buffer_selection & 0xF) == 1)
            {
                // SG compress single source and single target buffers
                sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
                sg_cin[0].sg_len  = srcSize;
                sg_cin_len = 1;
                sg_cout[0].sg_base = compressedBuffer;
                sg_cout[0].sg_len  = compressedBufferLength;
                sg_cout_len = 1;
                dstSizeBound = dstSize = compressedBufferLength;
            }
            else
            {
                // SG compress random number and size source and target buffers
                sg_cin_len  = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);
                sg_cout_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);

                // single source buffer
                if (1 == sg_cin_len) {
                    sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
                    sg_cin[0].sg_len  = srcSize;

                    DISPLAYUPDATE(4, "INFO: single source buf size %i\n", (int)srcSize);
                }
                else {
                    // multiple source buffers
                    if (srcSize > sg_cin_len*max_buf_size/2) {
                        srcSize = sg_cin_len*max_buf_size/2;
                        num_buf_size_distribution_deviations++;
                        DISPLAYUPDATE(4, "NOTE: source buffer total size deviation %i\n", (int)num_buf_size_distribution_deviations);
                    }

                    size_t exact_src_size = 0;
                    unsigned int buf_size_mean = srcSize / sg_cin_len;
                    for (i = 0; i < sg_cin_len; i++) {
                        size_t buf_size = rnd_exponential(buf_size_mean, min_src_buf_size, max_buf_size);
                        DISPLAYUPDATE(4, "INFO: source buf %i size %i\n", i, (int)buf_size);

                        if (srcStart+exact_src_size+buf_size > srcDataLength) {
                            buf_size = srcDataLength-(srcStart+exact_src_size);
                        }
                        sg_cin[i].sg_base = sg_in_buf_potential[i*2+1].sg_base;
                        sg_cin[i].sg_len  = buf_size;
                        memcpy((void *)sg_cin[i].sg_base, (BYTE*)srcBuffer+srcStart+exact_src_size, buf_size);
                        exact_src_size += buf_size;
                        if (srcStart+exact_src_size == srcDataLength) {
                            num_buf_size_distribution_deviations++;
                            sg_cin_len = i+1;
                            DISPLAYUPDATE(4, "NOTE: final source buffer size deviation %i (buffers number limited to %i)\n", (int)num_buf_size_distribution_deviations, (int)sg_cin_len);
                        }
                    }
                    srcSize = exact_src_size;
                }

                // we can now derive the required limit for output
                dstSizeBound = LZ4_SG_compressBound(srcSize, sg_cin_len, sg_cout_len);

                // single target buffer
                if (1 == sg_cout_len) {
                    sg_cout[0].sg_base = compressedBuffer;
                    sg_cout[0].sg_len  = compressedBufferLength;
                }
                else {
                    // multiple target buffers
                    int finalBufferTruncated = 0;
                    dstSize = 0;
                    unsigned int buf_size_mean = dstSizeBound / sg_cout_len;
                    for (i = 0; i < sg_cout_len; i++) {
                        const size_t min_buf_size = (i == 0) ? min_first_dest_buf_size : min_dst_buf_size;
                        size_t buf_size = rnd_exponential(buf_size_mean, min_buf_size, max_buf_size);
                        DISPLAYUPDATE(4, "INFO: target buf %i size %i\n", (int)i, (int)buf_size);

                        if (dstSize+buf_size > dstSizeBound) {
                            buf_size = dstSizeBound-dstSize;
                            finalBufferTruncated = 1;
                        }
                        dstSize += buf_size;

                        sg_cout[i].sg_base = sg_out_buf_potential[i*2+1].sg_base;
                        sg_cout[i].sg_len  = buf_size;
                        if (finalBufferTruncated) {
                            num_buf_size_distribution_deviations++;

                            if (buf_size < min_buf_size) {
                                // merge truncated with previous?
                                if (i > 0) {
                                    sg_cout[i-1].sg_len += buf_size;
                                    if (sg_cout[i-1].sg_len > max_buf_size) {
                                        // skip, too much hassle
                                        DISPLAYUPDATE(4, "NOTE: unable to truncate final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
                                        sg_cout_len = 0; break;
                                    }
                                }
                                else {
                                    // can this happen?
                                    DISPLAYUPDATE(4, "NOTE: unable to truncate first and final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
                                    sg_cout_len = 0; break;
                                }
                                sg_cout_len = i;
                            }
                            else {
                                sg_cout_len = i+1;
                            }
                            DISPLAYUPDATE(4, "NOTE: final target buffer size truncated (%i), buffers number limited to %i, final's size is now %i (deviations %i)\n",
                                    (int)buf_size, (int)sg_cout_len, (int)sg_cout[sg_cout_len-1].sg_len, (int)num_buf_size_distribution_deviations);
                        }
                    }

                    // skip/abort condition
                    if (0 == sg_cout_len) continue;
                }

                if ((buffer_selection & 0xF) == 0) {
                    //TODO: select a random input and output buffer and split it in two,
                    // feeding consecutive addresses as consecutive entries in SGL

                }
            }

            crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);

            size_t sourceSizeOut = srcSize;
            result = LZ4_SG_compress(&sg_cin[0], sg_cin_len, &sg_cout[0], sg_cout_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
            if (((result == 0) || (sourceSizeOut != srcSize)) && (dstSize < dstSizeBound)) {
                // forgive compression failure when output total size is lower than bound
                num_buf_size_distribution_deviations++;
                DISPLAYUPDATE(4, "NOTE: dstSize %i < %i dstSizeBound, compression attempt failed, not totally unexpected (deviations %i), skipping\n",
                        (int)dstSize, (int)dstSizeBound, (int)num_buf_size_distribution_deviations);
                continue;
            }

            CHECK(result <= 0, "Compression failed (error %i)", (int)result);
            CHECK(sourceSizeOut != srcSize, "Compression stopped at %i out of %i", (int)sourceSizeOut, (int)srcSize);
            cSize = result;
        }
        else
        {
            // LZ4F compression - use it in order to verify SGL decompress compatibility with it
            DISPLAYUPDATE(4, "INFO: LZ4F compression\n");

// alternative
//            size_t dstMaxSize = LZ4F_compressFrameBound(srcSize, prefsPtr);
//            DISPLAYLEVEL(3, "compressFrame srcSize %zu dstMaxSize %zu\n",
//                    srcSize, dstMaxSize);
//            cSize = LZ4F_compressFrame(compressedBuffer, dstMaxSize, (char*)srcBuffer + srcStart, srcSize, prefsPtr);
//            CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));

            crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);

            unsigned BSId   = 4 + (FUZ_rand(&randState) & 3);
            unsigned BMId   = FUZ_rand(&randState) & 1;
            unsigned CCflag = FUZ_rand(&randState) & 1;
            unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
            U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
            LZ4F_preferences_t prefs;
            LZ4F_compressOptions_t cOptions;

            LZ4F_preferences_t* prefsPtr = &prefs;
            memset(&prefs, 0, sizeof(prefs));
            memset(&cOptions, 0, sizeof(cOptions));
            prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
            prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
            prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
            prefs.frameInfo.contentSize = frameContentSize;
            prefs.autoFlush = autoflush;
            prefs.compressionLevel = FUZ_rand(&randState) % 5;
            if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;

            const BYTE* ip = (const BYTE*)srcBuffer + srcStart;
            const BYTE* const iend = ip + srcSize;
            BYTE* op = (BYTE*)compressedBuffer;
            BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL);
            unsigned maxBits = FUZ_highbit((U32)srcSize);
            result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
            CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result);
            op += result;
            while (ip < iend)
            {
                unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits;
                size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
                size_t oSize = LZ4F_compressBound(iSize, prefsPtr);
                unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
                if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
                cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);

                DISPLAYLEVEL(3, "compressUpdate ip %d iSize %zu oSize %zu forceFlush %d\n",
                        (int)(ip-((const BYTE*)srcBuffer + srcStart)), iSize, oSize, forceFlush);
                result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
                CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                op += result;
                ip += iSize;

                if (forceFlush)
                {
                    result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
                    CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
                    op += result;
                }
            }
            result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
            CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
            op += result;
            cSize = op-(BYTE*)compressedBuffer;
        }

        //DECOMPRESS
        test_selection = FUZ_rand(&randState);

        if (lz4f_compress_compatibility_test || ((test_selection % 2) == 0))
        {
            //TODO: SGL decompress with random buffer sizes

            // SGL decompress with same buffer sizes used for compression
            // prepare din with cout's data
            sg_din_len  = sg_cout_len;
            for (i = 0; i < sg_din_len; i++) {
                sg_din[i].sg_len  = sg_cout[i].sg_len;
                if (sg_cout[i].sg_len <= max_buf_size) {
                    // enough room to copy - do it
                    sg_din[i].sg_base = sg_in_buf_potential[i*2+0].sg_base;
                    if (sg_din[i].sg_base != sg_cout[i].sg_base) {
                        memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, sg_cout[i].sg_len);
                    }
                }
                else {
                    // this is probably single output buffer - skip copy, use directly
                    sg_din[i].sg_base = sg_cout[i].sg_base;
                }
            }
            // prepare dout to receive decompressed data
            sg_dout_len = sg_cin_len;
            for (i = 0; i < sg_dout_len; i++) {
                sg_dout[i].sg_len  = sg_cin[i].sg_len;
                if (sg_cin[i].sg_len <= max_buf_size) {
                    // enough room to decompress into independent buffer
                    sg_dout[i].sg_base = sg_out_buf_potential[i*2+0].sg_base;
                }
                else {
                    // this is probably single input buffer, use an external output buffer
                    sg_dout[i].sg_base = decodedBuffer;
                }
            }

            size_t sourceSizeOut = cSize;
            size_t maxOutputSize = srcSize;
            int decomp_result = LZ4_SG_decompress(&sg_din[0], sg_din_len, &sg_dout[0], sg_dout_len, &sourceSizeOut, maxOutputSize);
            CHECK(decomp_result <= 0, "SG decompression failed (error %i)", (int)decomp_result);
            CHECK(decomp_result != (int)srcSize, "SG decompression stopped at  %i", (int)decomp_result);

            // verify result checksum
            size_t total_checked = 0;
            XXH64_reset(&xxh64, 1);
            for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
                size_t cur_size = sg_dout[i].sg_len;
                size_t rem = decomp_result - total_checked;
                if (rem < cur_size) cur_size = rem;
                total_checked += cur_size;

                XXH64_update(&xxh64, sg_dout[i].sg_base, cur_size);
            }
            crcDecoded = XXH64_digest(&xxh64);
            if (crcDecoded != crcOrig) {
                DISPLAYLEVEL(1, "checked %i out of %i (crcDecoded %08x, crcOrig %08x)\n",
                        (int)total_checked, decomp_result, (unsigned)crcDecoded, (unsigned)crcOrig);
                // locate error if any
                total_checked = 0;
                for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
                    size_t cur_size = sg_dout[i].sg_len;
                    size_t rem = decomp_result - total_checked;
                    if (rem < cur_size) cur_size = rem;
                    total_checked += cur_size;

                    U64 crc_in  = XXH64(sg_cin [i].sg_base, cur_size, 1);
                    U64 crc_out = XXH64(sg_dout[i].sg_base, cur_size, 1);
                    if (crc_in != crc_out) {
                        locateBuffDiff(sg_cin[i].sg_base, sg_dout[i].sg_base, cur_size);
                        break;
                    }
                }
                DISPLAYLEVEL(1, "checked %i out of %i\n",
                        (int)total_checked, decomp_result);
            }
            CHECK(crcDecoded != crcOrig, "Decompression corruption");
        }
        else
        {
            // prepare compressedBuffer from SGL
            size_t total_copied = 0;
            for (i = 0; i < sg_cout_len; i++) {
                size_t buf_size_bytes = cSize - total_copied;
                if (buf_size_bytes == 0) break;
                if (buf_size_bytes > sg_cout[i].sg_len) buf_size_bytes = sg_cout[i].sg_len;
                if (((char *)compressedBuffer)+total_copied != sg_cout[i].sg_base) {
                    memcpy(((char *)compressedBuffer)+total_copied, sg_cout[i].sg_base, buf_size_bytes);
                }
                total_copied += buf_size_bytes;
            }

            LZ4F_decompressOptions_t dOptions;
            memset(&dOptions, 0, sizeof(dOptions));

            const BYTE* ip = (const BYTE*)compressedBuffer;
            const BYTE* const iend = ip + cSize;
            BYTE* op = (BYTE*)decodedBuffer;
            BYTE* const oend = op + srcDataLength;
            size_t totalOut = 0;
            unsigned maxBits = FUZ_highbit((U32)cSize);
            XXH64_reset(&xxh64, 1);
            if (maxBits < 3) maxBits = 3;
            while (ip < iend)
            {
                unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
                unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
                size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
                size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
                if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
                if (oSize > (size_t)(oend-op)) oSize = oend-op;
                dOptions.stableDst = FUZ_rand(&randState) & 1;
                result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
                if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
                    locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
                CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s ip %d)",
                        (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result), (int)(ip-(const BYTE*)compressedBuffer));
                XXH64_update(&xxh64, op, (U32)oSize);
                totalOut += oSize;
                op += oSize;
                ip += iSize;
            }
            CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
            if (totalOut)   /* otherwise, it's a skippable frame */
            {
                crcDecoded = XXH64_digest(&xxh64);
                if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
                CHECK(crcDecoded != crcOrig, "Decompression corruption");
            }
        }
    }

    DISPLAYLEVEL(2, "\rAll tests completed   \n");

_end:
    LZ4F_freeDecompressionContext(dCtx);
    LZ4F_freeCompressionContext(cCtx);
    free(srcBuffer);
    free(compressedBuffer);
    free(decodedBuffer);
    for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
        free((void *)(sg_in_buf_potential [i].sg_base));
        free(         sg_out_buf_potential[i].sg_base);
    }

    if (num_buf_size_distribution_deviations > 0) {
        DISPLAYLEVEL(2, "NOTE: %i buffer size deviations \n", (int)num_buf_size_distribution_deviations);
    }

    if (pause)
    {
        DISPLAY("press enter to finish \n");
        (void)getchar();
    }
    return testResult;

_output_error:
    testResult = 1;
    goto _end;

    // unreachable
    return -1;
#undef CHECK
}
Exemplo n.º 6
0
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
    BYTE* bufferP0    = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP1    = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP15   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP90   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP100  = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferDst   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
    size_t const bufferDstSize = BUFFERSIZE+64;
    unsigned testNb;
    size_t const maxTestSizeMask = 0x1FFFF;   /* 128 KB - 1 */
    U32 rootSeed = seed;
    U32 time = FUZ_GetMilliStart();

    generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
    generate (bufferP1  , BUFFERSIZE, 0.01, &rootSeed);
    generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
    generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
    memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);
    memset(bufferDst, 0, BUFFERSIZE);

    { U32 u; for (u=0; u<startTestNb; u++) FUZ_rand (&rootSeed); }

    for (testNb=startTestNb; testNb<totalTest; testNb++) {
        U32 roundSeed = rootSeed ^ 0xEDA5B371;
        FUZ_rand(&rootSeed);
        int tag=0;
        BYTE* bufferTest = NULL;

        DISPLAYLEVEL (4, "\r test %5u  ", testNb);
        if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE) {
            DISPLAY ("\r test %5u  ", testNb);
            time = FUZ_GetMilliStart();
        }

        /* Compression / Decompression tests */
        DISPLAYLEVEL (4,"%3i ", tag++);
        {   /* determine test sample */
            size_t const sizeOrig = (FUZ_rand(&roundSeed) & maxTestSizeMask) + 1;
            size_t const offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
            size_t sizeCompressed;
            U32 hashOrig;

            if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
            else {
                switch(FUZ_rand(&roundSeed) & 3)
                {
                    case 0: bufferTest = bufferP0 + offset; break;
                    case 1: bufferTest = bufferP1 + offset; break;
                    case 2: bufferTest = bufferP90 + offset; break;
                    default : bufferTest = bufferP100 + offset; break;
                }
            }
            hashOrig = XXH32 (bufferTest, sizeOrig, 0);

            /* compression test */
            sizeCompressed = HUF_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
            CHECK(HUF_isError(sizeCompressed), "HUF_compress failed");
            if (sizeCompressed > 1) {   /* don't check uncompressed & rle corner cases */
                /* failed compression test */
                {   BYTE const saved = bufferVerif[sizeCompressed-1] = 253;
                    size_t const errorCode = HUF_compress (bufferVerif, sizeCompressed-1, bufferTest, sizeOrig);
                    CHECK(errorCode!=0, "HUF_compress should have failed (too small destination buffer)")
                    CHECK(bufferVerif[sizeCompressed-1] != saved, "HUF_compress w/ too small dst : bufferVerif overflow");
                }

                /* decompression test */
                {   BYTE const saved = bufferVerif[sizeOrig] = 253;
                    size_t const result = HUF_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
                    CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress : bufferVerif overflow");
                    CHECK(HUF_isError(result), "HUF_decompress failed : %s", HUF_getErrorName(result));
                    {   U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
                        if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig);
                        CHECK(hashEnd != hashOrig, "HUF_decompress : Decompressed data corrupted");
                }   }

                /* quad decoder test (more fragile) */
                /*
                if (sizeOrig > 64)
                {   BYTE const saved = bufferVerif[sizeOrig] = 253;
                    size_t const result = HUF_decompress4X6 (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
                    CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress4X6 : bufferVerif overflow");
                    CHECK(HUF_isError(result), "HUF_decompress4X6 failed : %s", HUF_getErrorName(result));
                    {   U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
                        if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig);
                        CHECK(hashEnd != hashOrig, "HUF_decompress4X6 : Decompressed data corrupted");
                }   }
                */

                /* truncated src decompression test */
                if (sizeCompressed>4) {
                    /* note : in some rare cases, the truncated bitStream may still generate by chance a valid output of correct size */
                    size_t const missing = (FUZ_rand(&roundSeed) % (sizeCompressed-3)) + 2;   /* no problem, as sizeCompressed > 4 */
                    size_t const tooSmallSize = sizeCompressed - missing;
                    void* cBufferTooSmall = malloc(tooSmallSize);   /* valgrind will catch read overflows */
                    CHECK(cBufferTooSmall == NULL, "not enough memory !");
                    memcpy(cBufferTooSmall, bufferDst, tooSmallSize);
                    { size_t const errorCode = HUF_decompress(bufferVerif, sizeOrig, cBufferTooSmall, tooSmallSize);
                      CHECK(!HUF_isError(errorCode) && (errorCode!=sizeOrig), "HUF_decompress should have failed ! (truncated src buffer)"); }
                    free(cBufferTooSmall);
            }   }
        }   /* Compression / Decompression tests */

        /* Attempt decompression on bogus data */
        {   size_t const maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
            size_t const sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
            BYTE const saved = (bufferDst[maxDstSize] = 253);
            size_t result;
            DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
            result = HUF_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
            CHECK(!HUF_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
            CHECK(bufferDst[maxDstSize] != saved, "HUF_decompress noise : bufferDst overflow");
        }
    }   /* for (testNb=startTestNb; testNb<totalTest; testNb++) */

    /* exit */
    free (bufferP0);
    free (bufferP1);
    free (bufferP15);
    free (bufferP90);
    free (bufferP100);
    free (bufferDst);
    free (bufferVerif);
}
Exemplo n.º 7
0
static void FUZ_tests (const U32 startSeed, U32 totalTest, U32 startTestNb)
{
    size_t bufferDstSize = BUFFERSIZE*sizeof(U16) + 64;
    U16* bufferP8    = (U16*) malloc (bufferDstSize);
    void* bufferDst  =        malloc (bufferDstSize);
    U16* bufferVerif = (U16*) malloc (bufferDstSize);
    unsigned testNb;
    const size_t maxTestSizeMask = 0x1FFFF;
    U32 time = FUZ_GetMilliStart();
    U32 seed = startSeed;

    generateU16 (bufferP8, BUFFERSIZE, 0.08, seed);

    if (startTestNb)
    {
        U32 i;
        for (i=0; i<startTestNb; i++)
            FUZ_rand (&seed);
    }

    for (testNb=startTestNb; testNb<totalTest; testNb++)
    {
        U16* bufferTest;
        int tag=0;
        U32 roundSeed = seed ^ 0xEDA5B371;
        FUZ_rand(&seed);

        DISPLAYLEVEL (4, "\r test %5u      ", testNb);
        if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
        {
            DISPLAY ("\r test %5u      ", testNb);
            time = FUZ_GetMilliStart();
        }

        /* Compression / Decompression tests */
        {
            size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
            size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
            size_t sizeCompressed;
            U64 hashOrig;
            bufferTest = bufferP8 + offset;

            DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
            hashOrig = XXH64 (bufferTest, sizeOrig * sizeof(U16), 0);
            sizeCompressed = FSE_compressU16 (bufferDst, bufferDstSize, bufferTest, sizeOrig, FSE_MAX_SYMBOL_VALUE, 12);
            CHECK(FSE_isError(sizeCompressed), "\r test %5u : FSE_compressU16 failed !", testNb);
            if (sizeCompressed > 1)   /* don't check uncompressed & rle corner cases */
            {
                U64 hashEnd;
                U16 saved = (bufferVerif[sizeOrig] = 1024 + 250);
                size_t dstSize;
                size_t result;

                /* basic decompression test : should work */
                DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
                result = FSE_decompressU16 (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
                CHECK(bufferVerif[sizeOrig] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
                CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
                hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
                CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);

                /* larger output buffer than necessary : should work */
                DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
                result = FSE_decompressU16 (bufferVerif, sizeOrig + (FUZ_rand(&roundSeed) & 31) + 1, bufferDst, sizeCompressed);
                CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
                hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
                CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);

                /* smaller output buffer than required : should fail */
                DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
                dstSize = (FUZ_rand(&roundSeed) & 31) + 1;
                if (dstSize >= sizeOrig) dstSize = 1;
                dstSize = sizeOrig - dstSize;
                saved = (bufferVerif[dstSize] = 1024 + 250);
                result = FSE_decompressU16 (bufferVerif, dstSize, bufferDst, sizeCompressed);
                CHECK(bufferVerif[dstSize] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
                CHECK(!FSE_isError(result), "\r test %5u : FSE_decompressU16 should have failed ! (origSize = %u shorts, dstSize = %u bytes)", testNb, (U32)sizeOrig, (U32)dstSize);
            }
        }
    }

    /* clean */
    free (bufferP8);
    free (bufferDst);
    free (bufferVerif);
}