示例#1
0
size_t FSE_decompressU16(U16* dst, size_t maxDstSize,
                  const void* cSrc, size_t cSrcSize)
{
    const BYTE* const istart = (const BYTE*) cSrc;
    const BYTE* ip = istart;
    short   counting[FSE_MAX_SYMBOL_VALUE+1];
    DTable_max_t dt;
    unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
    unsigned tableLog;
    size_t errorCode;

    /* Sanity check */
    if (cSrcSize<2) return ERROR(srcSize_wrong);   /* specific corner cases (uncompressed & rle) */

    /* normal FSE decoding mode */
    errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
    if (FSE_isError(errorCode)) return errorCode;
    ip += errorCode;
    cSrcSize -= errorCode;
	unsigned scrambler = 1;
    errorCode = FSE_buildDTableU16 (dt, counting, maxSymbolValue, tableLog, scrambler);
    if (FSE_isError(errorCode)) return errorCode;

    return FSE_decompressU16_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
}
示例#2
0
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
{
    const BYTE* const istart = (const BYTE*)cSrc;
    const BYTE* ip = istart;
    short counting[FSE_MAX_SYMBOL_VALUE+1];
    unsigned tableLog;
    unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;

    /* normal FSE decoding mode */
    size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
    if (FSE_isError(NCountLength)) return NCountLength;
    //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong);   /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
    if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
    ip += NCountLength;
    cSrcSize -= NCountLength;

    CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );

    return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace);   /* always return, even if it is an error code */
}
size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
{
    const BYTE* const istart = (const BYTE*)cSrc;
    const BYTE* ip = istart;
    short counting[FSE_MAX_SYMBOL_VALUE+1];
    DTable_max_t dt;   /* Static analyzer seems unable to understand this table will be properly initialized later */
    unsigned tableLog;
    unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;

    if (cSrcSize<2) return ERROR(srcSize_wrong);   /* too small input size */

    /* normal FSE decoding mode */
    {   size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
        if (FSE_isError(NCountLength)) return NCountLength;
        if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong);   /* too small input size */
        ip += NCountLength;
        cSrcSize -= NCountLength;
    }

    CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) );

    return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);   /* always return, even if it is an error code */
}
示例#4
0
static void unitTest(void)
{
    BYTE* testBuff = (BYTE*)malloc(TBSIZE);
    BYTE* cBuff = (BYTE*)malloc(FSE_COMPRESSBOUND(TBSIZE));
    BYTE* verifBuff = (BYTE*)malloc(TBSIZE);
    size_t errorCode;
    U32 seed=0, testNb=0, lseed=0;
    U32 count[256];

    if ((!testBuff) || (!cBuff) || (!verifBuff))
    {
        DISPLAY("Not enough memory, exiting ... \n");
        free(testBuff);
        free(cBuff);
        free(verifBuff);
        return;
    }

    /* FSE_count */
    {
        U32 max, i;
        for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&lseed) & 63) + '0';
        max = '0' + 63;
        errorCode = FSE_count(count, &max, testBuff, TBSIZE);
        CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
        max -= 1;
        errorCode = FSE_count(count, &max, testBuff, TBSIZE);
        CHECK(!FSE_isError(errorCode), "Error : FSE_count() should have failed : value > max");
        max = 65000;
        errorCode = FSE_count(count, &max, testBuff, TBSIZE);
        CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
    }

    /* FSE_optimalTableLog */
    {
        U32 max, i, tableLog=12;
        size_t testSize = 999;
        for (i=0; i< testSize; i++) testBuff[i] = (BYTE)FUZ_rand(&lseed);
        max = 256;
        FSE_count(count, &max, testBuff, testSize);
        tableLog = FSE_optimalTableLog(tableLog, testSize, max);
        CHECK(tableLog<=8, "Too small tableLog");
    }

    /* FSE_normalizeCount */
    {
        S16 norm[256];
        U32 max = 256;
        FSE_count(count, &max, testBuff, TBSIZE);
        errorCode = FSE_normalizeCount(norm, 10, count, TBSIZE, max);
        CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
        errorCode = FSE_normalizeCount(norm, 8, count, TBSIZE, 256);
        CHECK(!FSE_isError(errorCode), "Error : FSE_normalizeCount() should have failed (max >= 1<<tableLog)");
        /* limit corner case : try to make internal rank overflow */
        {
            U32 i;
            U32 total = 0;
            count[0] =  940;
            count[1] =  910;
            count[2] =  470;
            count[3] =  190;
            count[4] =   90;
            for(i=5; i<=255; i++) count[i] = 6;
            for (i=0; i<=255; i++) total += count[i];
            errorCode = FSE_normalizeCount(norm, 10, count, total, 255);
            CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
            count[0] =  300;
            count[1] =  300;
            count[2] =  300;
            count[3] =  300;
            count[4] =   50;
            for(i=5; i<=80; i++) count[i] = 4;
            total = 0; for (i=0; i<=80; i++) total += count[i];
            errorCode = FSE_normalizeCount(norm, 10, count, total, 80);
            CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
        }
    }

    /* FSE_writeNCount, FSE_readNCount */
    {
        S16 norm[129];
        BYTE header[513];
        U32 max, tableLog, i;
        size_t headerSize;

        for (i=0; i< TBSIZE; i++) testBuff[i] = i % 127;
        max = 128;
        errorCode = FSE_count(count, &max, testBuff, TBSIZE);
        CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
        tableLog = FSE_optimalTableLog(0, TBSIZE, max);
        errorCode = FSE_normalizeCount(norm, tableLog, count, TBSIZE, max);
        CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");

        headerSize = FSE_NCountWriteBound(max, tableLog);

        headerSize = FSE_writeNCount(header, 513, norm, max, tableLog);
        CHECK(FSE_isError(headerSize), "Error : FSE_writeNCount() should have worked");

        header[headerSize-1] = 0;
        errorCode = FSE_writeNCount(header, headerSize-1, norm, max, tableLog);
        CHECK(!FSE_isError(errorCode), "Error : FSE_writeNCount() should have failed");
        CHECK (header[headerSize-1] != 0, "Error : FSE_writeNCount() buffer overwrite");

        errorCode = FSE_writeNCount(header, headerSize+1, norm, max, tableLog);
        CHECK(FSE_isError(errorCode), "Error : FSE_writeNCount() should have worked");

        max = 129;
        errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
        CHECK(FSE_isError(errorCode), "Error : FSE_readNCount() should have worked : (error %s)", FSE_getErrorName(errorCode));

        max = 64;
        errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
        CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (max too small)");

        max = 129;
        errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize-1);
        CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");

        {
            void* smallBuffer = malloc(headerSize-1);   /* outbound read can be caught by valgrind */
            CHECK(smallBuffer==NULL, "Error : Not enough memory (FSE_readNCount unit test)");
            memcpy(smallBuffer, header, headerSize-1);
            max = 129;
            errorCode = FSE_readNCount(norm, &max, &tableLog, smallBuffer, headerSize-1);
            CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");
            free(smallBuffer);
        }
    }


    /* FSE_buildCTable_raw & FSE_buildDTable_raw */
    {
        U32 ct[FSE_CTABLE_SIZE_U32(8, 256)];
        U32 dt[FSE_DTABLE_SIZE_U32(8)];
        U64 crcOrig, crcVerif;
        size_t cSize, verifSize;

        U32 i;
        for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&seed) & 63) + '0';
        crcOrig = XXH64(testBuff, TBSIZE, 0);

        errorCode = FSE_buildCTable_raw(ct, 8);
        CHECK(FSE_isError(errorCode), "FSE_buildCTable_raw should have worked");
        errorCode = FSE_buildDTable_raw(dt, 8);
        CHECK(FSE_isError(errorCode), "FSE_buildDTable_raw should have worked");

        cSize = FSE_compress_usingCTable(cBuff, FSE_COMPRESSBOUND(TBSIZE), testBuff, TBSIZE, ct);
        CHECK(FSE_isError(cSize), "FSE_compress_usingCTable should have worked using raw CTable");

        verifSize = FSE_decompress_usingDTable(verifBuff, TBSIZE, cBuff, cSize, dt);
        CHECK(FSE_isError(verifSize), "FSE_decompress_usingDTable should have worked using raw DTable");

        crcVerif = XXH64(verifBuff, verifSize, 0);
        CHECK(crcOrig != crcVerif, "Raw regenerated data is corrupted");
    }

    /* known corner case */
    {
        BYTE sample8[8] = { 0, 0, 0, 2, 0, 0, 0, 0 };
        BYTE* rBuff;
        errorCode = FSE_compress(cBuff, TBSIZE, sample8, 8);
        CHECK(FSE_isError(errorCode), "FSE_compress failed compressing sample8");
        rBuff = (BYTE*)malloc(errorCode);   /* in order to catch read overflow with Valgrind */
        CHECK(rBuff==NULL, "Not enough memory for rBuff");
        memcpy(rBuff, cBuff, errorCode);
        errorCode = FSE_decompress(verifBuff, sizeof(sample8), rBuff, errorCode);
        CHECK(errorCode != sizeof(sample8), "FSE_decompress failed regenerating sample8");
        free(rBuff);
    }

    free(testBuff);
    free(cBuff);
    free(verifBuff);
    DISPLAY("Unit tests completed\n");
}
示例#5
0
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
    BYTE* bufferP0    = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP1    = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP15   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP90   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferP100  = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferDst   = (BYTE*) malloc (BUFFERSIZE+64);
    BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
    size_t bufferDstSize = BUFFERSIZE+64;
    unsigned testNb, maxSV, tableLog;
    const size_t maxTestSizeMask = 0x1FFFF;
    U32 rootSeed = seed;
    U32 time = FUZ_GetMilliStart();

    generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
    generate (bufferP1  , BUFFERSIZE, 0.01, &rootSeed);
    generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
    generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
    memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);

    if (startTestNb)
    {
        U32 i;
        for (i=0; i<startTestNb; i++)
            FUZ_rand (&rootSeed);
    }

    for (testNb=startTestNb; testNb<totalTest; testNb++)
    {
        BYTE* bufferTest;
        int tag=0;
        U32 roundSeed = rootSeed ^ 0xEDA5B371;
        FUZ_rand(&rootSeed);

        DISPLAYLEVEL (4, "\r test %5u  ", testNb);
        if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
        {
            DISPLAY ("\r test %5u  ", testNb);
            time = FUZ_GetMilliStart();
        }

        /* Compression / Decompression tests */
        {
            /* determine test sample */
            size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
            size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
            size_t sizeCompressed;
            U32 hashOrig;

            if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
            else
            {
                switch(FUZ_rand(&roundSeed) & 3)
                {
                    case 0: bufferTest = bufferP0 + offset; break;
                    case 1: bufferTest = bufferP1 + offset; break;
                    case 2: bufferTest = bufferP90 + offset; break;
                    default : bufferTest = bufferP100 + offset; break;
                }
            }
            DISPLAYLEVEL (4,"%3i ", tag++);;
            hashOrig = XXH32 (bufferTest, sizeOrig, 0);

            /* compress test */
            sizeCompressed = FSE_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
            CHECK(FSE_isError(sizeCompressed), "Compression failed !");

            if (sizeCompressed > 1)   /* don't check uncompressed & rle corner cases */
            {
                /* failed compression test*/
                {
                    size_t errorCode;
                    void* tooSmallDBuffer = malloc(sizeCompressed-1);   /* overflows detected with Valgrind */
                    CHECK(tooSmallDBuffer==NULL, "Not enough memory for tooSmallDBuffer test");
                    errorCode = FSE_compress (tooSmallDBuffer, sizeCompressed-1, bufferTest, sizeOrig);
                    CHECK(errorCode!=0, "Compression should have failed : destination buffer too small");
                    free(tooSmallDBuffer);
                }

                /* decompression test */
                {
                    U32 hashEnd;
                    BYTE saved = (bufferVerif[sizeOrig] = 254);
                    size_t result = FSE_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
                    CHECK(bufferVerif[sizeOrig] != saved, "Output buffer overrun (bufferVerif) : write beyond specified end");
                    CHECK(FSE_isError(result), "Decompression failed");
                    hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
                    CHECK(hashEnd != hashOrig, "Decompressed data corrupted");
                }
            }
        }

        /* Attempt header decoding on bogus data */
        {
            short count[256];
            size_t result;
            DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
            maxSV = 255;
            result = FSE_readNCount (count, &maxSV, &tableLog, bufferTest, FSE_NCOUNTBOUND);
            if (!FSE_isError(result))   /* an error would be normal */
            {
                int checkCount;
                CHECK(result > FSE_NCOUNTBOUND, "FSE_readHeader() reads too far (buffer overflow)");
                CHECK(maxSV > 255, "count table overflow (%u)", maxSV+1);
                checkCount = FUZ_checkCount(count, tableLog, maxSV);
                CHECK(checkCount==-1, "symbol distribution corrupted");
            }
        }

        /* Attempt decompression on bogus data */
        {
            size_t maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
            size_t sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
            BYTE saved = (bufferDst[maxDstSize] = 253);
            size_t result;
            DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
            result = FSE_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
            CHECK(!FSE_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
            CHECK(bufferDst[maxDstSize] != saved, "Output buffer bufferDst corrupted");
        }
    }

    /* exit */
    free (bufferP0);
    free (bufferP1);
    free (bufferP15);
    free (bufferP90);
    free (bufferP100);
    free (bufferDst);
    free (bufferVerif);
}