static size_t HUF_decompress1X4_usingDTable_internal( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) { BIT_DStream_t bitD; /* Init */ { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); if (HUF_isError(errorCode)) return errorCode; } /* decode */ { BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; DTableDesc const dtd = HUF_getDTableDesc(DTable); HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); } /* check */ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); /* decoded size */ return dstSize; }
size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) { U32 tableLog = 0; U32 nbSymbols = 0; size_t iSize; void* const dtPtr = DTable + 1; HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; U32* rankVal; BYTE* huffWeight; size_t spaceUsed32 = 0; rankVal = (U32 *)workSpace + spaceUsed32; spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; huffWeight = (BYTE *)((U32 *)workSpace + spaceUsed32); spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge); workSpace = (U32 *)workSpace + spaceUsed32; wkspSize -= (spaceUsed32 << 2); HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* Table header */ { DTableDesc dtd = HUF_getDTableDesc(DTable); if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ dtd.tableType = 0; dtd.tableLog = (BYTE)tableLog; memcpy(DTable, &dtd, sizeof(dtd)); } /* Calculate starting value for each rank */ { U32 n, nextRankStart = 0; for (n=1; n<tableLog+1; n++) { U32 const current = nextRankStart; nextRankStart += (rankVal[n] << (n-1)); rankVal[n] = current; } } /* fill DTable */ { U32 n; for (n=0; n<nbSymbols; n++) { U32 const w = huffWeight[n]; U32 const length = (1 << w) >> 1; U32 u; HUF_DEltX2 D; D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); for (u = rankVal[w]; u < rankVal[w] + length; u++) dt[u] = D; rankVal[w] += length; } } return iSize; }
size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { const BYTE* ip = (const BYTE*) cSrc; size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); }
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; return HUF_decompress4X6_usingDTable (dst, dstSize, ip, cSrcSize, DTable); }
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize) { const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); }
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) { BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ U32 tableLog = 0; U32 nbSymbols = 0; size_t iSize; void* const dtPtr = DTable + 1; HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* Table header */ { DTableDesc dtd = HUF_getDTableDesc(DTable); if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ dtd.tableType = 0; dtd.tableLog = (BYTE)tableLog; memcpy(DTable, &dtd, sizeof(dtd)); } /* Prepare ranks */ { U32 n, nextRankStart = 0; for (n=1; n<tableLog+1; n++) { U32 current = nextRankStart; nextRankStart += (rankVal[n] << (n-1)); rankVal[n] = current; } } /* fill DTable */ { U32 n; for (n=0; n<nbSymbols; n++) { U32 const w = huffWeight[n]; U32 const length = (1 << w) >> 1; U32 i; HUF_DEltX2 D; D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); for (i = rankVal[w]; i < rankVal[w] + length; i++) dt[i] = D; rankVal[w] += length; } } return iSize; }
FORCE_INLINE int Lizard_writeStream(int useHuff, Lizard_stream_t* ctx, BYTE* streamPtr, uint32_t streamLen, BYTE** op, BYTE* oend) { if (useHuff && streamLen > 1024) { #ifndef LIZARD_NO_HUFFMAN int useHuffBuf; if (*op + 6 > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 6 > oend[%p]\n", *op, oend); return -1; } useHuffBuf = ((size_t)(oend - (*op + 6)) < HUF_compressBound(streamLen)); if (useHuffBuf) { if (streamLen > LIZARD_BLOCK_SIZE) { LIZARD_LOG_COMPRESS("streamLen[%d] > LIZARD_BLOCK_SIZE\n", streamLen); return -1; } ctx->comprStreamLen = (U32)HUF_compress(ctx->huffBase, ctx->huffEnd - ctx->huffBase, streamPtr, streamLen); } else { ctx->comprStreamLen = (U32)HUF_compress(*op + 6, oend - (*op + 6), streamPtr, streamLen); } if (!HUF_isError(ctx->comprStreamLen)) { if (ctx->comprStreamLen > 0 && (LIZARD_MINIMAL_HUFF_GAIN(ctx->comprStreamLen) < streamLen)) { /* compressible */ MEM_writeLE24(*op, streamLen); MEM_writeLE24(*op+3, ctx->comprStreamLen); if (useHuffBuf) { if ((size_t)(oend - (*op + 6)) < ctx->comprStreamLen) { LIZARD_LOG_COMPRESS("*op[%p] oend[%p] comprStreamLen[%d]\n", *op, oend, (int)ctx->comprStreamLen); return -1; } memcpy(*op + 6, ctx->huffBase, ctx->comprStreamLen); } *op += ctx->comprStreamLen + 6; LIZARD_LOG_COMPRESS("HUF_compress streamLen=%d comprStreamLen=%d\n", (int)streamLen, (int)ctx->comprStreamLen); return 1; } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR comprStreamLen=%d streamLen=%d\n", (int)ctx->comprStreamLen, (int)streamLen); } } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR %d: %s\n", (int)ctx->comprStreamLen, HUF_getErrorName(ctx->comprStreamLen)); } #else LIZARD_LOG_COMPRESS("compiled with LIZARD_NO_HUFFMAN\n"); (void)ctx; return -1; #endif } else ctx->comprStreamLen = 0; if (*op + 3 + streamLen > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 3 + streamLen[%d] > oend[%p]\n", *op, streamLen, oend); return -1; } MEM_writeLE24(*op, streamLen); *op += 3; memcpy(*op, streamPtr, streamLen); *op += streamLen; LIZARD_LOG_COMPRESS("Uncompressed streamLen=%d\n", (int)streamLen); return 0; }
static size_t HUF_decompress1X2_usingDTable_internal( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) { BYTE* op = (BYTE*)dst; BYTE* const oend = op + dstSize; const void* dtPtr = DTable + 1; const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; BIT_DStream_t bitD; DTableDesc const dtd = HUF_getDTableDesc(DTable); U32 const dtLog = dtd.tableLog; { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); if (HUF_isError(errorCode)) return errorCode; } HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); /* check */ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); return dstSize; }
size_t HUF_decompress1X6_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const U32* DTable) { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; BIT_DStream_t bitD; /* Init */ { size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize); if (HUF_isError(errorCode)) return errorCode; } /* finish bitStreams one by one */ { U32 const dtLog = DTable[0]; HUF_decodeStreamX6(ostart, &bitD, oend, DTable, dtLog); } /* check */ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); /* decoded size */ return dstSize; }
static size_t HUF_decompress4X4_usingDTable_internal( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) { if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; const void* const dtPtr = DTable+1; const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; /* Init */ BIT_DStream_t bitD1; BIT_DStream_t bitD2; BIT_DStream_t bitD3; BIT_DStream_t bitD4; size_t const length1 = MEM_readLE16(istart); size_t const length2 = MEM_readLE16(istart+2); size_t const length3 = MEM_readLE16(istart+4); size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE* const istart2 = istart1 + length1; const BYTE* const istart3 = istart2 + length2; const BYTE* const istart4 = istart3 + length3; size_t const segmentSize = (dstSize+3) / 4; BYTE* const opStart2 = ostart + segmentSize; BYTE* const opStart3 = opStart2 + segmentSize; BYTE* const opStart4 = opStart3 + segmentSize; BYTE* op1 = ostart; BYTE* op2 = opStart2; BYTE* op3 = opStart3; BYTE* op4 = opStart4; U32 endSignal; DTableDesc const dtd = HUF_getDTableDesc(DTable); U32 const dtLog = dtd.tableLog; if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); if (HUF_isError(errorCode)) return errorCode; } { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); if (HUF_isError(errorCode)) return errorCode; } { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); if (HUF_isError(errorCode)) return errorCode; } { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); if (HUF_isError(errorCode)) return errorCode; } /* 16-32 symbols per loop (4-8 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { HUF_DECODE_SYMBOLX4_2(op1, &bitD1); HUF_DECODE_SYMBOLX4_2(op2, &bitD2); HUF_DECODE_SYMBOLX4_2(op3, &bitD3); HUF_DECODE_SYMBOLX4_2(op4, &bitD4); HUF_DECODE_SYMBOLX4_1(op1, &bitD1); HUF_DECODE_SYMBOLX4_1(op2, &bitD2); HUF_DECODE_SYMBOLX4_1(op3, &bitD3); HUF_DECODE_SYMBOLX4_1(op4, &bitD4); HUF_DECODE_SYMBOLX4_2(op1, &bitD1); HUF_DECODE_SYMBOLX4_2(op2, &bitD2); HUF_DECODE_SYMBOLX4_2(op3, &bitD3); HUF_DECODE_SYMBOLX4_2(op4, &bitD4); HUF_DECODE_SYMBOLX4_0(op1, &bitD1); HUF_DECODE_SYMBOLX4_0(op2, &bitD2); HUF_DECODE_SYMBOLX4_0(op3, &bitD3); HUF_DECODE_SYMBOLX4_0(op4, &bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); } /* check corruption */ if (op1 > opStart2) return ERROR(corruption_detected); if (op2 > opStart3) return ERROR(corruption_detected); if (op3 > opStart4) return ERROR(corruption_detected); /* note : op4 already verified within main loop */ /* finish bitStreams one by one */ HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); /* check */ { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); if (!endCheck) return ERROR(corruption_detected); } /* decoded size */ return dstSize; } }
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) { BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; U32* const rankStart = rankStart0+1; rankVal_t rankVal; U32 tableLog, maxW, sizeOfSort, nbSymbols; DTableDesc dtd = HUF_getDTableDesc(DTable); U32 const maxTableLog = dtd.maxTableLog; size_t iSize; void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ /* find maxWeight */ for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ /* Get start index of each weight */ { U32 w, nextRankStart = 0; for (w=1; w<maxW+1; w++) { U32 current = nextRankStart; nextRankStart += rankStats[w]; rankStart[w] = current; } rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ sizeOfSort = nextRankStart; } /* sort symbols by weight */ { U32 s; for (s=0; s<nbSymbols; s++) { U32 const w = weightList[s]; U32 const r = rankStart[w]++; sortedSymbol[r].symbol = (BYTE)s; sortedSymbol[r].weight = (BYTE)w; } rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ } /* Build rankVal */ { U32* const rankVal0 = rankVal[0]; { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ U32 nextRankVal = 0; U32 w; for (w=1; w<maxW+1; w++) { U32 current = nextRankVal; nextRankVal += rankStats[w] << (w+rescale); rankVal0[w] = current; } } { U32 const minBits = tableLog+1 - maxW; U32 consumed; for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { U32* const rankValPtr = rankVal[consumed]; U32 w; for (w = 1; w < maxW+1; w++) { rankValPtr[w] = rankVal0[w] >> consumed; } } } } HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog+1); dtd.tableLog = (BYTE)maxTableLog; dtd.tableType = 1; memcpy(DTable, &dtd, sizeof(dtd)); return iSize; }
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb) { BYTE* bufferP0 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP1 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP15 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP90 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP100 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferDst = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64); size_t const bufferDstSize = BUFFERSIZE+64; unsigned testNb; size_t const maxTestSizeMask = 0x1FFFF; /* 128 KB - 1 */ U32 rootSeed = seed; U32 time = FUZ_GetMilliStart(); generateNoise (bufferP0, BUFFERSIZE, &rootSeed); generate (bufferP1 , BUFFERSIZE, 0.01, &rootSeed); generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed); generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed); memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE); memset(bufferDst, 0, BUFFERSIZE); { U32 u; for (u=0; u<startTestNb; u++) FUZ_rand (&rootSeed); } for (testNb=startTestNb; testNb<totalTest; testNb++) { U32 roundSeed = rootSeed ^ 0xEDA5B371; FUZ_rand(&rootSeed); int tag=0; BYTE* bufferTest = NULL; DISPLAYLEVEL (4, "\r test %5u ", testNb); if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE) { DISPLAY ("\r test %5u ", testNb); time = FUZ_GetMilliStart(); } /* Compression / Decompression tests */ DISPLAYLEVEL (4,"%3i ", tag++); { /* determine test sample */ size_t const sizeOrig = (FUZ_rand(&roundSeed) & maxTestSizeMask) + 1; size_t const offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask)); size_t sizeCompressed; U32 hashOrig; if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset; else { switch(FUZ_rand(&roundSeed) & 3) { case 0: bufferTest = bufferP0 + offset; break; case 1: bufferTest = bufferP1 + offset; break; case 2: bufferTest = bufferP90 + offset; break; default : bufferTest = bufferP100 + offset; break; } } hashOrig = XXH32 (bufferTest, sizeOrig, 0); /* compression test */ sizeCompressed = HUF_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig); CHECK(HUF_isError(sizeCompressed), "HUF_compress failed"); if (sizeCompressed > 1) { /* don't check uncompressed & rle corner cases */ /* failed compression test */ { BYTE const saved = bufferVerif[sizeCompressed-1] = 253; size_t const errorCode = HUF_compress (bufferVerif, sizeCompressed-1, bufferTest, sizeOrig); CHECK(errorCode!=0, "HUF_compress should have failed (too small destination buffer)") CHECK(bufferVerif[sizeCompressed-1] != saved, "HUF_compress w/ too small dst : bufferVerif overflow"); } /* decompression test */ { BYTE const saved = bufferVerif[sizeOrig] = 253; size_t const result = HUF_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed); CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress : bufferVerif overflow"); CHECK(HUF_isError(result), "HUF_decompress failed : %s", HUF_getErrorName(result)); { U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0); if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig); CHECK(hashEnd != hashOrig, "HUF_decompress : Decompressed data corrupted"); } } /* quad decoder test (more fragile) */ /* if (sizeOrig > 64) { BYTE const saved = bufferVerif[sizeOrig] = 253; size_t const result = HUF_decompress4X6 (bufferVerif, sizeOrig, bufferDst, sizeCompressed); CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress4X6 : bufferVerif overflow"); CHECK(HUF_isError(result), "HUF_decompress4X6 failed : %s", HUF_getErrorName(result)); { U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0); if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig); CHECK(hashEnd != hashOrig, "HUF_decompress4X6 : Decompressed data corrupted"); } } */ /* truncated src decompression test */ if (sizeCompressed>4) { /* note : in some rare cases, the truncated bitStream may still generate by chance a valid output of correct size */ size_t const missing = (FUZ_rand(&roundSeed) % (sizeCompressed-3)) + 2; /* no problem, as sizeCompressed > 4 */ size_t const tooSmallSize = sizeCompressed - missing; void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ CHECK(cBufferTooSmall == NULL, "not enough memory !"); memcpy(cBufferTooSmall, bufferDst, tooSmallSize); { size_t const errorCode = HUF_decompress(bufferVerif, sizeOrig, cBufferTooSmall, tooSmallSize); CHECK(!HUF_isError(errorCode) && (errorCode!=sizeOrig), "HUF_decompress should have failed ! (truncated src buffer)"); } free(cBufferTooSmall); } } } /* Compression / Decompression tests */ /* Attempt decompression on bogus data */ { size_t const maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask; size_t const sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask; BYTE const saved = (bufferDst[maxDstSize] = 253); size_t result; DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);; result = HUF_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed); CHECK(!HUF_isError(result) && (result > maxDstSize), "Decompression overran output buffer"); CHECK(bufferDst[maxDstSize] != saved, "HUF_decompress noise : bufferDst overflow"); } } /* for (testNb=startTestNb; testNb<totalTest; testNb++) */ /* exit */ free (bufferP0); free (bufferP1); free (bufferP15); free (bufferP90); free (bufferP100); free (bufferDst); free (bufferVerif); }
size_t HUF_decompress4X6_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const U32* DTable) { /* Check */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (dstSize < 64) return ERROR(dstSize_tooSmall); /* only work for dstSize >= 64 */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; const U32 dtLog = DTable[0]; const void* const ddPtr = DTable+1; const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr; const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr; /* Init */ BIT_DStream_t bitD1; BIT_DStream_t bitD2; BIT_DStream_t bitD3; BIT_DStream_t bitD4; const size_t length1 = MEM_readLE16(istart); const size_t length2 = MEM_readLE16(istart+2); const size_t length3 = MEM_readLE16(istart+4); size_t length4; const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE* const istart2 = istart1 + length1; const BYTE* const istart3 = istart2 + length2; const BYTE* const istart4 = istart3 + length3; const size_t segmentSize = (dstSize+3) / 4; BYTE* const opStart2 = ostart + segmentSize; BYTE* const opStart3 = opStart2 + segmentSize; BYTE* const opStart4 = opStart3 + segmentSize; BYTE* op1 = ostart; BYTE* op2 = opStart2; BYTE* op3 = opStart3; BYTE* op4 = opStart4; U32 endSignal; length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); if (HUF_isError(errorCode)) return errorCode; } { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); if (HUF_isError(errorCode)) return errorCode; } { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); if (HUF_isError(errorCode)) return errorCode; } { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); if (HUF_isError(errorCode)) return errorCode; } /* 4-64 symbols per loop (1-16 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); if (endSignal==BIT_DStream_unfinished) { HUF_DECODE_ROUNDX6; if (sizeof(bitD1.bitContainer)==4) { /* need to decode at least 4 bytes per stream */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); HUF_DECODE_ROUNDX6; } { U32 const saved2 = MEM_read32(opStart2); /* saved from overwrite */ U32 const saved3 = MEM_read32(opStart3); U32 const saved4 = MEM_read32(opStart4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; ) { HUF_DECODE_ROUNDX6; endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); } MEM_write32(opStart2, saved2); MEM_write32(opStart3, saved3); MEM_write32(opStart4, saved4); } } /* check corruption */ if (op1 > opStart2) return ERROR(corruption_detected); if (op2 > opStart3) return ERROR(corruption_detected); if (op3 > opStart4) return ERROR(corruption_detected); /* note : op4 already verified within main loop */ /* finish bitStreams one by one */ HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog); HUF_decodeStreamX6(op2, &bitD2, opStart3, DTable, dtLog); HUF_decodeStreamX6(op3, &bitD3, opStart4, DTable, dtLog); HUF_decodeStreamX6(op4, &bitD4, oend, DTable, dtLog); /* check */ endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); if (!endSignal) return ERROR(corruption_detected); /* decoded size */ return dstSize; } }
/* note : same preparation as X4 */ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) { BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; U32* const rankStart = rankStart0+1; U32 tableLog, maxW, sizeOfSort, nbSymbols; rankVal_t rankVal; const U32 memLog = DTable[0]; size_t iSize; if (memLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable is too small */ /* find maxWeight */ for (maxW = tableLog; maxW && rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ /* Get start index of each weight */ { U32 w, nextRankStart = 0; for (w=1; w<maxW+1; w++) { U32 current = nextRankStart; nextRankStart += rankStats[w]; rankStart[w] = current; } rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ sizeOfSort = nextRankStart; } /* sort symbols by weight */ { U32 s; for (s=0; s<nbSymbols; s++) { U32 w = weightList[s]; U32 r = rankStart[w]++; sortedSymbol[r].symbol = (BYTE)s; sortedSymbol[r].weight = (BYTE)w; } rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ } /* Build rankVal */ { const U32 minBits = tableLog+1 - maxW; U32 nextRankVal = 0; U32 w, consumed; const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ U32* rankVal0 = rankVal[0]; for (w=1; w<maxW+1; w++) { U32 current = nextRankVal; nextRankVal += rankStats[w] << (w+rescale); rankVal0[w] = current; } for (consumed = minBits; consumed <= memLog - minBits; consumed++) { U32* rankValPtr = rankVal[consumed]; for (w = 1; w < maxW+1; w++) { rankValPtr[w] = rankVal0[w] >> consumed; } } } /* fill tables */ { void* ddPtr = DTable+1; HUF_DDescX6* DDescription = (HUF_DDescX6*)ddPtr; void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1)); HUF_DSeqX6* DSequence = (HUF_DSeqX6*)dsPtr; HUF_DSeqX6 DSeq; HUF_DDescX6 DDesc; DSeq.sequence = 0; DDesc.nbBits = 0; DDesc.nbBytes = 0; HUF_fillDTableX6LevelN(DDescription, DSequence, memLog, (const U32 (*)[HUF_TABLELOG_ABSOLUTEMAX + 1])rankVal, 0, 1, maxW, sortedSymbol, sizeOfSort, rankStart0, tableLog+1, DSeq, DDesc); } return iSize; }