size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) { BYTE* const ostart = (BYTE*) dst; BYTE* op = ostart; BYTE* const oend = ostart + dstSize; U32 maxSymbolValue = HUF_TABLELOG_MAX; U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER]; U32 count[HUF_TABLELOG_MAX+1]; S16 norm[HUF_TABLELOG_MAX+1]; /* init conditions */ if (wtSize <= 1) return 0; /* Not compressible */ /* Scan input and build symbol stats */ { CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) ); if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ } tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); /* Write table description header */ { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); op += hSize; } /* Compress */ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); if (cSize == 0) return 0; /* not enough space for compressed data */ op += cSize; } return op-ostart; }
static void unitTest(void) { BYTE* testBuff = (BYTE*)malloc(TBSIZE); BYTE* cBuff = (BYTE*)malloc(FSE_COMPRESSBOUND(TBSIZE)); BYTE* verifBuff = (BYTE*)malloc(TBSIZE); size_t errorCode; U32 seed=0, testNb=0, lseed=0; U32 count[256]; if ((!testBuff) || (!cBuff) || (!verifBuff)) { DISPLAY("Not enough memory, exiting ... \n"); free(testBuff); free(cBuff); free(verifBuff); return; } /* FSE_count */ { U32 max, i; for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&lseed) & 63) + '0'; max = '0' + 63; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); max -= 1; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(!FSE_isError(errorCode), "Error : FSE_count() should have failed : value > max"); max = 65000; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); } /* FSE_optimalTableLog */ { U32 max, i, tableLog=12; size_t testSize = 999; for (i=0; i< testSize; i++) testBuff[i] = (BYTE)FUZ_rand(&lseed); max = 256; FSE_count(count, &max, testBuff, testSize); tableLog = FSE_optimalTableLog(tableLog, testSize, max); CHECK(tableLog<=8, "Too small tableLog"); } /* FSE_normalizeCount */ { S16 norm[256]; U32 max = 256; FSE_count(count, &max, testBuff, TBSIZE); errorCode = FSE_normalizeCount(norm, 10, count, TBSIZE, max); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); errorCode = FSE_normalizeCount(norm, 8, count, TBSIZE, 256); CHECK(!FSE_isError(errorCode), "Error : FSE_normalizeCount() should have failed (max >= 1<<tableLog)"); /* limit corner case : try to make internal rank overflow */ { U32 i; U32 total = 0; count[0] = 940; count[1] = 910; count[2] = 470; count[3] = 190; count[4] = 90; for(i=5; i<=255; i++) count[i] = 6; for (i=0; i<=255; i++) total += count[i]; errorCode = FSE_normalizeCount(norm, 10, count, total, 255); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); count[0] = 300; count[1] = 300; count[2] = 300; count[3] = 300; count[4] = 50; for(i=5; i<=80; i++) count[i] = 4; total = 0; for (i=0; i<=80; i++) total += count[i]; errorCode = FSE_normalizeCount(norm, 10, count, total, 80); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); } } /* FSE_writeNCount, FSE_readNCount */ { S16 norm[129]; BYTE header[513]; U32 max, tableLog, i; size_t headerSize; for (i=0; i< TBSIZE; i++) testBuff[i] = i % 127; max = 128; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); tableLog = FSE_optimalTableLog(0, TBSIZE, max); errorCode = FSE_normalizeCount(norm, tableLog, count, TBSIZE, max); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); headerSize = FSE_NCountWriteBound(max, tableLog); headerSize = FSE_writeNCount(header, 513, norm, max, tableLog); CHECK(FSE_isError(headerSize), "Error : FSE_writeNCount() should have worked"); header[headerSize-1] = 0; errorCode = FSE_writeNCount(header, headerSize-1, norm, max, tableLog); CHECK(!FSE_isError(errorCode), "Error : FSE_writeNCount() should have failed"); CHECK (header[headerSize-1] != 0, "Error : FSE_writeNCount() buffer overwrite"); errorCode = FSE_writeNCount(header, headerSize+1, norm, max, tableLog); CHECK(FSE_isError(errorCode), "Error : FSE_writeNCount() should have worked"); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize); CHECK(FSE_isError(errorCode), "Error : FSE_readNCount() should have worked : (error %s)", FSE_getErrorName(errorCode)); max = 64; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (max too small)"); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize-1); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)"); { void* smallBuffer = malloc(headerSize-1); /* outbound read can be caught by valgrind */ CHECK(smallBuffer==NULL, "Error : Not enough memory (FSE_readNCount unit test)"); memcpy(smallBuffer, header, headerSize-1); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, smallBuffer, headerSize-1); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)"); free(smallBuffer); } } /* FSE_buildCTable_raw & FSE_buildDTable_raw */ { U32 ct[FSE_CTABLE_SIZE_U32(8, 256)]; U32 dt[FSE_DTABLE_SIZE_U32(8)]; U64 crcOrig, crcVerif; size_t cSize, verifSize; U32 i; for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&seed) & 63) + '0'; crcOrig = XXH64(testBuff, TBSIZE, 0); errorCode = FSE_buildCTable_raw(ct, 8); CHECK(FSE_isError(errorCode), "FSE_buildCTable_raw should have worked"); errorCode = FSE_buildDTable_raw(dt, 8); CHECK(FSE_isError(errorCode), "FSE_buildDTable_raw should have worked"); cSize = FSE_compress_usingCTable(cBuff, FSE_COMPRESSBOUND(TBSIZE), testBuff, TBSIZE, ct); CHECK(FSE_isError(cSize), "FSE_compress_usingCTable should have worked using raw CTable"); verifSize = FSE_decompress_usingDTable(verifBuff, TBSIZE, cBuff, cSize, dt); CHECK(FSE_isError(verifSize), "FSE_decompress_usingDTable should have worked using raw DTable"); crcVerif = XXH64(verifBuff, verifSize, 0); CHECK(crcOrig != crcVerif, "Raw regenerated data is corrupted"); } /* known corner case */ { BYTE sample8[8] = { 0, 0, 0, 2, 0, 0, 0, 0 }; BYTE* rBuff; errorCode = FSE_compress(cBuff, TBSIZE, sample8, 8); CHECK(FSE_isError(errorCode), "FSE_compress failed compressing sample8"); rBuff = (BYTE*)malloc(errorCode); /* in order to catch read overflow with Valgrind */ CHECK(rBuff==NULL, "Not enough memory for rBuff"); memcpy(rBuff, cBuff, errorCode); errorCode = FSE_decompress(verifBuff, sizeof(sample8), rBuff, errorCode); CHECK(errorCode != sizeof(sample8), "FSE_decompress failed regenerating sample8"); free(rBuff); } free(testBuff); free(cBuff); free(verifBuff); DISPLAY("Unit tests completed\n"); }
size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize) { BYTE *const ostart = (BYTE *)dst; BYTE *op = ostart; BYTE *const oend = ostart + dstSize; U32 maxSymbolValue = HUF_TABLELOG_MAX; U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; FSE_CTable *CTable; U32 *count; S16 *norm; size_t spaceUsed32 = 0; HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32)); CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32); spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX); count = (U32 *)workspace + spaceUsed32; spaceUsed32 += HUF_TABLELOG_MAX + 1; norm = (S16 *)((U32 *)workspace + spaceUsed32); spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2; if ((spaceUsed32 << 2) > workspaceSize) return ERROR(tableLog_tooLarge); workspace = (U32 *)workspace + spaceUsed32; workspaceSize -= (spaceUsed32 << 2); /* init conditions */ if (wtSize <= 1) return 0; /* Not compressible */ /* Scan input and build symbol stats */ { CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize)); if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ } tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue)); /* Write table description header */ { CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog)); op += hSize; } /* Compress */ CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize)); { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable)); if (cSize == 0) return 0; /* not enough space for compressed data */ op += cSize; } return op - ostart; }