size_t FSE_compressU16(void* dst, size_t maxDstSize,
const unsigned short* src, size_t srcSize,
unsigned maxSymbolValue, unsigned tableLog)
{
const U16* const istart = src;
const U16* ip = istart;
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const omax = ostart + maxDstSize;
U32 counting[FSE_MAX_SYMBOL_VALUE+1] = {0};
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
CTable_max_t ct;
size_t errorCode;
/* early out */
if (srcSize <= 1) return srcSize;
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Scan for stats */
errorCode = FSE_countU16 (counting, &maxSymbolValue, ip, srcSize);
if (FSE_isError(errorCode)) return errorCode;
if (errorCode == srcSize) return 1; /* Input data is one constant element x srcSize times. Use RLE compression. */
/* Normalize */
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
errorCode = FSE_normalizeCount (norm, tableLog, counting, srcSize, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
/* Write table description header */
errorCode = FSE_writeNCount (op, omax-op, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
op += errorCode;
/* Compress */
errorCode = FSE_buildCTableU16 (ct, norm, maxSymbolValue, tableLog, 1);
if (FSE_isError(errorCode)) return errorCode;
op += FSE_compressU16_usingCTable (op, omax - op, ip, srcSize, ct);
/* check compressibility */
if ( (size_t)(op-ostart) >= (size_t)(srcSize-1)*(sizeof(U16)) )
return 0; /* no compression */
return op-ostart;
}
size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 maxSymbolValue = HUF_TABLELOG_MAX;
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
U32 count[HUF_TABLELOG_MAX+1];
S16 norm[HUF_TABLELOG_MAX+1];
/* init conditions */
if (wtSize <= 1) return 0; /* Not compressible */
/* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) );
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
}
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
/* Write table description header */
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
op += hSize;
}
/* Compress */
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
if (cSize == 0) return 0; /* not enough space for compressed data */
op += cSize;
}
return op-ostart;
}
static void unitTest(void)
{
BYTE* testBuff = (BYTE*)malloc(TBSIZE);
BYTE* cBuff = (BYTE*)malloc(FSE_COMPRESSBOUND(TBSIZE));
BYTE* verifBuff = (BYTE*)malloc(TBSIZE);
size_t errorCode;
U32 seed=0, testNb=0, lseed=0;
U32 count[256];
if ((!testBuff) || (!cBuff) || (!verifBuff))
{
DISPLAY("Not enough memory, exiting ... \n");
free(testBuff);
free(cBuff);
free(verifBuff);
return;
}
/* FSE_count */
{
U32 max, i;
for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&lseed) & 63) + '0';
max = '0' + 63;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
max -= 1;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(!FSE_isError(errorCode), "Error : FSE_count() should have failed : value > max");
max = 65000;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
}
/* FSE_optimalTableLog */
{
U32 max, i, tableLog=12;
size_t testSize = 999;
for (i=0; i< testSize; i++) testBuff[i] = (BYTE)FUZ_rand(&lseed);
max = 256;
FSE_count(count, &max, testBuff, testSize);
tableLog = FSE_optimalTableLog(tableLog, testSize, max);
CHECK(tableLog<=8, "Too small tableLog");
}
/* FSE_normalizeCount */
{
S16 norm[256];
U32 max = 256;
FSE_count(count, &max, testBuff, TBSIZE);
errorCode = FSE_normalizeCount(norm, 10, count, TBSIZE, max);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
errorCode = FSE_normalizeCount(norm, 8, count, TBSIZE, 256);
CHECK(!FSE_isError(errorCode), "Error : FSE_normalizeCount() should have failed (max >= 1<<tableLog)");
/* limit corner case : try to make internal rank overflow */
{
U32 i;
U32 total = 0;
count[0] = 940;
count[1] = 910;
count[2] = 470;
count[3] = 190;
count[4] = 90;
for(i=5; i<=255; i++) count[i] = 6;
for (i=0; i<=255; i++) total += count[i];
errorCode = FSE_normalizeCount(norm, 10, count, total, 255);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
count[0] = 300;
count[1] = 300;
count[2] = 300;
count[3] = 300;
count[4] = 50;
for(i=5; i<=80; i++) count[i] = 4;
total = 0; for (i=0; i<=80; i++) total += count[i];
errorCode = FSE_normalizeCount(norm, 10, count, total, 80);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
}
}
/* FSE_writeNCount, FSE_readNCount */
{
S16 norm[129];
BYTE header[513];
U32 max, tableLog, i;
size_t headerSize;
for (i=0; i< TBSIZE; i++) testBuff[i] = i % 127;
max = 128;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
tableLog = FSE_optimalTableLog(0, TBSIZE, max);
errorCode = FSE_normalizeCount(norm, tableLog, count, TBSIZE, max);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
headerSize = FSE_NCountWriteBound(max, tableLog);
headerSize = FSE_writeNCount(header, 513, norm, max, tableLog);
CHECK(FSE_isError(headerSize), "Error : FSE_writeNCount() should have worked");
header[headerSize-1] = 0;
errorCode = FSE_writeNCount(header, headerSize-1, norm, max, tableLog);
CHECK(!FSE_isError(errorCode), "Error : FSE_writeNCount() should have failed");
CHECK (header[headerSize-1] != 0, "Error : FSE_writeNCount() buffer overwrite");
errorCode = FSE_writeNCount(header, headerSize+1, norm, max, tableLog);
CHECK(FSE_isError(errorCode), "Error : FSE_writeNCount() should have worked");
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
CHECK(FSE_isError(errorCode), "Error : FSE_readNCount() should have worked : (error %s)", FSE_getErrorName(errorCode));
max = 64;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (max too small)");
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize-1);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");
{
void* smallBuffer = malloc(headerSize-1); /* outbound read can be caught by valgrind */
CHECK(smallBuffer==NULL, "Error : Not enough memory (FSE_readNCount unit test)");
memcpy(smallBuffer, header, headerSize-1);
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, smallBuffer, headerSize-1);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");
free(smallBuffer);
}
}
/* FSE_buildCTable_raw & FSE_buildDTable_raw */
{
U32 ct[FSE_CTABLE_SIZE_U32(8, 256)];
U32 dt[FSE_DTABLE_SIZE_U32(8)];
U64 crcOrig, crcVerif;
size_t cSize, verifSize;
U32 i;
for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&seed) & 63) + '0';
crcOrig = XXH64(testBuff, TBSIZE, 0);
errorCode = FSE_buildCTable_raw(ct, 8);
CHECK(FSE_isError(errorCode), "FSE_buildCTable_raw should have worked");
errorCode = FSE_buildDTable_raw(dt, 8);
CHECK(FSE_isError(errorCode), "FSE_buildDTable_raw should have worked");
cSize = FSE_compress_usingCTable(cBuff, FSE_COMPRESSBOUND(TBSIZE), testBuff, TBSIZE, ct);
CHECK(FSE_isError(cSize), "FSE_compress_usingCTable should have worked using raw CTable");
verifSize = FSE_decompress_usingDTable(verifBuff, TBSIZE, cBuff, cSize, dt);
CHECK(FSE_isError(verifSize), "FSE_decompress_usingDTable should have worked using raw DTable");
crcVerif = XXH64(verifBuff, verifSize, 0);
CHECK(crcOrig != crcVerif, "Raw regenerated data is corrupted");
}
/* known corner case */
{
BYTE sample8[8] = { 0, 0, 0, 2, 0, 0, 0, 0 };
BYTE* rBuff;
errorCode = FSE_compress(cBuff, TBSIZE, sample8, 8);
CHECK(FSE_isError(errorCode), "FSE_compress failed compressing sample8");
rBuff = (BYTE*)malloc(errorCode); /* in order to catch read overflow with Valgrind */
CHECK(rBuff==NULL, "Not enough memory for rBuff");
memcpy(rBuff, cBuff, errorCode);
errorCode = FSE_decompress(verifBuff, sizeof(sample8), rBuff, errorCode);
CHECK(errorCode != sizeof(sample8), "FSE_decompress failed regenerating sample8");
free(rBuff);
}
free(testBuff);
free(cBuff);
free(verifBuff);
DISPLAY("Unit tests completed\n");
}
size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize)
{
BYTE *const ostart = (BYTE *)dst;
BYTE *op = ostart;
BYTE *const oend = ostart + dstSize;
U32 maxSymbolValue = HUF_TABLELOG_MAX;
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
FSE_CTable *CTable;
U32 *count;
S16 *norm;
size_t spaceUsed32 = 0;
HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32));
CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32);
spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX);
count = (U32 *)workspace + spaceUsed32;
spaceUsed32 += HUF_TABLELOG_MAX + 1;
norm = (S16 *)((U32 *)workspace + spaceUsed32);
spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2;
if ((spaceUsed32 << 2) > workspaceSize)
return ERROR(tableLog_tooLarge);
workspace = (U32 *)workspace + spaceUsed32;
workspaceSize -= (spaceUsed32 << 2);
/* init conditions */
if (wtSize <= 1)
return 0; /* Not compressible */
/* Scan input and build symbol stats */
{
CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize));
if (maxCount == wtSize)
return 1; /* only a single symbol in src : rle */
if (maxCount == 1)
return 0; /* each symbol present maximum once => not compressible */
}
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue));
/* Write table description header */
{
CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
op += hSize;
}
/* Compress */
CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize));
{
CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable));
if (cSize == 0)
return 0; /* not enough space for compressed data */
op += cSize;
}
return op - ostart;
}
