static void unitTest(void)
{
U16 testBuffU16[TBSIZE];
size_t errorCode;
U32 startSeed=0, testNb=0;
/* FSE_countU16 */
{
U32 table[FSE_MAX_SYMBOL_VALUE+2];
U32 max, i;
for (i=0; i< TBSIZE; i++) testBuffU16[i] = i % (FSE_MAX_SYMBOL_VALUE+1);
max = FSE_MAX_SYMBOL_VALUE;
errorCode = FSE_countU16(table, testBuffU16, TBSIZE, &max);
CHECK(FSE_isError(errorCode), "FSE_countU16() should have worked");
max = FSE_MAX_SYMBOL_VALUE+1;
errorCode = FSE_countU16(table, testBuffU16, TBSIZE, &max);
CHECK(!FSE_isError(errorCode), "FSE_countU16() should have failed : max too large");
max = FSE_MAX_SYMBOL_VALUE-1;
errorCode = FSE_countU16(table, testBuffU16, TBSIZE, &max);
CHECK(!FSE_isError(errorCode), "FSE_countU16() should have failed : max too low");
}
DISPLAY("Unit tests completed\n");
}
size_t FSE_decompressU16(U16* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize)
{
const BYTE* const istart = (const BYTE*) cSrc;
const BYTE* ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1];
DTable_max_t dt;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
unsigned tableLog;
size_t errorCode;
/* Sanity check */
if (cSrcSize<2) return ERROR(srcSize_wrong); /* specific corner cases (uncompressed & rle) */
/* normal FSE decoding mode */
errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(errorCode)) return errorCode;
ip += errorCode;
cSrcSize -= errorCode;
unsigned scrambler = 1;
errorCode = FSE_buildDTableU16 (dt, counting, maxSymbolValue, tableLog, scrambler);
if (FSE_isError(errorCode)) return errorCode;
return FSE_decompressU16_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
}
size_t FSE_compressU16(void* dst, size_t maxDstSize,
const unsigned short* src, size_t srcSize,
unsigned maxSymbolValue, unsigned tableLog)
{
const U16* const istart = src;
const U16* ip = istart;
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const omax = ostart + maxDstSize;
U32 counting[FSE_MAX_SYMBOL_VALUE+1] = {0};
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
CTable_max_t ct;
size_t errorCode;
/* early out */
if (srcSize <= 1) return srcSize;
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Scan for stats */
errorCode = FSE_countU16 (counting, &maxSymbolValue, ip, srcSize);
if (FSE_isError(errorCode)) return errorCode;
if (errorCode == srcSize) return 1; /* Input data is one constant element x srcSize times. Use RLE compression. */
/* Normalize */
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
errorCode = FSE_normalizeCount (norm, tableLog, counting, srcSize, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
/* Write table description header */
errorCode = FSE_writeNCount (op, omax-op, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
op += errorCode;
/* Compress */
errorCode = FSE_buildCTableU16 (ct, norm, maxSymbolValue, tableLog, 1);
if (FSE_isError(errorCode)) return errorCode;
op += FSE_compressU16_usingCTable (op, omax - op, ip, srcSize, ct);
/* check compressibility */
if ( (size_t)(op-ostart) >= (size_t)(srcSize-1)*(sizeof(U16)) )
return 0; /* no compression */
return op-ostart;
}
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
{
const BYTE* const istart = (const BYTE*)cSrc;
const BYTE* ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1];
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
/* normal FSE decoding mode */
size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(NCountLength)) return NCountLength;
//if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
ip += NCountLength;
cSrcSize -= NCountLength;
CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );
return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
}
size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* const istart = (const BYTE*)cSrc;
const BYTE* ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1];
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
/* normal FSE decoding mode */
{ size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(NCountLength)) return NCountLength;
if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
ip += NCountLength;
cSrcSize -= NCountLength;
}
CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) );
return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
}
static void unitTest(void)
{
BYTE* testBuff = (BYTE*)malloc(TBSIZE);
BYTE* cBuff = (BYTE*)malloc(FSE_COMPRESSBOUND(TBSIZE));
BYTE* verifBuff = (BYTE*)malloc(TBSIZE);
size_t errorCode;
U32 seed=0, testNb=0, lseed=0;
U32 count[256];
if ((!testBuff) || (!cBuff) || (!verifBuff))
{
DISPLAY("Not enough memory, exiting ... \n");
free(testBuff);
free(cBuff);
free(verifBuff);
return;
}
/* FSE_count */
{
U32 max, i;
for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&lseed) & 63) + '0';
max = '0' + 63;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
max -= 1;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(!FSE_isError(errorCode), "Error : FSE_count() should have failed : value > max");
max = 65000;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
}
/* FSE_optimalTableLog */
{
U32 max, i, tableLog=12;
size_t testSize = 999;
for (i=0; i< testSize; i++) testBuff[i] = (BYTE)FUZ_rand(&lseed);
max = 256;
FSE_count(count, &max, testBuff, testSize);
tableLog = FSE_optimalTableLog(tableLog, testSize, max);
CHECK(tableLog<=8, "Too small tableLog");
}
/* FSE_normalizeCount */
{
S16 norm[256];
U32 max = 256;
FSE_count(count, &max, testBuff, TBSIZE);
errorCode = FSE_normalizeCount(norm, 10, count, TBSIZE, max);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
errorCode = FSE_normalizeCount(norm, 8, count, TBSIZE, 256);
CHECK(!FSE_isError(errorCode), "Error : FSE_normalizeCount() should have failed (max >= 1<<tableLog)");
/* limit corner case : try to make internal rank overflow */
{
U32 i;
U32 total = 0;
count[0] = 940;
count[1] = 910;
count[2] = 470;
count[3] = 190;
count[4] = 90;
for(i=5; i<=255; i++) count[i] = 6;
for (i=0; i<=255; i++) total += count[i];
errorCode = FSE_normalizeCount(norm, 10, count, total, 255);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
count[0] = 300;
count[1] = 300;
count[2] = 300;
count[3] = 300;
count[4] = 50;
for(i=5; i<=80; i++) count[i] = 4;
total = 0; for (i=0; i<=80; i++) total += count[i];
errorCode = FSE_normalizeCount(norm, 10, count, total, 80);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
}
}
/* FSE_writeNCount, FSE_readNCount */
{
S16 norm[129];
BYTE header[513];
U32 max, tableLog, i;
size_t headerSize;
for (i=0; i< TBSIZE; i++) testBuff[i] = i % 127;
max = 128;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
tableLog = FSE_optimalTableLog(0, TBSIZE, max);
errorCode = FSE_normalizeCount(norm, tableLog, count, TBSIZE, max);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
headerSize = FSE_NCountWriteBound(max, tableLog);
headerSize = FSE_writeNCount(header, 513, norm, max, tableLog);
CHECK(FSE_isError(headerSize), "Error : FSE_writeNCount() should have worked");
header[headerSize-1] = 0;
errorCode = FSE_writeNCount(header, headerSize-1, norm, max, tableLog);
CHECK(!FSE_isError(errorCode), "Error : FSE_writeNCount() should have failed");
CHECK (header[headerSize-1] != 0, "Error : FSE_writeNCount() buffer overwrite");
errorCode = FSE_writeNCount(header, headerSize+1, norm, max, tableLog);
CHECK(FSE_isError(errorCode), "Error : FSE_writeNCount() should have worked");
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
CHECK(FSE_isError(errorCode), "Error : FSE_readNCount() should have worked : (error %s)", FSE_getErrorName(errorCode));
max = 64;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (max too small)");
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize-1);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");
{
void* smallBuffer = malloc(headerSize-1); /* outbound read can be caught by valgrind */
CHECK(smallBuffer==NULL, "Error : Not enough memory (FSE_readNCount unit test)");
memcpy(smallBuffer, header, headerSize-1);
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, smallBuffer, headerSize-1);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");
free(smallBuffer);
}
}
/* FSE_buildCTable_raw & FSE_buildDTable_raw */
{
U32 ct[FSE_CTABLE_SIZE_U32(8, 256)];
U32 dt[FSE_DTABLE_SIZE_U32(8)];
U64 crcOrig, crcVerif;
size_t cSize, verifSize;
U32 i;
for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&seed) & 63) + '0';
crcOrig = XXH64(testBuff, TBSIZE, 0);
errorCode = FSE_buildCTable_raw(ct, 8);
CHECK(FSE_isError(errorCode), "FSE_buildCTable_raw should have worked");
errorCode = FSE_buildDTable_raw(dt, 8);
CHECK(FSE_isError(errorCode), "FSE_buildDTable_raw should have worked");
cSize = FSE_compress_usingCTable(cBuff, FSE_COMPRESSBOUND(TBSIZE), testBuff, TBSIZE, ct);
CHECK(FSE_isError(cSize), "FSE_compress_usingCTable should have worked using raw CTable");
verifSize = FSE_decompress_usingDTable(verifBuff, TBSIZE, cBuff, cSize, dt);
CHECK(FSE_isError(verifSize), "FSE_decompress_usingDTable should have worked using raw DTable");
crcVerif = XXH64(verifBuff, verifSize, 0);
CHECK(crcOrig != crcVerif, "Raw regenerated data is corrupted");
}
/* known corner case */
{
BYTE sample8[8] = { 0, 0, 0, 2, 0, 0, 0, 0 };
BYTE* rBuff;
errorCode = FSE_compress(cBuff, TBSIZE, sample8, 8);
CHECK(FSE_isError(errorCode), "FSE_compress failed compressing sample8");
rBuff = (BYTE*)malloc(errorCode); /* in order to catch read overflow with Valgrind */
CHECK(rBuff==NULL, "Not enough memory for rBuff");
memcpy(rBuff, cBuff, errorCode);
errorCode = FSE_decompress(verifBuff, sizeof(sample8), rBuff, errorCode);
CHECK(errorCode != sizeof(sample8), "FSE_decompress failed regenerating sample8");
free(rBuff);
}
free(testBuff);
free(cBuff);
free(verifBuff);
DISPLAY("Unit tests completed\n");
}
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
BYTE* bufferP0 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP1 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP15 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP90 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP100 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferDst = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
size_t bufferDstSize = BUFFERSIZE+64;
unsigned testNb, maxSV, tableLog;
const size_t maxTestSizeMask = 0x1FFFF;
U32 rootSeed = seed;
U32 time = FUZ_GetMilliStart();
generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
generate (bufferP1 , BUFFERSIZE, 0.01, &rootSeed);
generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);
if (startTestNb)
{
U32 i;
for (i=0; i<startTestNb; i++)
FUZ_rand (&rootSeed);
}
for (testNb=startTestNb; testNb<totalTest; testNb++)
{
BYTE* bufferTest;
int tag=0;
U32 roundSeed = rootSeed ^ 0xEDA5B371;
FUZ_rand(&rootSeed);
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
{
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
{
/* determine test sample */
size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U32 hashOrig;
if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
else
{
switch(FUZ_rand(&roundSeed) & 3)
{
case 0: bufferTest = bufferP0 + offset; break;
case 1: bufferTest = bufferP1 + offset; break;
case 2: bufferTest = bufferP90 + offset; break;
default : bufferTest = bufferP100 + offset; break;
}
}
DISPLAYLEVEL (4,"%3i ", tag++);;
hashOrig = XXH32 (bufferTest, sizeOrig, 0);
/* compress test */
sizeCompressed = FSE_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
CHECK(FSE_isError(sizeCompressed), "Compression failed !");
if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */
{
/* failed compression test*/
{
size_t errorCode;
void* tooSmallDBuffer = malloc(sizeCompressed-1); /* overflows detected with Valgrind */
CHECK(tooSmallDBuffer==NULL, "Not enough memory for tooSmallDBuffer test");
errorCode = FSE_compress (tooSmallDBuffer, sizeCompressed-1, bufferTest, sizeOrig);
CHECK(errorCode!=0, "Compression should have failed : destination buffer too small");
free(tooSmallDBuffer);
}
/* decompression test */
{
U32 hashEnd;
BYTE saved = (bufferVerif[sizeOrig] = 254);
size_t result = FSE_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "Output buffer overrun (bufferVerif) : write beyond specified end");
CHECK(FSE_isError(result), "Decompression failed");
hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
CHECK(hashEnd != hashOrig, "Decompressed data corrupted");
}
}
}
/* Attempt header decoding on bogus data */
{
short count[256];
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
maxSV = 255;
result = FSE_readNCount (count, &maxSV, &tableLog, bufferTest, FSE_NCOUNTBOUND);
if (!FSE_isError(result)) /* an error would be normal */
{
int checkCount;
CHECK(result > FSE_NCOUNTBOUND, "FSE_readHeader() reads too far (buffer overflow)");
CHECK(maxSV > 255, "count table overflow (%u)", maxSV+1);
checkCount = FUZ_checkCount(count, tableLog, maxSV);
CHECK(checkCount==-1, "symbol distribution corrupted");
}
}
/* Attempt decompression on bogus data */
{
size_t maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
size_t sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
BYTE saved = (bufferDst[maxDstSize] = 253);
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
result = FSE_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
CHECK(!FSE_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
CHECK(bufferDst[maxDstSize] != saved, "Output buffer bufferDst corrupted");
}
}
/* exit */
free (bufferP0);
free (bufferP1);
free (bufferP15);
free (bufferP90);
free (bufferP100);
free (bufferDst);
free (bufferVerif);
}
/*
Compressed format : MAGICNUMBER - STREAMDESCRIPTOR - ( BLOCKHEADER - COMPRESSEDBLOCK ) - STREAMCRC
MAGICNUMBER - 4 bytes - Designates compression algo
STREAMDESCRIPTOR - 1 byte
bits 0-3 : max block size, 2^value from 0 to 0xA; min 0=>1KB, max 0x6=>64KB, typical 5=>32 KB
bits 4-7 = 0 : reserved;
BLOCKHEADER - 1-5 bytes
1st byte :
bits 6-7 : blockType (compressed, raw, rle, crc (end of Frame)
bit 5 : full block
** if not full block **
2nd & 3rd byte : regenerated size of block (big endian); note : 0 = 64 KB
** if blockType==compressed **
next 2 bytes : compressed size of block
COMPRESSEDBLOCK
the compressed data itself.
STREAMCRC - 3 bytes (including 1-byte blockheader)
22 bits (xxh32() >> 5) checksum of the original data, big endian
*/
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
char* in_buff;
char* out_buff;
FILE* finput;
FILE* foutput;
size_t sizeCheck;
size_t inputBlockSize = FIO_blockID_to_blockSize(g_blockSizeId);
XXH32_state_t xxhState;
typedef size_t (*compressor_t) (void* dst, size_t dstSize, const void* src, size_t srcSize);
compressor_t compressor;
unsigned magicNumber;
/* Init */
XXH32_reset (&xxhState, FSE_CHECKSUM_SEED);
get_fileHandle(input_filename, output_filename, &finput, &foutput);
switch (g_compressor)
{
case FIO_fse:
compressor = FSE_compress;
magicNumber = FIO_magicNumber_fse;
break;
case FIO_huff0:
compressor = HUF_compress;
magicNumber = FIO_magicNumber_huff0;
break;
case FIO_zlibh:
compressor = FIO_ZLIBH_compress;
magicNumber = FIO_magicNumber_zlibh;
break;
default :
EXM_THROW(20, "unknown compressor selection");
}
/* Allocate Memory */
if (inputBlockSize==0) EXM_THROW(0, "impossible problem, to please static analyzer");
in_buff = (char*)malloc(inputBlockSize);
out_buff = (char*)malloc(FSE_compressBound(inputBlockSize) + 5);
if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
/* Write Frame Header */
FIO_writeLE32(out_buff, magicNumber);
out_buff[4] = (char)g_blockSizeId; /* Max Block Size descriptor */
sizeCheck = fwrite(out_buff, 1, FIO_FRAMEHEADERSIZE, foutput);
if (sizeCheck!=FIO_FRAMEHEADERSIZE) EXM_THROW(22, "Write error : cannot write header");
compressedfilesize += FIO_FRAMEHEADERSIZE;
/* Main compression loop */
while (1)
{
/* Fill input Buffer */
size_t cSize;
size_t inSize = fread(in_buff, (size_t)1, (size_t)inputBlockSize, finput);
if (inSize==0) break;
filesize += inSize;
XXH32_update(&xxhState, in_buff, inSize);
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */
cSize = compressor(out_buff + FIO_maxBlockHeaderSize, FSE_compressBound(inputBlockSize), in_buff, inSize);
if (FSE_isError(cSize)) EXM_THROW(23, "Compression error : %s ", FSE_getErrorName(cSize));
/* Write cBlock */
switch(cSize)
{
size_t headerSize;
case 0: /* raw */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_raw << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
if (sizeCheck!=headerSize) EXM_THROW(24, "Write error : cannot write block header");
sizeCheck = fwrite(in_buff, 1, inSize, foutput);
if (sizeCheck!=(size_t)(inSize)) EXM_THROW(25, "Write error : cannot write block");
compressedfilesize += inSize + headerSize;
break;
case 1: /* rle */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_rle << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
out_buff[headerSize] = in_buff[0];
sizeCheck = fwrite(out_buff, 1, headerSize+1, foutput);
if (sizeCheck!=(headerSize+1)) EXM_THROW(26, "Write error : cannot write rle block");
compressedfilesize += headerSize + 1;
break;
default : /* compressed */
if (inSize == inputBlockSize)
{
out_buff[2] = (BYTE)((bt_compressed << 6) + BIT5);
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = 3;
}
else
{
out_buff[0] = (BYTE)(bt_compressed << 6);
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[2] = (BYTE)inSize;
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = FIO_maxBlockHeaderSize;
}
sizeCheck = fwrite(out_buff+(FIO_maxBlockHeaderSize-headerSize), 1, headerSize+cSize, foutput);
if (sizeCheck!=(headerSize+cSize)) EXM_THROW(27, "Write error : cannot write rle block");
compressedfilesize += headerSize + cSize;
break;
}
static void FUZ_tests (const U32 startSeed, U32 totalTest, U32 startTestNb)
{
size_t bufferDstSize = BUFFERSIZE*sizeof(U16) + 64;
U16* bufferP8 = (U16*) malloc (bufferDstSize);
void* bufferDst = malloc (bufferDstSize);
U16* bufferVerif = (U16*) malloc (bufferDstSize);
unsigned testNb;
const size_t maxTestSizeMask = 0x1FFFF;
U32 time = FUZ_GetMilliStart();
U32 seed = startSeed;
generateU16 (bufferP8, BUFFERSIZE, 0.08, seed);
if (startTestNb)
{
U32 i;
for (i=0; i<startTestNb; i++)
FUZ_rand (&seed);
}
for (testNb=startTestNb; testNb<totalTest; testNb++)
{
U16* bufferTest;
int tag=0;
U32 roundSeed = seed ^ 0xEDA5B371;
FUZ_rand(&seed);
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
{
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
{
size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U64 hashOrig;
bufferTest = bufferP8 + offset;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
hashOrig = XXH64 (bufferTest, sizeOrig * sizeof(U16), 0);
sizeCompressed = FSE_compressU16 (bufferDst, bufferDstSize, bufferTest, sizeOrig, FSE_MAX_SYMBOL_VALUE, 12);
CHECK(FSE_isError(sizeCompressed), "\r test %5u : FSE_compressU16 failed !", testNb);
if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */
{
U64 hashEnd;
U16 saved = (bufferVerif[sizeOrig] = 1024 + 250);
size_t dstSize;
size_t result;
/* basic decompression test : should work */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
result = FSE_decompressU16 (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);
/* larger output buffer than necessary : should work */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
result = FSE_decompressU16 (bufferVerif, sizeOrig + (FUZ_rand(&roundSeed) & 31) + 1, bufferDst, sizeCompressed);
CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);
/* smaller output buffer than required : should fail */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
dstSize = (FUZ_rand(&roundSeed) & 31) + 1;
if (dstSize >= sizeOrig) dstSize = 1;
dstSize = sizeOrig - dstSize;
saved = (bufferVerif[dstSize] = 1024 + 250);
result = FSE_decompressU16 (bufferVerif, dstSize, bufferDst, sizeCompressed);
CHECK(bufferVerif[dstSize] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
CHECK(!FSE_isError(result), "\r test %5u : FSE_decompressU16 should have failed ! (origSize = %u shorts, dstSize = %u bytes)", testNb, (U32)sizeOrig, (U32)dstSize);
}
}
}
/* clean */
free (bufferP8);
free (bufferDst);
free (bufferVerif);
}
