static void FUZ_generateSynthetic(void* buffer, size_t bufferSize, double proba, U32* seed)
{
BYTE* BBuffer = (BYTE*)buffer;
unsigned pos = 0;
U32 P32 = (U32)(32768 * proba);
// First Byte
BBuffer[pos++] = (BYTE)((FUZ_rand(seed) & 0x3F) + '0');
while (pos < bufferSize)
{
// Select : Literal (noise) or copy (within 64K)
if (FUZ_RAND15BITS < P32)
{
// Copy (within 64K)
size_t match, end;
size_t length = FUZ_RANDLENGTH + 4;
size_t offset = FUZ_RAND15BITS + 1;
if (offset > pos) offset = pos;
if (pos + length > bufferSize) length = bufferSize - pos;
match = pos - offset;
end = pos + length;
while (pos < end) BBuffer[pos++] = BBuffer[match++];
}
else
{
// Literal (noise)
size_t end;
size_t length = FUZ_RANDLENGTH;
if (pos + length > bufferSize) length = bufferSize - pos;
end = pos + length;
while (pos < end) BBuffer[pos++] = (BYTE)((FUZ_rand(seed) & 0x3F) + '0');
}
}
}
void FUZ_fillCompressibleNoiseBuffer(void* buffer, int bufferSize, double proba, U32* seed)
{
BYTE* BBuffer = (BYTE*)buffer;
int pos = 0;
U32 P32 = (U32)(32768 * proba);
// First Byte
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
// Select : Literal (noise) or copy (within 64K)
if (FUZ_RAND15BITS < P32)
{
// Copy (within 64K)
int ref, d;
int length = FUZ_RANDLENGTH + 4;
int offset = FUZ_RAND15BITS + 1;
if (offset > pos) offset = pos;
if (pos + length > bufferSize) length = bufferSize - pos;
ref = pos - offset;
d = pos + length;
while (pos < d) BBuffer[pos++] = BBuffer[ref++];
}
else
{
// Literal (noise)
int d;
int length = FUZ_RANDLENGTH;
if (pos + length > bufferSize) length = bufferSize - pos;
d = pos + length;
while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
}
}
}
static void generate (void* buffer, size_t buffSize, double p, U32* seed)
{
char table[PROBATABLESIZE] = {0};
int remaining = PROBATABLESIZE;
int pos = 0;
int s = 0;
char* op = (char*) buffer;
char* oend = op + buffSize;
/* Build Table */
while (remaining)
{
int n = (int) (remaining * p);
int end;
if (!n) n=1;
end = pos + n;
while (pos<end) table[pos++]= (char) s;
s++;
remaining -= n;
}
/* Fill buffer */
while (op<oend)
{
const int r = FUZ_rand (seed) & (PROBATABLESIZE-1);
*op++ = table[r];
}
}
static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed)
{
BYTE* BBuffer = (BYTE*)buffer;
unsigned pos = 0;
U32 P32 = (U32)(32768 * proba);
/* First Byte */
BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
/* Select : Literal (noise) or copy (within 64K) */
if (FUZ_RAND15BITS < P32)
{
/* Copy (within 64K) */
unsigned match, end;
unsigned length = FUZ_RANDLENGTH + 4;
unsigned offset = FUZ_RAND15BITS + 1;
if (offset > pos) offset = pos;
if (pos + length > bufferSize) length = bufferSize - pos;
match = pos - offset;
end = pos + length;
while (pos < end) BBuffer[pos++] = BBuffer[match++];
}
else
{
/* Literal (noise) */
unsigned end;
unsigned length = FUZ_RANDLENGTH;
if (pos + length > bufferSize) length = bufferSize - pos;
end = pos + length;
while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
}
}
}
static void generateU16 (U16* buffer, size_t buffSize, double p, U32 seedSrc)
{
U16 tableU16[PROBATABLESIZE];
U32 remaining = PROBATABLESIZE;
U32 pos = 0;
U16* op = buffer;
U16* const oend = op + buffSize;
U16 val16 = 240;
U16 max16 = FSE_MAX_SYMBOL_VALUE;
U32 seed = seedSrc;
/* Build Symbol Table */
while (remaining)
{
const U32 n = (U32) (remaining * p) + 1;
const U32 end = pos + n;
while (pos<end) tableU16[pos++] = val16;
val16++;
if (val16 >= max16) val16 = 1;
remaining -= n;
}
/* Fill buffer */
while (op<oend)
{
const U32 r = FUZ_rand(&seed) & (PROBATABLESIZE-1);
*op++ = tableU16[r];
}
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* cNoiseBuffer[5];
BYTE* srcBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
size_t dstBufferSize = srcBufferSize;
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZBUFF_CCtx* zc;
ZBUFF_DCtx* zd;
XXH64_state_t crc64;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
zc = ZBUFF_createCCtx();
zd = ZBUFF_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
copyBuffer= (BYTE*)malloc (copyBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
srcBuffer = cNoiseBuffer[2];
memset(copyBuffer, 0x65, copyBufferSize);
memcpy(copyBuffer, srcBuffer, MIN(copyBufferSize,srcBufferSize)); /* make copyBuffer considered initialized */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart;
size_t cSize;
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
size_t errorCode;
U32 sampleSizeLog, buffNb, n, nbChunks;
U64 crcOrig, crcDest;
/* init */
DISPLAYUPDATE(2, "\r%6u", testNb);
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2; /* select buffer */
else
{
buffNb >>= 3;
if (buffNb & 7)
{
const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3];
}
else
{
const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3];
}
}
srcBuffer = cNoiseBuffer[buffNb];
/* Multi - segments compression test */
XXH64_reset(&crc64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
totalTestSize = 0;
cSize = 0;
for (n=0; n<nbChunks; n++)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
readSize = sampleSize;
/* random size output buffer */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
XXH64_update(&crc64, srcBuffer+sampleStart, readSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
cSize += genSize;
totalTestSize += readSize;
if ((FUZ_rand(&lseed) & 15) == 0)
{
/* add a few random flushes operations, to mess around */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
cSize += genSize;
}
if (totalTestSize > maxTestSize) break;
}
genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
CHECK (errorCode != 0, "frame epilogue not fully consumed");
cSize += genSize;
crcOrig = XXH64_digest(&crc64);
/* multi - fragments decompression test */
ZBUFF_decompressInit(zd);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalGenSize += genSize;
totalCSize += readSize;
}
CHECK (errorCode != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
/* noisy/erroneous src decompression test */
/* add some noise */
nbChunks = (FUZ_rand(&lseed) & 7) + 2;
for (n=0; n<nbChunks; n++)
{
size_t cStart;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
if (sampleSize > cSize/3) sampleSize = cSize/3;
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
cStart = FUZ_rand(&lseed) % (cSize - sampleSize);
memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize);
}
/* try decompression on noisy data */
ZBUFF_decompressInit(zd);
totalCSize = 0;
totalGenSize = 0;
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) )
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
readSize = sampleSize;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
if (ZBUFF_isError(errorCode)) break; /* error correctly detected */
totalGenSize += genSize;
totalCSize += readSize;
}
}
int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
unsigned long long hcbytes = 0;
unsigned long long ccbytes = 0;
void* CNBuffer;
char* compressedBuffer;
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
unsigned int randState=seed;
int ret, cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
U32 crcOrig, crcCheck;
int displayRefresh;
// init
memset(&LZ4dict, 0, sizeof(LZ4dict));
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// display refresh rate
switch(displayLevel)
{
case 0: displayRefresh = nbCycles+1; break;
case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
case 2: displayRefresh=89; break;
default : displayRefresh=1;
}
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
// synd rand & dict
int dictSize, blockSize, blockStart;
char* dict;
char* block;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
int testNb = 0;
char* dict;
char* block;
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
if ((cycleNb%displayRefresh) == 0)
{
printf("\r%7i /%7i - ", cycleNb, nbCycles);
fflush(stdout);
}
// Select block to test
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
/* Compression tests */
// Test compression HC
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
// Test compression HC using external state
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
compressedSize = ret;
/* Decompression tests */
crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with more than enough output size => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
decodedBuffer[blockSize+1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with output size being one byte too short => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
// Test decoding with output size being 10 bytes too short => must fail
FUZ_DISPLAYTEST;
if (blockSize>10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
}
// Test decoding with input size being one byte too short => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
// Test decoding with input size being one byte too large => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
// Test partial decoding with target output size being max/2 => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test partial decoding with target output size being just below max => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
/* Test Compression with limited output size */
// Test compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
// Test compression with output size being exactly what's necessary and external state (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
// Test compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
/* Dictionary tests */
// Compress using dictionary
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
LZ4_free (LZ4continue);
// Decompress with dictionary as prefix
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
// Compress using External dictionary
FUZ_DISPLAYTEST;
dict -= 9; // Separation, so it is an ExtDict
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
if (blockSize > 10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
}
// Fill stats
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
printf("\r%7i /%7i - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
// unalloc
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 0;
_output_error:
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 1;
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* cNoiseBuffer[5];
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
BYTE* mirrorBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* refCtx;
ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
refCtx = ZSTD_createCCtx();
ctx = ZSTD_createCCtx();
dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
srcBuffer = cNoiseBuffer[2];
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig, crcDest;
int cLevel;
BYTE* sampleBuffer;
const BYTE* dict;
size_t dictSize;
/* init */
if (nbTests >= testNb)
{ DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2;
else
{
buffNb >>= 3;
if (buffNb & 7)
{
const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3];
}
else
{
const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3];
}
}
srcBuffer = cNoiseBuffer[buffNb];
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
/* create sample buffer (to catch read error with valgrind & sanitizers) */
sampleBuffer = (BYTE*)malloc(sampleSize);
CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression test */
cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
/* compression failure test : too small dest buffer */
if (cSize > 3)
{
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
static const U32 endMark = 0x4DC2B1A9;
U32 endCheck;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
}
/* successfull decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
free(sampleBuffer); /* no longer useful after this point */
/* truncated src decompression test */
{
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)");
free(cBufferTooSmall);
}
/* too small dst decompression test */
if (sampleSize > 3)
{
const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = sampleSize - missing;
static const BYTE token = 0xA9;
dstBuffer[tooSmallSize] = token;
errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
}
/* noisy src decompression test */
if (cSize > 6)
{
const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4));
size_t pos = 4; /* preserve magic number (too easy to detect) */
U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
size_t mask = (1<<nbBits) - 1;
size_t skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
while (pos < cSize)
{
/* add noise */
size_t noiseStart, noiseLength;
nbBits = FUZ_rand(&lseed) % maxNbBits;
if (nbBits>0) nbBits--;
mask = (1<<nbBits) - 1;
noiseLength = (FUZ_rand(&lseed) & mask) + 1;
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength;
/* keep some original src */
nbBits = FUZ_rand(&lseed) % maxNbBits;
mask = (1<<nbBits) - 1;
skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
}
/* decompress noisy source */
{
U32 noiseSrc = FUZ_rand(&lseed) % 5;
const U32 endMark = 0xA9B1C3D6;
U32 endCheck;
srcBuffer = cNoiseBuffer[noiseSrc];
memcpy(dstBuffer+sampleSize, &endMark, 4);
errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
/* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
memcpy(&endCheck, dstBuffer+sampleSize, 4);
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
}
}
/* Streaming compression of scattered segments test */
XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
errorCode = ZSTD_compressBegin(refCtx, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_duplicateCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0; cSize = 0;
for (n=0; n<nbChunks; n++)
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
/* avoid invalid dstBufferTooSmall */
break;
if (totalTestSize+sampleSize > maxTestSize) break;
errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize;
}
errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
crcOrig = XXH64_digest(xxh64);
/* streaming decompression test */
errorCode = ZSTD_resetDCtx(dctx);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
totalGenSize += genSize;
totalCSize += inSize;
}
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig)
errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
}
static void unitTest(void)
{
BYTE* testBuff = (BYTE*)malloc(TBSIZE);
BYTE* cBuff = (BYTE*)malloc(FSE_COMPRESSBOUND(TBSIZE));
BYTE* verifBuff = (BYTE*)malloc(TBSIZE);
size_t errorCode;
U32 seed=0, testNb=0, lseed=0;
U32 count[256];
if ((!testBuff) || (!cBuff) || (!verifBuff))
{
DISPLAY("Not enough memory, exiting ... \n");
free(testBuff);
free(cBuff);
free(verifBuff);
return;
}
/* FSE_count */
{
U32 max, i;
for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&lseed) & 63) + '0';
max = '0' + 63;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
max -= 1;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(!FSE_isError(errorCode), "Error : FSE_count() should have failed : value > max");
max = 65000;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
}
/* FSE_optimalTableLog */
{
U32 max, i, tableLog=12;
size_t testSize = 999;
for (i=0; i< testSize; i++) testBuff[i] = (BYTE)FUZ_rand(&lseed);
max = 256;
FSE_count(count, &max, testBuff, testSize);
tableLog = FSE_optimalTableLog(tableLog, testSize, max);
CHECK(tableLog<=8, "Too small tableLog");
}
/* FSE_normalizeCount */
{
S16 norm[256];
U32 max = 256;
FSE_count(count, &max, testBuff, TBSIZE);
errorCode = FSE_normalizeCount(norm, 10, count, TBSIZE, max);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
errorCode = FSE_normalizeCount(norm, 8, count, TBSIZE, 256);
CHECK(!FSE_isError(errorCode), "Error : FSE_normalizeCount() should have failed (max >= 1<<tableLog)");
/* limit corner case : try to make internal rank overflow */
{
U32 i;
U32 total = 0;
count[0] = 940;
count[1] = 910;
count[2] = 470;
count[3] = 190;
count[4] = 90;
for(i=5; i<=255; i++) count[i] = 6;
for (i=0; i<=255; i++) total += count[i];
errorCode = FSE_normalizeCount(norm, 10, count, total, 255);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
count[0] = 300;
count[1] = 300;
count[2] = 300;
count[3] = 300;
count[4] = 50;
for(i=5; i<=80; i++) count[i] = 4;
total = 0; for (i=0; i<=80; i++) total += count[i];
errorCode = FSE_normalizeCount(norm, 10, count, total, 80);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
}
}
/* FSE_writeNCount, FSE_readNCount */
{
S16 norm[129];
BYTE header[513];
U32 max, tableLog, i;
size_t headerSize;
for (i=0; i< TBSIZE; i++) testBuff[i] = i % 127;
max = 128;
errorCode = FSE_count(count, &max, testBuff, TBSIZE);
CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked");
tableLog = FSE_optimalTableLog(0, TBSIZE, max);
errorCode = FSE_normalizeCount(norm, tableLog, count, TBSIZE, max);
CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked");
headerSize = FSE_NCountWriteBound(max, tableLog);
headerSize = FSE_writeNCount(header, 513, norm, max, tableLog);
CHECK(FSE_isError(headerSize), "Error : FSE_writeNCount() should have worked");
header[headerSize-1] = 0;
errorCode = FSE_writeNCount(header, headerSize-1, norm, max, tableLog);
CHECK(!FSE_isError(errorCode), "Error : FSE_writeNCount() should have failed");
CHECK (header[headerSize-1] != 0, "Error : FSE_writeNCount() buffer overwrite");
errorCode = FSE_writeNCount(header, headerSize+1, norm, max, tableLog);
CHECK(FSE_isError(errorCode), "Error : FSE_writeNCount() should have worked");
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
CHECK(FSE_isError(errorCode), "Error : FSE_readNCount() should have worked : (error %s)", FSE_getErrorName(errorCode));
max = 64;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (max too small)");
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize-1);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");
{
void* smallBuffer = malloc(headerSize-1); /* outbound read can be caught by valgrind */
CHECK(smallBuffer==NULL, "Error : Not enough memory (FSE_readNCount unit test)");
memcpy(smallBuffer, header, headerSize-1);
max = 129;
errorCode = FSE_readNCount(norm, &max, &tableLog, smallBuffer, headerSize-1);
CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)");
free(smallBuffer);
}
}
/* FSE_buildCTable_raw & FSE_buildDTable_raw */
{
U32 ct[FSE_CTABLE_SIZE_U32(8, 256)];
U32 dt[FSE_DTABLE_SIZE_U32(8)];
U64 crcOrig, crcVerif;
size_t cSize, verifSize;
U32 i;
for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&seed) & 63) + '0';
crcOrig = XXH64(testBuff, TBSIZE, 0);
errorCode = FSE_buildCTable_raw(ct, 8);
CHECK(FSE_isError(errorCode), "FSE_buildCTable_raw should have worked");
errorCode = FSE_buildDTable_raw(dt, 8);
CHECK(FSE_isError(errorCode), "FSE_buildDTable_raw should have worked");
cSize = FSE_compress_usingCTable(cBuff, FSE_COMPRESSBOUND(TBSIZE), testBuff, TBSIZE, ct);
CHECK(FSE_isError(cSize), "FSE_compress_usingCTable should have worked using raw CTable");
verifSize = FSE_decompress_usingDTable(verifBuff, TBSIZE, cBuff, cSize, dt);
CHECK(FSE_isError(verifSize), "FSE_decompress_usingDTable should have worked using raw DTable");
crcVerif = XXH64(verifBuff, verifSize, 0);
CHECK(crcOrig != crcVerif, "Raw regenerated data is corrupted");
}
/* known corner case */
{
BYTE sample8[8] = { 0, 0, 0, 2, 0, 0, 0, 0 };
BYTE* rBuff;
errorCode = FSE_compress(cBuff, TBSIZE, sample8, 8);
CHECK(FSE_isError(errorCode), "FSE_compress failed compressing sample8");
rBuff = (BYTE*)malloc(errorCode); /* in order to catch read overflow with Valgrind */
CHECK(rBuff==NULL, "Not enough memory for rBuff");
memcpy(rBuff, cBuff, errorCode);
errorCode = FSE_decompress(verifBuff, sizeof(sample8), rBuff, errorCode);
CHECK(errorCode != sizeof(sample8), "FSE_decompress failed regenerating sample8");
free(rBuff);
}
free(testBuff);
free(cBuff);
free(verifBuff);
DISPLAY("Unit tests completed\n");
}
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
BYTE* bufferP0 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP1 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP15 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP90 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP100 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferDst = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
size_t bufferDstSize = BUFFERSIZE+64;
unsigned testNb, maxSV, tableLog;
const size_t maxTestSizeMask = 0x1FFFF;
U32 rootSeed = seed;
U32 time = FUZ_GetMilliStart();
generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
generate (bufferP1 , BUFFERSIZE, 0.01, &rootSeed);
generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);
if (startTestNb)
{
U32 i;
for (i=0; i<startTestNb; i++)
FUZ_rand (&rootSeed);
}
for (testNb=startTestNb; testNb<totalTest; testNb++)
{
BYTE* bufferTest;
int tag=0;
U32 roundSeed = rootSeed ^ 0xEDA5B371;
FUZ_rand(&rootSeed);
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
{
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
{
/* determine test sample */
size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U32 hashOrig;
if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
else
{
switch(FUZ_rand(&roundSeed) & 3)
{
case 0: bufferTest = bufferP0 + offset; break;
case 1: bufferTest = bufferP1 + offset; break;
case 2: bufferTest = bufferP90 + offset; break;
default : bufferTest = bufferP100 + offset; break;
}
}
DISPLAYLEVEL (4,"%3i ", tag++);;
hashOrig = XXH32 (bufferTest, sizeOrig, 0);
/* compress test */
sizeCompressed = FSE_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
CHECK(FSE_isError(sizeCompressed), "Compression failed !");
if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */
{
/* failed compression test*/
{
size_t errorCode;
void* tooSmallDBuffer = malloc(sizeCompressed-1); /* overflows detected with Valgrind */
CHECK(tooSmallDBuffer==NULL, "Not enough memory for tooSmallDBuffer test");
errorCode = FSE_compress (tooSmallDBuffer, sizeCompressed-1, bufferTest, sizeOrig);
CHECK(errorCode!=0, "Compression should have failed : destination buffer too small");
free(tooSmallDBuffer);
}
/* decompression test */
{
U32 hashEnd;
BYTE saved = (bufferVerif[sizeOrig] = 254);
size_t result = FSE_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "Output buffer overrun (bufferVerif) : write beyond specified end");
CHECK(FSE_isError(result), "Decompression failed");
hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
CHECK(hashEnd != hashOrig, "Decompressed data corrupted");
}
}
}
/* Attempt header decoding on bogus data */
{
short count[256];
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
maxSV = 255;
result = FSE_readNCount (count, &maxSV, &tableLog, bufferTest, FSE_NCOUNTBOUND);
if (!FSE_isError(result)) /* an error would be normal */
{
int checkCount;
CHECK(result > FSE_NCOUNTBOUND, "FSE_readHeader() reads too far (buffer overflow)");
CHECK(maxSV > 255, "count table overflow (%u)", maxSV+1);
checkCount = FUZ_checkCount(count, tableLog, maxSV);
CHECK(checkCount==-1, "symbol distribution corrupted");
}
}
/* Attempt decompression on bogus data */
{
size_t maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
size_t sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
BYTE saved = (bufferDst[maxDstSize] = 253);
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
result = FSE_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
CHECK(!FSE_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
CHECK(bufferDst[maxDstSize] != saved, "Output buffer bufferDst corrupted");
}
}
/* exit */
free (bufferP0);
free (bufferP1);
free (bufferP15);
free (bufferP90);
free (bufferP100);
free (bufferDst);
free (bufferVerif);
}
static void generateNoise (void* buffer, size_t buffSize, U32* seed)
{
BYTE* op = (BYTE*)buffer;
BYTE* const oend = op + buffSize;
while (op<oend) *op++ = (BYTE)FUZ_rand(seed);
}
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
{
unsigned testResult = 0;
unsigned testNb = 0;
void* srcBuffer = NULL;
void* compressedBuffer = NULL;
void* decodedBuffer = NULL;
U32 coreRand = seed;
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cCtx = NULL;
size_t result;
const U32 startTime = FUZ_GetMilliStart();
XXH64_state_t xxh64;
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
// backup all allocated addresses, from which we will later select buffers
const size_t max_buf_size = 131 KB;
size_t num_buf_size_distribution_deviations = 0;
LZ4SG_in_t sg_in_buf_potential [2*MAX_SG_BUFFERS];
LZ4SG_out_t sg_out_buf_potential[2*MAX_SG_BUFFERS];
LZ4SG_in_t sg_cin [MAX_SG_BUFFERS];
LZ4SG_out_t sg_cout[MAX_SG_BUFFERS];
LZ4SG_in_t sg_din [MAX_SG_BUFFERS];
LZ4SG_out_t sg_dout[MAX_SG_BUFFERS];
size_t sg_cin_len, sg_cout_len, sg_din_len, sg_dout_len;
const size_t maxDstSize = LZ4_SG_compressBound(srcDataLength, NELEMS(sg_cin), NELEMS(sg_cout));
unsigned int i;
for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
sg_in_buf_potential [i].sg_base = malloc(max_buf_size);
sg_in_buf_potential [i].sg_len = max_buf_size;
sg_out_buf_potential[i].sg_base = malloc(max_buf_size);
sg_out_buf_potential[i].sg_len = max_buf_size;
}
/* Init */
duration *= 1000;
/* Create buffers */
result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
srcBuffer = malloc(srcDataLength);
CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
const size_t compressedBufferLength = maxDstSize;
compressedBuffer = malloc(compressedBufferLength);
CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
/* jump to requested testNb */
for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
/* main fuzzer test loop */
for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
{
U32 randState = coreRand ^ prime1;
(void)FUZ_rand(&coreRand); /* update seed */
srand48(FUZ_rand(&randState));
DISPLAYUPDATE(2, "\r%5u ", testNb);
const size_t max_src_buf_size = (4 MB > srcDataLength) ? srcDataLength : 4 MB;
unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(max_src_buf_size-1) - 1)) + 1;
const size_t min_src_size = 20;
const size_t min_first_dest_buf_size = 21;
const size_t min_src_buf_size = 1;
const size_t min_dst_buf_size = 10;
size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + min_src_size;
size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
size_t cSize;
size_t dstSize;
size_t dstSizeBound;
U64 crcOrig, crcDecoded;
unsigned int test_selection = FUZ_rand(&randState);
//TODO: enable lz4f_compress_compatibility_test with LZ4_SG_decompress
int lz4f_compress_compatibility_test = 0;//(test_selection % 4) == 0;
if (!lz4f_compress_compatibility_test)
{
// SGL compress
unsigned int buffer_selection = FUZ_rand(&randState);
if ((buffer_selection & 0xF) == 1)
{
// SG compress single source and single target buffers
sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
sg_cin[0].sg_len = srcSize;
sg_cin_len = 1;
sg_cout[0].sg_base = compressedBuffer;
sg_cout[0].sg_len = compressedBufferLength;
sg_cout_len = 1;
dstSizeBound = dstSize = compressedBufferLength;
}
else
{
// SG compress random number and size source and target buffers
sg_cin_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);
sg_cout_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);
// single source buffer
if (1 == sg_cin_len) {
sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
sg_cin[0].sg_len = srcSize;
DISPLAYUPDATE(4, "INFO: single source buf size %i\n", (int)srcSize);
}
else {
// multiple source buffers
if (srcSize > sg_cin_len*max_buf_size/2) {
srcSize = sg_cin_len*max_buf_size/2;
num_buf_size_distribution_deviations++;
DISPLAYUPDATE(4, "NOTE: source buffer total size deviation %i\n", (int)num_buf_size_distribution_deviations);
}
size_t exact_src_size = 0;
unsigned int buf_size_mean = srcSize / sg_cin_len;
for (i = 0; i < sg_cin_len; i++) {
size_t buf_size = rnd_exponential(buf_size_mean, min_src_buf_size, max_buf_size);
DISPLAYUPDATE(4, "INFO: source buf %i size %i\n", i, (int)buf_size);
if (srcStart+exact_src_size+buf_size > srcDataLength) {
buf_size = srcDataLength-(srcStart+exact_src_size);
}
sg_cin[i].sg_base = sg_in_buf_potential[i*2+1].sg_base;
sg_cin[i].sg_len = buf_size;
memcpy((void *)sg_cin[i].sg_base, (BYTE*)srcBuffer+srcStart+exact_src_size, buf_size);
exact_src_size += buf_size;
if (srcStart+exact_src_size == srcDataLength) {
num_buf_size_distribution_deviations++;
sg_cin_len = i+1;
DISPLAYUPDATE(4, "NOTE: final source buffer size deviation %i (buffers number limited to %i)\n", (int)num_buf_size_distribution_deviations, (int)sg_cin_len);
}
}
srcSize = exact_src_size;
}
// we can now derive the required limit for output
dstSizeBound = LZ4_SG_compressBound(srcSize, sg_cin_len, sg_cout_len);
// single target buffer
if (1 == sg_cout_len) {
sg_cout[0].sg_base = compressedBuffer;
sg_cout[0].sg_len = compressedBufferLength;
}
else {
// multiple target buffers
int finalBufferTruncated = 0;
dstSize = 0;
unsigned int buf_size_mean = dstSizeBound / sg_cout_len;
for (i = 0; i < sg_cout_len; i++) {
const size_t min_buf_size = (i == 0) ? min_first_dest_buf_size : min_dst_buf_size;
size_t buf_size = rnd_exponential(buf_size_mean, min_buf_size, max_buf_size);
DISPLAYUPDATE(4, "INFO: target buf %i size %i\n", (int)i, (int)buf_size);
if (dstSize+buf_size > dstSizeBound) {
buf_size = dstSizeBound-dstSize;
finalBufferTruncated = 1;
}
dstSize += buf_size;
sg_cout[i].sg_base = sg_out_buf_potential[i*2+1].sg_base;
sg_cout[i].sg_len = buf_size;
if (finalBufferTruncated) {
num_buf_size_distribution_deviations++;
if (buf_size < min_buf_size) {
// merge truncated with previous?
if (i > 0) {
sg_cout[i-1].sg_len += buf_size;
if (sg_cout[i-1].sg_len > max_buf_size) {
// skip, too much hassle
DISPLAYUPDATE(4, "NOTE: unable to truncate final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
sg_cout_len = 0; break;
}
}
else {
// can this happen?
DISPLAYUPDATE(4, "NOTE: unable to truncate first and final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
sg_cout_len = 0; break;
}
sg_cout_len = i;
}
else {
sg_cout_len = i+1;
}
DISPLAYUPDATE(4, "NOTE: final target buffer size truncated (%i), buffers number limited to %i, final's size is now %i (deviations %i)\n",
(int)buf_size, (int)sg_cout_len, (int)sg_cout[sg_cout_len-1].sg_len, (int)num_buf_size_distribution_deviations);
}
}
// skip/abort condition
if (0 == sg_cout_len) continue;
}
if ((buffer_selection & 0xF) == 0) {
//TODO: select a random input and output buffer and split it in two,
// feeding consecutive addresses as consecutive entries in SGL
}
}
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
size_t sourceSizeOut = srcSize;
result = LZ4_SG_compress(&sg_cin[0], sg_cin_len, &sg_cout[0], sg_cout_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
if (((result == 0) || (sourceSizeOut != srcSize)) && (dstSize < dstSizeBound)) {
// forgive compression failure when output total size is lower than bound
num_buf_size_distribution_deviations++;
DISPLAYUPDATE(4, "NOTE: dstSize %i < %i dstSizeBound, compression attempt failed, not totally unexpected (deviations %i), skipping\n",
(int)dstSize, (int)dstSizeBound, (int)num_buf_size_distribution_deviations);
continue;
}
CHECK(result <= 0, "Compression failed (error %i)", (int)result);
CHECK(sourceSizeOut != srcSize, "Compression stopped at %i out of %i", (int)sourceSizeOut, (int)srcSize);
cSize = result;
}
else
{
// LZ4F compression - use it in order to verify SGL decompress compatibility with it
DISPLAYUPDATE(4, "INFO: LZ4F compression\n");
// alternative
// size_t dstMaxSize = LZ4F_compressFrameBound(srcSize, prefsPtr);
// DISPLAYLEVEL(3, "compressFrame srcSize %zu dstMaxSize %zu\n",
// srcSize, dstMaxSize);
// cSize = LZ4F_compressFrame(compressedBuffer, dstMaxSize, (char*)srcBuffer + srcStart, srcSize, prefsPtr);
// CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
unsigned BSId = 4 + (FUZ_rand(&randState) & 3);
unsigned BMId = FUZ_rand(&randState) & 1;
unsigned CCflag = FUZ_rand(&randState) & 1;
unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
LZ4F_preferences_t prefs;
LZ4F_compressOptions_t cOptions;
LZ4F_preferences_t* prefsPtr = &prefs;
memset(&prefs, 0, sizeof(prefs));
memset(&cOptions, 0, sizeof(cOptions));
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
prefs.frameInfo.contentSize = frameContentSize;
prefs.autoFlush = autoflush;
prefs.compressionLevel = FUZ_rand(&randState) % 5;
if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
const BYTE* ip = (const BYTE*)srcBuffer + srcStart;
const BYTE* const iend = ip + srcSize;
BYTE* op = (BYTE*)compressedBuffer;
BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL);
unsigned maxBits = FUZ_highbit((U32)srcSize);
result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result);
op += result;
while (ip < iend)
{
unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
size_t oSize = LZ4F_compressBound(iSize, prefsPtr);
unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
DISPLAYLEVEL(3, "compressUpdate ip %d iSize %zu oSize %zu forceFlush %d\n",
(int)(ip-((const BYTE*)srcBuffer + srcStart)), iSize, oSize, forceFlush);
result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
op += result;
ip += iSize;
if (forceFlush)
{
result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
op += result;
}
}
result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
op += result;
cSize = op-(BYTE*)compressedBuffer;
}
//DECOMPRESS
test_selection = FUZ_rand(&randState);
if (lz4f_compress_compatibility_test || ((test_selection % 2) == 0))
{
//TODO: SGL decompress with random buffer sizes
// SGL decompress with same buffer sizes used for compression
// prepare din with cout's data
sg_din_len = sg_cout_len;
for (i = 0; i < sg_din_len; i++) {
sg_din[i].sg_len = sg_cout[i].sg_len;
if (sg_cout[i].sg_len <= max_buf_size) {
// enough room to copy - do it
sg_din[i].sg_base = sg_in_buf_potential[i*2+0].sg_base;
if (sg_din[i].sg_base != sg_cout[i].sg_base) {
memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, sg_cout[i].sg_len);
}
}
else {
// this is probably single output buffer - skip copy, use directly
sg_din[i].sg_base = sg_cout[i].sg_base;
}
}
// prepare dout to receive decompressed data
sg_dout_len = sg_cin_len;
for (i = 0; i < sg_dout_len; i++) {
sg_dout[i].sg_len = sg_cin[i].sg_len;
if (sg_cin[i].sg_len <= max_buf_size) {
// enough room to decompress into independent buffer
sg_dout[i].sg_base = sg_out_buf_potential[i*2+0].sg_base;
}
else {
// this is probably single input buffer, use an external output buffer
sg_dout[i].sg_base = decodedBuffer;
}
}
size_t sourceSizeOut = cSize;
size_t maxOutputSize = srcSize;
int decomp_result = LZ4_SG_decompress(&sg_din[0], sg_din_len, &sg_dout[0], sg_dout_len, &sourceSizeOut, maxOutputSize);
CHECK(decomp_result <= 0, "SG decompression failed (error %i)", (int)decomp_result);
CHECK(decomp_result != (int)srcSize, "SG decompression stopped at %i", (int)decomp_result);
// verify result checksum
size_t total_checked = 0;
XXH64_reset(&xxh64, 1);
for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
size_t cur_size = sg_dout[i].sg_len;
size_t rem = decomp_result - total_checked;
if (rem < cur_size) cur_size = rem;
total_checked += cur_size;
XXH64_update(&xxh64, sg_dout[i].sg_base, cur_size);
}
crcDecoded = XXH64_digest(&xxh64);
if (crcDecoded != crcOrig) {
DISPLAYLEVEL(1, "checked %i out of %i (crcDecoded %08x, crcOrig %08x)\n",
(int)total_checked, decomp_result, (unsigned)crcDecoded, (unsigned)crcOrig);
// locate error if any
total_checked = 0;
for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
size_t cur_size = sg_dout[i].sg_len;
size_t rem = decomp_result - total_checked;
if (rem < cur_size) cur_size = rem;
total_checked += cur_size;
U64 crc_in = XXH64(sg_cin [i].sg_base, cur_size, 1);
U64 crc_out = XXH64(sg_dout[i].sg_base, cur_size, 1);
if (crc_in != crc_out) {
locateBuffDiff(sg_cin[i].sg_base, sg_dout[i].sg_base, cur_size);
break;
}
}
DISPLAYLEVEL(1, "checked %i out of %i\n",
(int)total_checked, decomp_result);
}
CHECK(crcDecoded != crcOrig, "Decompression corruption");
}
else
{
// prepare compressedBuffer from SGL
size_t total_copied = 0;
for (i = 0; i < sg_cout_len; i++) {
size_t buf_size_bytes = cSize - total_copied;
if (buf_size_bytes == 0) break;
if (buf_size_bytes > sg_cout[i].sg_len) buf_size_bytes = sg_cout[i].sg_len;
if (((char *)compressedBuffer)+total_copied != sg_cout[i].sg_base) {
memcpy(((char *)compressedBuffer)+total_copied, sg_cout[i].sg_base, buf_size_bytes);
}
total_copied += buf_size_bytes;
}
LZ4F_decompressOptions_t dOptions;
memset(&dOptions, 0, sizeof(dOptions));
const BYTE* ip = (const BYTE*)compressedBuffer;
const BYTE* const iend = ip + cSize;
BYTE* op = (BYTE*)decodedBuffer;
BYTE* const oend = op + srcDataLength;
size_t totalOut = 0;
unsigned maxBits = FUZ_highbit((U32)cSize);
XXH64_reset(&xxh64, 1);
if (maxBits < 3) maxBits = 3;
while (ip < iend)
{
unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
if (oSize > (size_t)(oend-op)) oSize = oend-op;
dOptions.stableDst = FUZ_rand(&randState) & 1;
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s ip %d)",
(int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result), (int)(ip-(const BYTE*)compressedBuffer));
XXH64_update(&xxh64, op, (U32)oSize);
totalOut += oSize;
op += oSize;
ip += iSize;
}
CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
if (totalOut) /* otherwise, it's a skippable frame */
{
crcDecoded = XXH64_digest(&xxh64);
if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
CHECK(crcDecoded != crcOrig, "Decompression corruption");
}
}
}
DISPLAYLEVEL(2, "\rAll tests completed \n");
_end:
LZ4F_freeDecompressionContext(dCtx);
LZ4F_freeCompressionContext(cCtx);
free(srcBuffer);
free(compressedBuffer);
free(decodedBuffer);
for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
free((void *)(sg_in_buf_potential [i].sg_base));
free( sg_out_buf_potential[i].sg_base);
}
if (num_buf_size_distribution_deviations > 0) {
DISPLAYLEVEL(2, "NOTE: %i buffer size deviations \n", (int)num_buf_size_distribution_deviations);
}
if (pause)
{
DISPLAY("press enter to finish \n");
(void)getchar();
}
return testResult;
_output_error:
testResult = 1;
goto _end;
// unreachable
return -1;
#undef CHECK
}
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
(void)startTest; (void)compressibility;
/* allocation */
srcBuffer = malloc (srcBufferSize);
dstBuffer = malloc (dstBufferSize);
cBuffer = malloc (cBufferSize);
CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled");
/* Create initial sample */
FUZ_generateSynthetic(srcBuffer, srcBufferSize, 0.50, &coreSeed);
/* catch up testNb */
for (testNb=0; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for (testNb=startTest; testNb < nbTests; testNb++)
{
size_t sampleSize, sampleStart;
size_t cSize, dSize, dSupSize;
U32 sampleSizeLog;
U64 crcOrig, crcDest;
/* init */
DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1<<sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
/* compression tests*/
cSize = ZSTD_compress(cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize);
CHECK(ZSTD_isError(cSize), "ZSTD_compress failed");
/* decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s)", ZSTD_getErrorName(dSize));
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
}
DISPLAY("\rAll fuzzer tests completed \n");
_cleanup:
free(srcBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
BYTE* bufferP0 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP1 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP15 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP90 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP100 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferDst = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
size_t const bufferDstSize = BUFFERSIZE+64;
unsigned testNb;
size_t const maxTestSizeMask = 0x1FFFF; /* 128 KB - 1 */
U32 rootSeed = seed;
U32 time = FUZ_GetMilliStart();
generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
generate (bufferP1 , BUFFERSIZE, 0.01, &rootSeed);
generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);
memset(bufferDst, 0, BUFFERSIZE);
{ U32 u; for (u=0; u<startTestNb; u++) FUZ_rand (&rootSeed); }
for (testNb=startTestNb; testNb<totalTest; testNb++) {
U32 roundSeed = rootSeed ^ 0xEDA5B371;
FUZ_rand(&rootSeed);
int tag=0;
BYTE* bufferTest = NULL;
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE) {
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
DISPLAYLEVEL (4,"%3i ", tag++);
{ /* determine test sample */
size_t const sizeOrig = (FUZ_rand(&roundSeed) & maxTestSizeMask) + 1;
size_t const offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U32 hashOrig;
if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
else {
switch(FUZ_rand(&roundSeed) & 3)
{
case 0: bufferTest = bufferP0 + offset; break;
case 1: bufferTest = bufferP1 + offset; break;
case 2: bufferTest = bufferP90 + offset; break;
default : bufferTest = bufferP100 + offset; break;
}
}
hashOrig = XXH32 (bufferTest, sizeOrig, 0);
/* compression test */
sizeCompressed = HUF_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
CHECK(HUF_isError(sizeCompressed), "HUF_compress failed");
if (sizeCompressed > 1) { /* don't check uncompressed & rle corner cases */
/* failed compression test */
{ BYTE const saved = bufferVerif[sizeCompressed-1] = 253;
size_t const errorCode = HUF_compress (bufferVerif, sizeCompressed-1, bufferTest, sizeOrig);
CHECK(errorCode!=0, "HUF_compress should have failed (too small destination buffer)")
CHECK(bufferVerif[sizeCompressed-1] != saved, "HUF_compress w/ too small dst : bufferVerif overflow");
}
/* decompression test */
{ BYTE const saved = bufferVerif[sizeOrig] = 253;
size_t const result = HUF_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress : bufferVerif overflow");
CHECK(HUF_isError(result), "HUF_decompress failed : %s", HUF_getErrorName(result));
{ U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig);
CHECK(hashEnd != hashOrig, "HUF_decompress : Decompressed data corrupted");
} }
/* quad decoder test (more fragile) */
/*
if (sizeOrig > 64)
{ BYTE const saved = bufferVerif[sizeOrig] = 253;
size_t const result = HUF_decompress4X6 (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress4X6 : bufferVerif overflow");
CHECK(HUF_isError(result), "HUF_decompress4X6 failed : %s", HUF_getErrorName(result));
{ U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig);
CHECK(hashEnd != hashOrig, "HUF_decompress4X6 : Decompressed data corrupted");
} }
*/
/* truncated src decompression test */
if (sizeCompressed>4) {
/* note : in some rare cases, the truncated bitStream may still generate by chance a valid output of correct size */
size_t const missing = (FUZ_rand(&roundSeed) % (sizeCompressed-3)) + 2; /* no problem, as sizeCompressed > 4 */
size_t const tooSmallSize = sizeCompressed - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, bufferDst, tooSmallSize);
{ size_t const errorCode = HUF_decompress(bufferVerif, sizeOrig, cBufferTooSmall, tooSmallSize);
CHECK(!HUF_isError(errorCode) && (errorCode!=sizeOrig), "HUF_decompress should have failed ! (truncated src buffer)"); }
free(cBufferTooSmall);
} }
} /* Compression / Decompression tests */
/* Attempt decompression on bogus data */
{ size_t const maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
size_t const sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
BYTE const saved = (bufferDst[maxDstSize] = 253);
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
result = HUF_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
CHECK(!HUF_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
CHECK(bufferDst[maxDstSize] != saved, "HUF_decompress noise : bufferDst overflow");
}
} /* for (testNb=startTestNb; testNb<totalTest; testNb++) */
/* exit */
free (bufferP0);
free (bufferP1);
free (bufferP15);
free (bufferP90);
free (bufferP100);
free (bufferDst);
free (bufferVerif);
}
static void FUZ_tests (const U32 startSeed, U32 totalTest, U32 startTestNb)
{
size_t bufferDstSize = BUFFERSIZE*sizeof(U16) + 64;
U16* bufferP8 = (U16*) malloc (bufferDstSize);
void* bufferDst = malloc (bufferDstSize);
U16* bufferVerif = (U16*) malloc (bufferDstSize);
unsigned testNb;
const size_t maxTestSizeMask = 0x1FFFF;
U32 time = FUZ_GetMilliStart();
U32 seed = startSeed;
generateU16 (bufferP8, BUFFERSIZE, 0.08, seed);
if (startTestNb)
{
U32 i;
for (i=0; i<startTestNb; i++)
FUZ_rand (&seed);
}
for (testNb=startTestNb; testNb<totalTest; testNb++)
{
U16* bufferTest;
int tag=0;
U32 roundSeed = seed ^ 0xEDA5B371;
FUZ_rand(&seed);
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
{
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
{
size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U64 hashOrig;
bufferTest = bufferP8 + offset;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
hashOrig = XXH64 (bufferTest, sizeOrig * sizeof(U16), 0);
sizeCompressed = FSE_compressU16 (bufferDst, bufferDstSize, bufferTest, sizeOrig, FSE_MAX_SYMBOL_VALUE, 12);
CHECK(FSE_isError(sizeCompressed), "\r test %5u : FSE_compressU16 failed !", testNb);
if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */
{
U64 hashEnd;
U16 saved = (bufferVerif[sizeOrig] = 1024 + 250);
size_t dstSize;
size_t result;
/* basic decompression test : should work */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
result = FSE_decompressU16 (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);
/* larger output buffer than necessary : should work */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
result = FSE_decompressU16 (bufferVerif, sizeOrig + (FUZ_rand(&roundSeed) & 31) + 1, bufferDst, sizeCompressed);
CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);
/* smaller output buffer than required : should fail */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
dstSize = (FUZ_rand(&roundSeed) & 31) + 1;
if (dstSize >= sizeOrig) dstSize = 1;
dstSize = sizeOrig - dstSize;
saved = (bufferVerif[dstSize] = 1024 + 250);
result = FSE_decompressU16 (bufferVerif, dstSize, bufferDst, sizeCompressed);
CHECK(bufferVerif[dstSize] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
CHECK(!FSE_isError(result), "\r test %5u : FSE_decompressU16 should have failed ! (origSize = %u shorts, dstSize = %u bytes)", testNb, (U32)sizeOrig, (U32)dstSize);
}
}
}
/* clean */
free (bufferP8);
free (bufferDst);
free (bufferVerif);
}
