static void unitTest(void) { U16 testBuffU16[TBSIZE]; size_t errorCode; U32 startSeed=0, testNb=0; /* FSE_countU16 */ { U32 table[FSE_MAX_SYMBOL_VALUE+2]; U32 max, i; for (i=0; i< TBSIZE; i++) testBuffU16[i] = i % (FSE_MAX_SYMBOL_VALUE+1); max = FSE_MAX_SYMBOL_VALUE; errorCode = FSE_countU16(table, testBuffU16, TBSIZE, &max); CHECK(FSE_isError(errorCode), "FSE_countU16() should have worked"); max = FSE_MAX_SYMBOL_VALUE+1; errorCode = FSE_countU16(table, testBuffU16, TBSIZE, &max); CHECK(!FSE_isError(errorCode), "FSE_countU16() should have failed : max too large"); max = FSE_MAX_SYMBOL_VALUE-1; errorCode = FSE_countU16(table, testBuffU16, TBSIZE, &max); CHECK(!FSE_isError(errorCode), "FSE_countU16() should have failed : max too low"); } DISPLAY("Unit tests completed\n"); }
size_t FSE_decompressU16(U16* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) { const BYTE* const istart = (const BYTE*) cSrc; const BYTE* ip = istart; short counting[FSE_MAX_SYMBOL_VALUE+1]; DTable_max_t dt; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; unsigned tableLog; size_t errorCode; /* Sanity check */ if (cSrcSize<2) return ERROR(srcSize_wrong); /* specific corner cases (uncompressed & rle) */ /* normal FSE decoding mode */ errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); if (FSE_isError(errorCode)) return errorCode; ip += errorCode; cSrcSize -= errorCode; unsigned scrambler = 1; errorCode = FSE_buildDTableU16 (dt, counting, maxSymbolValue, tableLog, scrambler); if (FSE_isError(errorCode)) return errorCode; return FSE_decompressU16_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); }
size_t FSE_compressU16(void* dst, size_t maxDstSize, const unsigned short* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) { const U16* const istart = src; const U16* ip = istart; BYTE* const ostart = (BYTE*) dst; BYTE* op = ostart; BYTE* const omax = ostart + maxDstSize; U32 counting[FSE_MAX_SYMBOL_VALUE+1] = {0}; S16 norm[FSE_MAX_SYMBOL_VALUE+1]; CTable_max_t ct; size_t errorCode; /* early out */ if (srcSize <= 1) return srcSize; if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Scan for stats */ errorCode = FSE_countU16 (counting, &maxSymbolValue, ip, srcSize); if (FSE_isError(errorCode)) return errorCode; if (errorCode == srcSize) return 1; /* Input data is one constant element x srcSize times. Use RLE compression. */ /* Normalize */ tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); errorCode = FSE_normalizeCount (norm, tableLog, counting, srcSize, maxSymbolValue); if (FSE_isError(errorCode)) return errorCode; /* Write table description header */ errorCode = FSE_writeNCount (op, omax-op, norm, maxSymbolValue, tableLog); if (FSE_isError(errorCode)) return errorCode; op += errorCode; /* Compress */ errorCode = FSE_buildCTableU16 (ct, norm, maxSymbolValue, tableLog, 1); if (FSE_isError(errorCode)) return errorCode; op += FSE_compressU16_usingCTable (op, omax - op, ip, srcSize, ct); /* check compressibility */ if ( (size_t)(op-ostart) >= (size_t)(srcSize-1)*(sizeof(U16)) ) return 0; /* no compression */ return op-ostart; }
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) { const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; short counting[FSE_MAX_SYMBOL_VALUE+1]; unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; /* normal FSE decoding mode */ size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); if (FSE_isError(NCountLength)) return NCountLength; //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ if (tableLog > maxLog) return ERROR(tableLog_tooLarge); ip += NCountLength; cSrcSize -= NCountLength; CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ }
size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) { const BYTE* const istart = (const BYTE*)cSrc; const BYTE* ip = istart; short counting[FSE_MAX_SYMBOL_VALUE+1]; DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ unsigned tableLog; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ /* normal FSE decoding mode */ { size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); if (FSE_isError(NCountLength)) return NCountLength; if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ ip += NCountLength; cSrcSize -= NCountLength; } CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) ); return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ }
static void unitTest(void) { BYTE* testBuff = (BYTE*)malloc(TBSIZE); BYTE* cBuff = (BYTE*)malloc(FSE_COMPRESSBOUND(TBSIZE)); BYTE* verifBuff = (BYTE*)malloc(TBSIZE); size_t errorCode; U32 seed=0, testNb=0, lseed=0; U32 count[256]; if ((!testBuff) || (!cBuff) || (!verifBuff)) { DISPLAY("Not enough memory, exiting ... \n"); free(testBuff); free(cBuff); free(verifBuff); return; } /* FSE_count */ { U32 max, i; for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&lseed) & 63) + '0'; max = '0' + 63; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); max -= 1; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(!FSE_isError(errorCode), "Error : FSE_count() should have failed : value > max"); max = 65000; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); } /* FSE_optimalTableLog */ { U32 max, i, tableLog=12; size_t testSize = 999; for (i=0; i< testSize; i++) testBuff[i] = (BYTE)FUZ_rand(&lseed); max = 256; FSE_count(count, &max, testBuff, testSize); tableLog = FSE_optimalTableLog(tableLog, testSize, max); CHECK(tableLog<=8, "Too small tableLog"); } /* FSE_normalizeCount */ { S16 norm[256]; U32 max = 256; FSE_count(count, &max, testBuff, TBSIZE); errorCode = FSE_normalizeCount(norm, 10, count, TBSIZE, max); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); errorCode = FSE_normalizeCount(norm, 8, count, TBSIZE, 256); CHECK(!FSE_isError(errorCode), "Error : FSE_normalizeCount() should have failed (max >= 1<<tableLog)"); /* limit corner case : try to make internal rank overflow */ { U32 i; U32 total = 0; count[0] = 940; count[1] = 910; count[2] = 470; count[3] = 190; count[4] = 90; for(i=5; i<=255; i++) count[i] = 6; for (i=0; i<=255; i++) total += count[i]; errorCode = FSE_normalizeCount(norm, 10, count, total, 255); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); count[0] = 300; count[1] = 300; count[2] = 300; count[3] = 300; count[4] = 50; for(i=5; i<=80; i++) count[i] = 4; total = 0; for (i=0; i<=80; i++) total += count[i]; errorCode = FSE_normalizeCount(norm, 10, count, total, 80); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); } } /* FSE_writeNCount, FSE_readNCount */ { S16 norm[129]; BYTE header[513]; U32 max, tableLog, i; size_t headerSize; for (i=0; i< TBSIZE; i++) testBuff[i] = i % 127; max = 128; errorCode = FSE_count(count, &max, testBuff, TBSIZE); CHECK(FSE_isError(errorCode), "Error : FSE_count() should have worked"); tableLog = FSE_optimalTableLog(0, TBSIZE, max); errorCode = FSE_normalizeCount(norm, tableLog, count, TBSIZE, max); CHECK(FSE_isError(errorCode), "Error : FSE_normalizeCount() should have worked"); headerSize = FSE_NCountWriteBound(max, tableLog); headerSize = FSE_writeNCount(header, 513, norm, max, tableLog); CHECK(FSE_isError(headerSize), "Error : FSE_writeNCount() should have worked"); header[headerSize-1] = 0; errorCode = FSE_writeNCount(header, headerSize-1, norm, max, tableLog); CHECK(!FSE_isError(errorCode), "Error : FSE_writeNCount() should have failed"); CHECK (header[headerSize-1] != 0, "Error : FSE_writeNCount() buffer overwrite"); errorCode = FSE_writeNCount(header, headerSize+1, norm, max, tableLog); CHECK(FSE_isError(errorCode), "Error : FSE_writeNCount() should have worked"); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize); CHECK(FSE_isError(errorCode), "Error : FSE_readNCount() should have worked : (error %s)", FSE_getErrorName(errorCode)); max = 64; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (max too small)"); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, header, headerSize-1); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)"); { void* smallBuffer = malloc(headerSize-1); /* outbound read can be caught by valgrind */ CHECK(smallBuffer==NULL, "Error : Not enough memory (FSE_readNCount unit test)"); memcpy(smallBuffer, header, headerSize-1); max = 129; errorCode = FSE_readNCount(norm, &max, &tableLog, smallBuffer, headerSize-1); CHECK(!FSE_isError(errorCode), "Error : FSE_readNCount() should have failed (size too small)"); free(smallBuffer); } } /* FSE_buildCTable_raw & FSE_buildDTable_raw */ { U32 ct[FSE_CTABLE_SIZE_U32(8, 256)]; U32 dt[FSE_DTABLE_SIZE_U32(8)]; U64 crcOrig, crcVerif; size_t cSize, verifSize; U32 i; for (i=0; i< TBSIZE; i++) testBuff[i] = (FUZ_rand(&seed) & 63) + '0'; crcOrig = XXH64(testBuff, TBSIZE, 0); errorCode = FSE_buildCTable_raw(ct, 8); CHECK(FSE_isError(errorCode), "FSE_buildCTable_raw should have worked"); errorCode = FSE_buildDTable_raw(dt, 8); CHECK(FSE_isError(errorCode), "FSE_buildDTable_raw should have worked"); cSize = FSE_compress_usingCTable(cBuff, FSE_COMPRESSBOUND(TBSIZE), testBuff, TBSIZE, ct); CHECK(FSE_isError(cSize), "FSE_compress_usingCTable should have worked using raw CTable"); verifSize = FSE_decompress_usingDTable(verifBuff, TBSIZE, cBuff, cSize, dt); CHECK(FSE_isError(verifSize), "FSE_decompress_usingDTable should have worked using raw DTable"); crcVerif = XXH64(verifBuff, verifSize, 0); CHECK(crcOrig != crcVerif, "Raw regenerated data is corrupted"); } /* known corner case */ { BYTE sample8[8] = { 0, 0, 0, 2, 0, 0, 0, 0 }; BYTE* rBuff; errorCode = FSE_compress(cBuff, TBSIZE, sample8, 8); CHECK(FSE_isError(errorCode), "FSE_compress failed compressing sample8"); rBuff = (BYTE*)malloc(errorCode); /* in order to catch read overflow with Valgrind */ CHECK(rBuff==NULL, "Not enough memory for rBuff"); memcpy(rBuff, cBuff, errorCode); errorCode = FSE_decompress(verifBuff, sizeof(sample8), rBuff, errorCode); CHECK(errorCode != sizeof(sample8), "FSE_decompress failed regenerating sample8"); free(rBuff); } free(testBuff); free(cBuff); free(verifBuff); DISPLAY("Unit tests completed\n"); }
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb) { BYTE* bufferP0 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP1 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP15 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP90 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferP100 = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferDst = (BYTE*) malloc (BUFFERSIZE+64); BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64); size_t bufferDstSize = BUFFERSIZE+64; unsigned testNb, maxSV, tableLog; const size_t maxTestSizeMask = 0x1FFFF; U32 rootSeed = seed; U32 time = FUZ_GetMilliStart(); generateNoise (bufferP0, BUFFERSIZE, &rootSeed); generate (bufferP1 , BUFFERSIZE, 0.01, &rootSeed); generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed); generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed); memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE); if (startTestNb) { U32 i; for (i=0; i<startTestNb; i++) FUZ_rand (&rootSeed); } for (testNb=startTestNb; testNb<totalTest; testNb++) { BYTE* bufferTest; int tag=0; U32 roundSeed = rootSeed ^ 0xEDA5B371; FUZ_rand(&rootSeed); DISPLAYLEVEL (4, "\r test %5u ", testNb); if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE) { DISPLAY ("\r test %5u ", testNb); time = FUZ_GetMilliStart(); } /* Compression / Decompression tests */ { /* determine test sample */ size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1; size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask)); size_t sizeCompressed; U32 hashOrig; if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset; else { switch(FUZ_rand(&roundSeed) & 3) { case 0: bufferTest = bufferP0 + offset; break; case 1: bufferTest = bufferP1 + offset; break; case 2: bufferTest = bufferP90 + offset; break; default : bufferTest = bufferP100 + offset; break; } } DISPLAYLEVEL (4,"%3i ", tag++);; hashOrig = XXH32 (bufferTest, sizeOrig, 0); /* compress test */ sizeCompressed = FSE_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig); CHECK(FSE_isError(sizeCompressed), "Compression failed !"); if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */ { /* failed compression test*/ { size_t errorCode; void* tooSmallDBuffer = malloc(sizeCompressed-1); /* overflows detected with Valgrind */ CHECK(tooSmallDBuffer==NULL, "Not enough memory for tooSmallDBuffer test"); errorCode = FSE_compress (tooSmallDBuffer, sizeCompressed-1, bufferTest, sizeOrig); CHECK(errorCode!=0, "Compression should have failed : destination buffer too small"); free(tooSmallDBuffer); } /* decompression test */ { U32 hashEnd; BYTE saved = (bufferVerif[sizeOrig] = 254); size_t result = FSE_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed); CHECK(bufferVerif[sizeOrig] != saved, "Output buffer overrun (bufferVerif) : write beyond specified end"); CHECK(FSE_isError(result), "Decompression failed"); hashEnd = XXH32 (bufferVerif, sizeOrig, 0); CHECK(hashEnd != hashOrig, "Decompressed data corrupted"); } } } /* Attempt header decoding on bogus data */ { short count[256]; size_t result; DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++); maxSV = 255; result = FSE_readNCount (count, &maxSV, &tableLog, bufferTest, FSE_NCOUNTBOUND); if (!FSE_isError(result)) /* an error would be normal */ { int checkCount; CHECK(result > FSE_NCOUNTBOUND, "FSE_readHeader() reads too far (buffer overflow)"); CHECK(maxSV > 255, "count table overflow (%u)", maxSV+1); checkCount = FUZ_checkCount(count, tableLog, maxSV); CHECK(checkCount==-1, "symbol distribution corrupted"); } } /* Attempt decompression on bogus data */ { size_t maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask; size_t sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask; BYTE saved = (bufferDst[maxDstSize] = 253); size_t result; DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);; result = FSE_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed); CHECK(!FSE_isError(result) && (result > maxDstSize), "Decompression overran output buffer"); CHECK(bufferDst[maxDstSize] != saved, "Output buffer bufferDst corrupted"); } } /* exit */ free (bufferP0); free (bufferP1); free (bufferP15); free (bufferP90); free (bufferP100); free (bufferDst); free (bufferVerif); }
/* Compressed format : MAGICNUMBER - STREAMDESCRIPTOR - ( BLOCKHEADER - COMPRESSEDBLOCK ) - STREAMCRC MAGICNUMBER - 4 bytes - Designates compression algo STREAMDESCRIPTOR - 1 byte bits 0-3 : max block size, 2^value from 0 to 0xA; min 0=>1KB, max 0x6=>64KB, typical 5=>32 KB bits 4-7 = 0 : reserved; BLOCKHEADER - 1-5 bytes 1st byte : bits 6-7 : blockType (compressed, raw, rle, crc (end of Frame) bit 5 : full block ** if not full block ** 2nd & 3rd byte : regenerated size of block (big endian); note : 0 = 64 KB ** if blockType==compressed ** next 2 bytes : compressed size of block COMPRESSEDBLOCK the compressed data itself. STREAMCRC - 3 bytes (including 1-byte blockheader) 22 bits (xxh32() >> 5) checksum of the original data, big endian */ unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename) { U64 filesize = 0; U64 compressedfilesize = 0; char* in_buff; char* out_buff; FILE* finput; FILE* foutput; size_t sizeCheck; size_t inputBlockSize = FIO_blockID_to_blockSize(g_blockSizeId); XXH32_state_t xxhState; typedef size_t (*compressor_t) (void* dst, size_t dstSize, const void* src, size_t srcSize); compressor_t compressor; unsigned magicNumber; /* Init */ XXH32_reset (&xxhState, FSE_CHECKSUM_SEED); get_fileHandle(input_filename, output_filename, &finput, &foutput); switch (g_compressor) { case FIO_fse: compressor = FSE_compress; magicNumber = FIO_magicNumber_fse; break; case FIO_huff0: compressor = HUF_compress; magicNumber = FIO_magicNumber_huff0; break; case FIO_zlibh: compressor = FIO_ZLIBH_compress; magicNumber = FIO_magicNumber_zlibh; break; default : EXM_THROW(20, "unknown compressor selection"); } /* Allocate Memory */ if (inputBlockSize==0) EXM_THROW(0, "impossible problem, to please static analyzer"); in_buff = (char*)malloc(inputBlockSize); out_buff = (char*)malloc(FSE_compressBound(inputBlockSize) + 5); if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); /* Write Frame Header */ FIO_writeLE32(out_buff, magicNumber); out_buff[4] = (char)g_blockSizeId; /* Max Block Size descriptor */ sizeCheck = fwrite(out_buff, 1, FIO_FRAMEHEADERSIZE, foutput); if (sizeCheck!=FIO_FRAMEHEADERSIZE) EXM_THROW(22, "Write error : cannot write header"); compressedfilesize += FIO_FRAMEHEADERSIZE; /* Main compression loop */ while (1) { /* Fill input Buffer */ size_t cSize; size_t inSize = fread(in_buff, (size_t)1, (size_t)inputBlockSize, finput); if (inSize==0) break; filesize += inSize; XXH32_update(&xxhState, in_buff, inSize); DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); /* Compress Block */ cSize = compressor(out_buff + FIO_maxBlockHeaderSize, FSE_compressBound(inputBlockSize), in_buff, inSize); if (FSE_isError(cSize)) EXM_THROW(23, "Compression error : %s ", FSE_getErrorName(cSize)); /* Write cBlock */ switch(cSize) { size_t headerSize; case 0: /* raw */ if (inSize == inputBlockSize) { out_buff[0] = (BYTE)((bt_raw << 6) + BIT5); headerSize = 1; } else { out_buff[2] = (BYTE)inSize; out_buff[1] = (BYTE)(inSize >> 8); out_buff[0] = (BYTE)(bt_raw << 6); headerSize = 3; } sizeCheck = fwrite(out_buff, 1, headerSize, foutput); if (sizeCheck!=headerSize) EXM_THROW(24, "Write error : cannot write block header"); sizeCheck = fwrite(in_buff, 1, inSize, foutput); if (sizeCheck!=(size_t)(inSize)) EXM_THROW(25, "Write error : cannot write block"); compressedfilesize += inSize + headerSize; break; case 1: /* rle */ if (inSize == inputBlockSize) { out_buff[0] = (BYTE)((bt_rle << 6) + BIT5); headerSize = 1; } else { out_buff[2] = (BYTE)inSize; out_buff[1] = (BYTE)(inSize >> 8); out_buff[0] = (BYTE)(bt_raw << 6); headerSize = 3; } out_buff[headerSize] = in_buff[0]; sizeCheck = fwrite(out_buff, 1, headerSize+1, foutput); if (sizeCheck!=(headerSize+1)) EXM_THROW(26, "Write error : cannot write rle block"); compressedfilesize += headerSize + 1; break; default : /* compressed */ if (inSize == inputBlockSize) { out_buff[2] = (BYTE)((bt_compressed << 6) + BIT5); out_buff[3] = (BYTE)(cSize >> 8); out_buff[4] = (BYTE)cSize; headerSize = 3; } else { out_buff[0] = (BYTE)(bt_compressed << 6); out_buff[1] = (BYTE)(inSize >> 8); out_buff[2] = (BYTE)inSize; out_buff[3] = (BYTE)(cSize >> 8); out_buff[4] = (BYTE)cSize; headerSize = FIO_maxBlockHeaderSize; } sizeCheck = fwrite(out_buff+(FIO_maxBlockHeaderSize-headerSize), 1, headerSize+cSize, foutput); if (sizeCheck!=(headerSize+cSize)) EXM_THROW(27, "Write error : cannot write rle block"); compressedfilesize += headerSize + cSize; break; }
static void FUZ_tests (const U32 startSeed, U32 totalTest, U32 startTestNb) { size_t bufferDstSize = BUFFERSIZE*sizeof(U16) + 64; U16* bufferP8 = (U16*) malloc (bufferDstSize); void* bufferDst = malloc (bufferDstSize); U16* bufferVerif = (U16*) malloc (bufferDstSize); unsigned testNb; const size_t maxTestSizeMask = 0x1FFFF; U32 time = FUZ_GetMilliStart(); U32 seed = startSeed; generateU16 (bufferP8, BUFFERSIZE, 0.08, seed); if (startTestNb) { U32 i; for (i=0; i<startTestNb; i++) FUZ_rand (&seed); } for (testNb=startTestNb; testNb<totalTest; testNb++) { U16* bufferTest; int tag=0; U32 roundSeed = seed ^ 0xEDA5B371; FUZ_rand(&seed); DISPLAYLEVEL (4, "\r test %5u ", testNb); if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE) { DISPLAY ("\r test %5u ", testNb); time = FUZ_GetMilliStart(); } /* Compression / Decompression tests */ { size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1; size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask)); size_t sizeCompressed; U64 hashOrig; bufferTest = bufferP8 + offset; DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++); hashOrig = XXH64 (bufferTest, sizeOrig * sizeof(U16), 0); sizeCompressed = FSE_compressU16 (bufferDst, bufferDstSize, bufferTest, sizeOrig, FSE_MAX_SYMBOL_VALUE, 12); CHECK(FSE_isError(sizeCompressed), "\r test %5u : FSE_compressU16 failed !", testNb); if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */ { U64 hashEnd; U16 saved = (bufferVerif[sizeOrig] = 1024 + 250); size_t dstSize; size_t result; /* basic decompression test : should work */ DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++); result = FSE_decompressU16 (bufferVerif, sizeOrig, bufferDst, sizeCompressed); CHECK(bufferVerif[sizeOrig] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb); CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed); hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0); CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb); /* larger output buffer than necessary : should work */ DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++); result = FSE_decompressU16 (bufferVerif, sizeOrig + (FUZ_rand(&roundSeed) & 31) + 1, bufferDst, sizeCompressed); CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed); hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0); CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb); /* smaller output buffer than required : should fail */ DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++); dstSize = (FUZ_rand(&roundSeed) & 31) + 1; if (dstSize >= sizeOrig) dstSize = 1; dstSize = sizeOrig - dstSize; saved = (bufferVerif[dstSize] = 1024 + 250); result = FSE_decompressU16 (bufferVerif, dstSize, bufferDst, sizeCompressed); CHECK(bufferVerif[dstSize] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb); CHECK(!FSE_isError(result), "\r test %5u : FSE_decompressU16 should have failed ! (origSize = %u shorts, dstSize = %u bytes)", testNb, (U32)sizeOrig, (U32)dstSize); } } } /* clean */ free (bufferP8); free (bufferDst); free (bufferVerif); }