static void printVersion(void)
{
DISPLAY(WELCOME_MESSAGE);
/* format support */
DISPLAYLEVEL(3, "*** supports: zstd");
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
DISPLAYLEVEL(3, ", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
#endif
#ifdef ZSTD_GZCOMPRESS
DISPLAYLEVEL(3, ", gzip");
#endif
#ifdef ZSTD_LZ4COMPRESS
DISPLAYLEVEL(3, ", lz4");
#endif
#ifdef ZSTD_LZMACOMPRESS
DISPLAYLEVEL(3, ", lzma, xz ");
#endif
DISPLAYLEVEL(3, "\n");
/* posix support */
#ifdef _POSIX_C_SOURCE
DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
#endif
#ifdef _POSIX_VERSION
DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
#endif
#ifdef PLATFORM_POSIX_VERSION
DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
#endif
}
/**
* parseLegacyParameters() :
* reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
* @return 1 means that legacy dictionary builder parameters were correct
* @return 0 in case of malformed parameters
*/
static unsigned parseLegacyParameters(const char* stringPtr, unsigned* selectivity)
{
if (!longCommandWArg(&stringPtr, "s=") && !longCommandWArg(&stringPtr, "selectivity=")) { return 0; }
*selectivity = readU32FromChar(&stringPtr);
if (stringPtr[0] != 0) return 0;
DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity);
return 1;
}
static void get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
{
if (!strcmp (input_filename, stdinmark))
{
DISPLAYLEVEL(4,"Using stdin for input\n");
*pfinput = stdin;
SET_BINARY_MODE(stdin);
}
else
{
*pfinput = fopen(input_filename, "rb");
}
if (!strcmp (output_filename, stdoutmark))
{
DISPLAYLEVEL(4,"Using stdout for output\n");
*pfoutput = stdout;
SET_BINARY_MODE(stdout);
}
else
{
/* Check if destination file already exists */
*pfoutput=0;
if (strcmp(output_filename,nulmark)) *pfoutput = fopen( output_filename, "rb" );
if (*pfoutput!=0)
{
fclose(*pfoutput);
if (!g_overwrite)
{
char ch;
if (g_displayLevel <= 1) /* No interaction possible */
EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
ch = (char)getchar();
if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
}
}
*pfoutput = fopen( output_filename, "wb" );
}
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
}
static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
{
const BYTE* const b = (const BYTE*)ptr;
size_t u;
for (u=0; u<length; u++) {
BYTE c = b[u];
if (c<32 || c>126) c = '.'; /* non-printable char */
DISPLAYLEVEL(dlevel, "%c", c);
}
}
/**
* parseCoverParameters() :
* reads cover parameters from *stringPtr (e.g. "--train-cover=k=48,d=8,steps=32") into *params
* @return 1 means that cover parameters were correct
* @return 0 in case of malformed parameters
*/
static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params)
{
memset(params, 0, sizeof(*params));
for (; ;) {
if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
return 0;
}
if (stringPtr[0] != 0) return 0;
DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
return 1;
}
/** parseCompressionParameters() :
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
* @return 1 means that compression parameters were correct
* @return 0 in case of malformed parameters
*/
static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
{
for ( ; ;) {
if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
return 0;
}
if (stringPtr[0] != 0) return 0; /* check the end of string */
DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy);
return 1;
}
int main(int argc, const char** argv)
{
unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
double litProba = 0.0;
U64 size = SIZE_DEFAULT;
U32 seed = SEED_DEFAULT;
const char* const programName = argv[0];
int argNb;
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Handle commands. Aggregated commands are allowed */
if (*argument=='-') {
argument++;
while (*argument!=0) {
switch(*argument)
{
case 'h':
return usage(programName);
case 'g':
argument++;
size=0;
while ((*argument>='0') && (*argument<='9'))
size *= 10, size += *argument++ - '0';
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
if (*argument=='B') { argument++; }
break;
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
seed *= 10, seed += *argument++ - '0';
break;
case 'P':
argument++;
probaU32 = 0;
while ((*argument>='0') && (*argument<='9'))
probaU32 *= 10, probaU32 += *argument++ - '0';
if (probaU32>100) probaU32 = 100;
break;
case 'L': /* hidden argument : Literal distribution probability */
argument++;
litProba=0.;
while ((*argument>='0') && (*argument<='9'))
litProba *= 10, litProba += *argument++ - '0';
if (litProba>100.) litProba=100.;
litProba /= 100.;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default:
return usage(programName);
}
} } } /* for(argNb=1; argNb<argc; argNb++) */
DISPLAYLEVEL(4, "Compressible data Generator \n");
if (probaU32!=COMPRESSIBILITY_DEFAULT)
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
RDG_genStdout(size, (double)probaU32/100, litProba, seed);
DISPLAYLEVEL(1, "\n");
return 0;
}
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* compressedBuffer;
size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* decodedBuffer;
size_t decodedBufferSize = CNBufferSize;
U32 randState = seed;
size_t result, cSize, readSize, genSize;
U32 testNb=0;
ZBUFF_CCtx* zc = ZBUFF_createCCtx();
ZBUFF_DCtx* zd = ZBUFF_createDCtx();
/* Create compressible test buffer */
CNBuffer = malloc(CNBufferSize);
compressedBuffer = malloc(compressedBufferSize);
decodedBuffer = malloc(decodedBufferSize);
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd)
{
DISPLAY("Not enough memory, aborting\n");
goto _output_error;
}
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState);
/* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_compressInit(zc, 1);
readSize = CNBufferSize;
genSize = compressedBufferSize;
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(result)) goto _output_error;
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
cSize = genSize;
genSize = compressedBufferSize - cSize;
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
if (result != 0) goto _output_error; /* error, or some data not flushed */
cSize += genSize;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInit(zd);
readSize = cSize;
genSize = CNBufferSize;
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
{
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<CNBufferSize; i++)
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
_end:
ZBUFF_freeCCtx(zc);
ZBUFF_freeDCtx(zd);
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
int main(int argCount, const char** argv)
{
int argNb,
bench=0,
decode=0,
testmode=0,
forceStdout=0,
main_pause=0,
nextEntryIsDictionary=0,
operationResult=0,
dictBuild=0,
nextArgumentIsOutFileName=0,
nextArgumentIsMaxDict=0,
nextArgumentIsDictID=0,
nextArgumentIsFile=0;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0;
const char* programName = argv[0];
const char* outFileName = NULL;
const char* dictFileName = NULL;
char* dynNameSpace = NULL;
unsigned maxDictSize = g_defaultMaxDictSize;
unsigned dictID = 0;
int dictCLevel = g_defaultDictCLevel;
unsigned dictSelect = g_defaultSelectivityLevel;
#ifdef UTIL_HAS_CREATEFILELIST
const char** fileNamesTable = NULL;
char* fileNamesBuf = NULL;
unsigned fileNamesNb;
#endif
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */
(void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
displayOut = stderr;
/* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
{ size_t pos;
for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } }
programName += pos;
}
/* preset behaviors */
if (!strcmp(programName, ZSTD_UNZSTD)) decode=1;
if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
/* command switches */
for(argNb=1; argNb<argCount; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
if (nextArgumentIsFile==0) {
/* long commands (--long-word) */
if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; }
if (!strcmp(argument, "--decompress")) { decode=1; continue; }
if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; }
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
if (!strcmp(argument, "--test")) { testmode=1; decode=1; continue; }
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
/* '-' means stdin/stdout */
if (!strcmp(argument, "-")){
if (!filenameIdx) {
filenameIdx=1, filenameTable[0]=stdinmark;
outFileName=stdoutmark;
displayLevel-=(displayLevel==2);
continue;
} }
/* Decode commands (note : aggregated commands are allowed) */
if (argument[0]=='-') {
argument++;
while (argument[0]!=0) {
#ifndef ZSTD_NOCOMPRESS
/* compression Level */
if ((*argument>='0') && (*argument<='9')) {
cLevel = readU32FromChar(&argument);
dictCLevel = cLevel;
if (dictCLevel > ZSTD_maxCLevel())
CLEAN_RETURN(badusage(programName));
continue;
}
#endif
switch(argument[0])
{
/* Display help */
case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */
case 'H':
case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
/* Decoding */
case 'd': decode=1; argument++; break;
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); argument++; break;
/* Use file content as dictionary */
case 'D': nextEntryIsDictionary = 1; argument++; break;
/* Overwrite */
case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break;
/* Verbose mode */
case 'v': displayLevel++; argument++; break;
/* Quiet mode */
case 'q': displayLevel--; argument++; break;
/* keep source file (default); for gzip/xz compatibility */
case 'k': FIO_setRemoveSrcFile(0); argument++; break;
/* Checksum */
case 'C': argument++; FIO_setChecksumFlag(2); break;
/* test compressed file */
case 't': testmode=1; decode=1; argument++; break;
/* destination file name */
case 'o': nextArgumentIsOutFileName=1; argument++; break;
/* recursive */
case 'r': recursive=1; argument++; break;
#ifndef ZSTD_NOBENCH
/* Benchmark */
case 'b': bench=1; argument++; break;
/* range bench (benchmark only) */
case 'e':
/* compression Level */
argument++;
cLevelLast = readU32FromChar(&argument);
break;
/* Modify Nb Iterations (benchmark only) */
case 'i':
argument++;
{ U32 const iters = readU32FromChar(&argument);
BMK_setNotificationLevel(displayLevel);
BMK_SetNbIterations(iters);
}
break;
/* cut input into blocks (benchmark only) */
case 'B':
argument++;
{ size_t bSize = readU32FromChar(&argument);
if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
if (toupper(*argument)=='M') bSize<<=20, argument++;
if (toupper(*argument)=='B') argument++;
BMK_setNotificationLevel(displayLevel);
BMK_SetBlockSize(bSize);
}
break;
#endif /* ZSTD_NOBENCH */
/* Dictionary Selection level */
case 's':
argument++;
dictSelect = readU32FromChar(&argument);
break;
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
case 'p': argument++;
#ifndef ZSTD_NOBENCH
if ((*argument>='0') && (*argument<='9')) {
BMK_setAdditionalParam(readU32FromChar(&argument));
} else
#endif
main_pause=1;
break;
/* unknown command */
default : CLEAN_RETURN(badusage(programName));
}
}
continue;
} /* if (argument[0]=='-') */
if (nextArgumentIsMaxDict) {
nextArgumentIsMaxDict = 0;
maxDictSize = readU32FromChar(&argument);
if (toupper(*argument)=='K') maxDictSize <<= 10;
if (toupper(*argument)=='M') maxDictSize <<= 20;
continue;
}
if (nextArgumentIsDictID) {
nextArgumentIsDictID = 0;
dictID = readU32FromChar(&argument);
continue;
}
} /* if (nextArgumentIsAFile==0) */
if (nextEntryIsDictionary) {
nextEntryIsDictionary = 0;
dictFileName = argument;
continue;
}
if (nextArgumentIsOutFileName) {
nextArgumentIsOutFileName = 0;
outFileName = argument;
if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
continue;
}
/* add filename to list */
filenameTable[filenameIdx++] = argument;
}
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) {
fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
if (fileNamesTable) {
unsigned i;
for (i=0; i<fileNamesNb; i++) DISPLAYLEVEL(4, "%d %s\n", i, fileNamesTable[i]);
free((void*)filenameTable);
filenameTable = fileNamesTable;
filenameIdx = fileNamesNb;
}
}
#endif
/* Check if benchmark is selected */
if (bench) {
#ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(displayLevel);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
#endif
goto _end;
}
/* Check if dictionary builder is selected */
if (dictBuild) {
#ifndef ZSTD_NODICT
ZDICT_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.compressionLevel = dictCLevel;
dictParams.selectivityLevel = dictSelect;
dictParams.notificationLevel = displayLevel;
dictParams.dictID = dictID;
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
#endif
goto _end;
}
/* No input filename ==> use stdin and stdout */
filenameIdx += !filenameIdx; /*< default input is stdin */
if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark; /*< when input is stdin, default output is stdout */
/* Check if input/output defined as console; trigger an error in this case */
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode))
CLEAN_RETURN(badusage(programName));
/* user-selected output filename, only possible with a single file */
if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) {
DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
CLEAN_RETURN(filenameIdx);
}
/* No warning message in pipe mode (stdin + stdout) or multiple mode */
if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
#ifndef ZSTD_NOCOMPRESS
if (!decode) {
if (filenameIdx==1 && outFileName)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel);
} else
#endif
{ /* decompression */
#ifndef ZSTD_NODECOMPRESS
if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
else
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName);
#else
DISPLAY("Decompression not supported\n");
#endif
}
_end:
if (main_pause) waitEnter();
free(dynNameSpace);
#ifdef UTIL_HAS_CREATEFILELIST
if (fileNamesTable)
UTIL_freeFileList(fileNamesTable, fileNamesBuf);
else
#endif
free((void*)filenameTable);
return operationResult;
}
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
BYTE* bufferP0 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP1 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP15 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP90 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP100 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferDst = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
size_t const bufferDstSize = BUFFERSIZE+64;
unsigned testNb;
size_t const maxTestSizeMask = 0x1FFFF; /* 128 KB - 1 */
U32 rootSeed = seed;
U32 time = FUZ_GetMilliStart();
generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
generate (bufferP1 , BUFFERSIZE, 0.01, &rootSeed);
generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);
memset(bufferDst, 0, BUFFERSIZE);
{ U32 u; for (u=0; u<startTestNb; u++) FUZ_rand (&rootSeed); }
for (testNb=startTestNb; testNb<totalTest; testNb++) {
U32 roundSeed = rootSeed ^ 0xEDA5B371;
FUZ_rand(&rootSeed);
int tag=0;
BYTE* bufferTest = NULL;
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE) {
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
DISPLAYLEVEL (4,"%3i ", tag++);
{ /* determine test sample */
size_t const sizeOrig = (FUZ_rand(&roundSeed) & maxTestSizeMask) + 1;
size_t const offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U32 hashOrig;
if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
else {
switch(FUZ_rand(&roundSeed) & 3)
{
case 0: bufferTest = bufferP0 + offset; break;
case 1: bufferTest = bufferP1 + offset; break;
case 2: bufferTest = bufferP90 + offset; break;
default : bufferTest = bufferP100 + offset; break;
}
}
hashOrig = XXH32 (bufferTest, sizeOrig, 0);
/* compression test */
sizeCompressed = HUF_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
CHECK(HUF_isError(sizeCompressed), "HUF_compress failed");
if (sizeCompressed > 1) { /* don't check uncompressed & rle corner cases */
/* failed compression test */
{ BYTE const saved = bufferVerif[sizeCompressed-1] = 253;
size_t const errorCode = HUF_compress (bufferVerif, sizeCompressed-1, bufferTest, sizeOrig);
CHECK(errorCode!=0, "HUF_compress should have failed (too small destination buffer)")
CHECK(bufferVerif[sizeCompressed-1] != saved, "HUF_compress w/ too small dst : bufferVerif overflow");
}
/* decompression test */
{ BYTE const saved = bufferVerif[sizeOrig] = 253;
size_t const result = HUF_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress : bufferVerif overflow");
CHECK(HUF_isError(result), "HUF_decompress failed : %s", HUF_getErrorName(result));
{ U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig);
CHECK(hashEnd != hashOrig, "HUF_decompress : Decompressed data corrupted");
} }
/* quad decoder test (more fragile) */
/*
if (sizeOrig > 64)
{ BYTE const saved = bufferVerif[sizeOrig] = 253;
size_t const result = HUF_decompress4X6 (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "HUF_decompress4X6 : bufferVerif overflow");
CHECK(HUF_isError(result), "HUF_decompress4X6 failed : %s", HUF_getErrorName(result));
{ U32 const hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
if (hashEnd!=hashOrig) findDifferentByte(bufferVerif, sizeOrig, bufferTest, sizeOrig);
CHECK(hashEnd != hashOrig, "HUF_decompress4X6 : Decompressed data corrupted");
} }
*/
/* truncated src decompression test */
if (sizeCompressed>4) {
/* note : in some rare cases, the truncated bitStream may still generate by chance a valid output of correct size */
size_t const missing = (FUZ_rand(&roundSeed) % (sizeCompressed-3)) + 2; /* no problem, as sizeCompressed > 4 */
size_t const tooSmallSize = sizeCompressed - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, bufferDst, tooSmallSize);
{ size_t const errorCode = HUF_decompress(bufferVerif, sizeOrig, cBufferTooSmall, tooSmallSize);
CHECK(!HUF_isError(errorCode) && (errorCode!=sizeOrig), "HUF_decompress should have failed ! (truncated src buffer)"); }
free(cBufferTooSmall);
} }
} /* Compression / Decompression tests */
/* Attempt decompression on bogus data */
{ size_t const maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
size_t const sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
BYTE const saved = (bufferDst[maxDstSize] = 253);
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
result = HUF_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
CHECK(!HUF_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
CHECK(bufferDst[maxDstSize] != saved, "HUF_decompress noise : bufferDst overflow");
}
} /* for (testNb=startTestNb; testNb<totalTest; testNb++) */
/* exit */
free (bufferP0);
free (bufferP1);
free (bufferP15);
free (bufferP90);
free (bufferP100);
free (bufferDst);
free (bufferVerif);
}
int main(int argc, char** argv)
{
int i,
bench=0,
decode=0,
forceStdout=0,
main_pause=0,
rangeBench = 1;
unsigned fileNameStart = 0;
unsigned nbFiles = 0;
unsigned cLevel = 1;
const char* programName = argv[0];
const char* inFileName = NULL;
const char* outFileName = NULL;
char* dynNameSpace = NULL;
char extension[] = ZSTD_EXTENSION;
displayOut = stderr;
/* Pick out basename component. Don't rely on stdlib because of conflicting behavior. */
for (i = (int)strlen(programName); i > 0; i--)
{
if (programName[i] == '/') { i++; break; }
}
programName += i;
/* zstdcat preset behavior */
if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
/* unzstd preset behavior */
if (!strcmp(programName, ZSTD_UNZSTD))
decode=1;
/* command switches */
for(i=1; i<argc; i++)
{
char* argument = argv[i];
if(!argument) continue; /* Protection if argument empty */
/* long commands (--long-word) */
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--help")) { displayOut=stdout; return usage_advanced(programName); }
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
/* Decode commands (note : aggregated commands are allowed) */
if (argument[0]=='-')
{
/* '-' means stdin/stdout */
if (argument[1]==0)
{
if (!inFileName) inFileName=stdinmark;
else outFileName=stdoutmark;
continue;
}
argument++;
while (argument[0]!=0)
{
/* compression Level */
if ((*argument>='0') && (*argument<='9'))
{
cLevel = 0;
while ((*argument >= '0') && (*argument <= '9'))
{
cLevel *= 10;
cLevel += *argument - '0';
argument++;
}
continue;
}
switch(argument[0])
{
/* Display help */
case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; /* Version Only */
case 'H':
case 'h': displayOut=stdout; return usage_advanced(programName);
/* Compression (default) */
//case 'z': forceCompress = 1; break;
/* Decoding */
case 'd': decode=1; argument++; break;
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel=1; argument++; break;
// Test
//case 't': decode=1; LZ4IO_setOverwrite(1); output_filename=nulmark; break;
/* Overwrite */
case 'f': FIO_overwriteMode(); argument++; break;
/* Verbose mode */
case 'v': displayLevel=4; argument++; break;
/* Quiet mode */
case 'q': displayLevel--; argument++; break;
/* keep source file (default anyway, so useless; only for xz/lzma compatibility) */
case 'k': argument++; break;
/* Benchmark */
case 'b': bench=1; argument++; break;
/* Modify Nb Iterations (benchmark only) */
case 'i':
{
int iters= 0;
argument++;
while ((*argument >='0') && (*argument <='9'))
iters *= 10, iters += *argument++ - '0';
BMK_SetNbIterations(iters);
}
break;
/* cut input into blocks (benchmark only) */
case 'B':
{
size_t bSize = 0;
argument++;
while ((*argument >='0') && (*argument <='9'))
bSize *= 10, bSize += *argument++ - '0';
if (*argument=='K') bSize<<=10, argument++; /* allows using KB notation */
if (*argument=='M') bSize<<=20, argument++;
if (*argument=='B') argument++;
BMK_SetBlockSize(bSize);
}
break;
/* range bench (benchmark only) */
case 'r':
rangeBench = -1;
argument++;
break;
/* Pause at the end (hidden option) */
case 'p': main_pause=1; argument++; break;
/* unknown command */
default : return badusage(programName);
}
}
continue;
}
/* first provided filename is input */
if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argc-i; continue; }
/* second provided filename is output */
if (!outFileName)
{
outFileName = argument;
if (!strcmp (outFileName, nullString)) outFileName = nulmark;
continue;
}
}
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
/* No input filename ==> use stdin */
if(!inFileName) { inFileName=stdinmark; }
/* Check if input defined as console; trigger an error in this case */
if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
/* Check if benchmark is selected */
if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, cLevel*rangeBench); goto _end; }
/* No output filename ==> try to select one automatically (when possible) */
while (!outFileName)
{
if (!IS_CONSOLE(stdout)) { outFileName=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */
if (!decode) /* compression to file */
{
size_t l = strlen(inFileName);
dynNameSpace = (char*)calloc(1,l+5);
if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); }
strcpy(dynNameSpace, inFileName);
strcpy(dynNameSpace+l, ZSTD_EXTENSION);
outFileName = dynNameSpace;
DISPLAYLEVEL(2, "Compressed filename will be : %s \n", outFileName);
break;
}
/* decompression to file (automatic name will work only if input filename has correct format extension) */
{
size_t outl;
size_t inl = strlen(inFileName);
dynNameSpace = (char*)calloc(1,inl+1);
if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); }
outFileName = dynNameSpace;
strcpy(dynNameSpace, inFileName);
outl = inl;
if (inl>4)
while ((outl >= inl-4) && (inFileName[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0;
if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); return badusage(programName); }
DISPLAYLEVEL(2, "Decoding file %s \n", outFileName);
}
}
/* Check if output is defined as console; trigger an error in this case */
if (!strcmp(outFileName,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
/* No warning message in pure pipe mode (stdin + stdout) */
if (!strcmp(inFileName, stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
if (decode)
FIO_decompressFilename(outFileName, inFileName);
else
FIO_compressFilename(outFileName, inFileName, cLevel);
_end:
if (main_pause) waitEnter();
free(dynNameSpace);
return 0;
}
static int badusage(const char* exeName)
{
DISPLAYLEVEL(1, "Incorrect parameters\n");
if (displayLevel >= 1) usage(exeName);
exit(1);
}
int basicTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
int cSize, testSize;
LZ4F_preferences_t prefs;
LZ4F_compressionContext_t cctx = NULL;
U64 crcOrig;
LZ4SG_in_t sg_cin [MAX_SG_BUFFERS];
LZ4SG_out_t sg_cout[MAX_SG_BUFFERS];
LZ4SG_in_t sg_din [MAX_SG_BUFFERS];
LZ4SG_out_t sg_dout[MAX_SG_BUFFERS];
size_t sg_in_len, sg_out_len;
size_t maxDstSize = LZ4_SG_compressBound(COMPRESSIBLE_NOISE_LENGTH, NELEMS(sg_cin), NELEMS(sg_cout));
size_t sourceSizeOut;
/* Create compressible test buffer */
memset(&prefs, 0, sizeof(prefs));
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(maxDstSize);
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
/* Trivial tests : one input and one output buffers */
testSize = COMPRESSIBLE_NOISE_LENGTH;
DISPLAYLEVEL(3, "One input and one output buffers : \n");
sg_cin[0].sg_base = CNBuffer;
sg_cin[0].sg_len = COMPRESSIBLE_NOISE_LENGTH;
sg_in_len = 1;
sg_cout[0].sg_base = compressedBuffer;
sg_cout[0].sg_len = maxDstSize;
sg_out_len = 1;
sourceSizeOut = testSize;
cSize = LZ4_SG_compress(&sg_cin[0], sg_in_len, &sg_cout[0], sg_out_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
UT_VERIFY(cSize > 0, goto _output_error);
DISPLAYLEVEL(3, "Decompress test various valid sg_out and maxDstSize combinations \n");
{
U64 crcDest;
// sg_out.sg_len == maxDstSize == originalSize
sg_din[0].sg_base = compressedBuffer;
sg_din[0].sg_len = cSize;
sg_dout[0].sg_base = decodedBuffer;
sg_dout[0].sg_len = COMPRESSIBLE_NOISE_LENGTH;
sourceSizeOut = cSize;
maxDstSize = testSize;
memset(decodedBuffer, 0, testSize);
testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
UT_VERIFY(testResult > 0 , goto _output_error);
UT_VERIFY(testResult == testSize, goto _output_error);
UT_VERIFY( crcDest == crcOrig , goto _output_error);
// maxDstSize > originalSize
maxDstSize = COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH;
sourceSizeOut = cSize;
memset(decodedBuffer, 0, testSize);
testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
UT_VERIFY(testResult > 0 , goto _output_error);
UT_VERIFY(testResult == testSize, goto _output_error);
UT_VERIFY( crcDest == crcOrig , goto _output_error);
// sg_out.sg_len > originalSize
maxDstSize = testSize;
sg_dout[0].sg_len = COMPRESSIBLE_NOISE_LENGTH + DECODE_GUARD_LENGTH;
sourceSizeOut = cSize;
memset(decodedBuffer, 0, testSize);
testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
UT_VERIFY(testResult > 0 , goto _output_error);
UT_VERIFY(testResult == testSize, goto _output_error);
UT_VERIFY( crcDest == crcOrig , goto _output_error);
}
DISPLAYLEVEL(3, "Frame Decompression test (%08x): \n", (unsigned int)crcOrig);
{
int lz4f_result = verify_basic_LZ4F_decompression(compressedBuffer, cSize, crcOrig, decodedBuffer, testSize);
UT_VERIFY(0 == lz4f_result, goto _output_error);
}
// basic SGL test
{
// prepare SGL input
const size_t num_data_buffers = 16;
const size_t buf_size_bytes = 4 KB;
unsigned int i;
for (i = 0; i < 1+num_data_buffers; i++) {
sg_cin [i].sg_base = malloc(buf_size_bytes);
sg_cin [i].sg_len = buf_size_bytes;
sg_cout[i].sg_base = malloc(buf_size_bytes);
sg_cout[i].sg_len = buf_size_bytes;
sg_din [i].sg_base = malloc(buf_size_bytes);
sg_din [i].sg_len = buf_size_bytes;
sg_dout[i].sg_base = malloc(buf_size_bytes);
sg_dout[i].sg_len = buf_size_bytes;
}
for (i = 0; i < num_data_buffers; i++) {
memcpy((void *)sg_cin [i].sg_base, ((const char *)CNBuffer)+(i*buf_size_bytes), buf_size_bytes);
}
sg_in_len = num_data_buffers;
sg_out_len = 1+num_data_buffers;
testSize = num_data_buffers * buf_size_bytes;
maxDstSize = (1+num_data_buffers) * buf_size_bytes;
crcOrig = XXH64(CNBuffer, testSize, 1);
DISPLAYLEVEL(3, "Compress 16 4KB buffers into 17 4KB buffers : \n");
sourceSizeOut = testSize;
cSize = LZ4_SG_compress(&sg_cin[0], sg_in_len, &sg_cout[0], sg_out_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
UT_VERIFY(cSize > 0, goto _output_error);
DISPLAYLEVEL(3, "Decompress 17 4KB buffers into 16 4KB buffers : \n");
for (i = 0; i < 1+num_data_buffers; i++) {
memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, buf_size_bytes);
}
sourceSizeOut = cSize;
maxDstSize = testSize;
sg_in_len = 1+num_data_buffers;
sg_out_len = num_data_buffers;
memset(decodedBuffer, 0, testSize);
testResult = LZ4_SG_decompress(&sg_din[0], sg_in_len, &sg_dout[0], sg_out_len, &sourceSizeOut, maxDstSize);
DISPLAYLEVEL(3, "Decompressed %i bytes (out of %i bytes) to a %i bytes (out of %i bytes) dst_limit %i bytes\n", (int)sourceSizeOut, (int)cSize, (int)testResult, (int)testSize, (int)maxDstSize);
for (i = 0; i < num_data_buffers; i++) {
memcpy(((char *)decodedBuffer)+(i*buf_size_bytes), sg_dout[i].sg_base, buf_size_bytes);
}
UT_VERIFY(testResult > 0 , goto _output_error);
UT_VERIFY(testResult == testSize, goto _output_error);
U64 crcDest;
crcDest = XXH64(decodedBuffer, testSize, 1);
UT_VERIFY( crcDest == crcOrig , goto _output_error);
DISPLAYLEVEL(3, "verify frame decompress on concatenated buffer: \n");
for (i = 0; i < 1+num_data_buffers; i++) {
memcpy(((char *)compressedBuffer)+(i*buf_size_bytes), sg_cout[i].sg_base, buf_size_bytes);
}
int lz4f_result = verify_basic_LZ4F_decompression(compressedBuffer, cSize, crcOrig, decodedBuffer, testSize);
UT_VERIFY(0 == lz4f_result, goto _output_error);
// release SGL
for (i = 0; i < 1+num_data_buffers; i++) {
free((void *)sg_cin [i].sg_base); sg_cin [i].sg_base = NULL; sg_cin [i].sg_len = 0;
free(sg_cout[i].sg_base); sg_cout[i].sg_base = NULL; sg_cout[i].sg_len = 0;
free((void *)sg_din [i].sg_base); sg_din [i].sg_base = NULL; sg_din [i].sg_len = 0;
free(sg_dout[i].sg_base); sg_dout[i].sg_base = NULL; sg_dout[i].sg_len = 0;
}
}
DISPLAY("Basic tests completed \n");
testResult = 0;
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
LZ4F_freeCompressionContext(cctx); cctx = NULL;
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected ! \n");
locateBuffDiff(CNBuffer, decodedBuffer, testSize);
goto _end;
// unreachable
return -1;
}
static int verify_basic_LZ4F_decompression(const void* compressedBuffer, int cSize, U64 crcOrig, void* decodedBuffer, const size_t decodedBufferSize)
{
DISPLAYLEVEL(3, "%s (cSize %i, crcOrig %08x, decodedBufferSize %i)\n", __FUNCTION__, cSize, (unsigned int)crcOrig, (int)decodedBufferSize);
LZ4F_decompressionContext_t dCtx = NULL;
U64 crcDest;
size_t compressedBufferSize = cSize;
BYTE* op = (BYTE*) decodedBuffer;
BYTE* const oend = (BYTE*) decodedBuffer + decodedBufferSize;
const BYTE* ip = (const BYTE*) compressedBuffer;
const BYTE* const iend = (const BYTE*) compressedBuffer + cSize;
LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__);
memset(decodedBuffer, 0, decodedBufferSize);
DISPLAYLEVEL(3, "Single Block : \n");
size_t destSize = decodedBufferSize;
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &destSize, compressedBuffer, &compressedBufferSize, NULL);
UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__);
crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
DISPLAYLEVEL(3, "Regenerated %i bytes (%08x)\n", (int )decodedBufferSize, (unsigned int)crcDest);
UT_VERIFY(crcDest == crcOrig, return __LINE__);
memset(decodedBuffer, 0, decodedBufferSize);
DISPLAYLEVEL(4, "Reusing decompression context \n");
{
size_t iSize = compressedBufferSize - 4;
const BYTE* cBuff = (const BYTE*) compressedBuffer;
DISPLAYLEVEL(3, "Missing last 4 bytes : ");
destSize = decodedBufferSize;
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &destSize, cBuff, &iSize, NULL);
UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__);
UT_VERIFY(errorCode, return __LINE__);
crcDest = XXH64(decodedBuffer, destSize, 1);
DISPLAYLEVEL(3, "crcDest (%08x)\n", (unsigned int)crcDest);
DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned )errorCode);
cBuff += iSize;
iSize = errorCode;
errorCode = LZ4F_decompress(dCtx, decodedBuffer, &destSize, cBuff, &iSize, NULL);
UT_VERIFY(errorCode == 0, return __LINE__);
crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
DISPLAYLEVEL(3, "crcDest (%08x)\n", (unsigned int)crcDest);
UT_VERIFY(crcDest == crcOrig, return __LINE__);
}
{
size_t oSize = 0;
size_t iSize = 0;
LZ4F_frameInfo_t fi;
DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__);
DISPLAYLEVEL(3, " %u \n", (unsigned )errorCode);
DISPLAYLEVEL(3, "get FrameInfo on null input : ");
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
UT_VERIFY(errorCode == (size_t) -LZ4F_ERROR_frameHeader_incomplete, return __LINE__);
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
iSize = 6;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
UT_VERIFY(errorCode == (size_t) -LZ4F_ERROR_frameHeader_incomplete, return __LINE__);
DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
ip += iSize;
DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
iSize = 15 - iSize;
errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__);
DISPLAYLEVEL(3, " correctly decoded \n");
ip += iSize;
}
DISPLAYLEVEL(3, "Byte after byte : \n");
while (ip < iend) {
size_t oSize = oend - op;
size_t iSize = 1;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__);
op += oSize;
ip += iSize;
}
crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
UT_VERIFY(crcDest == crcOrig, return __LINE__);
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned )(op - (BYTE* )decodedBuffer), (unsigned )decodedBufferSize);
errorCode = LZ4F_freeDecompressionContext(dCtx);
UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__);
dCtx = NULL;
return 0;
}
int main(int argc, char** argv)
{
U32 seed=0;
int seedset=0;
int argNb;
int nbTests = nbTestsDefault;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 duration=0;
/* Check command line */
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Decode command (note : aggregated commands are allowed) */
if (argument[0]=='-')
{
if (!strcmp(argument, "--no-prompt"))
{
no_prompt=1;
seedset=1;
displayLevel=1;
continue;
}
argument++;
while (*argument!=0)
{
switch(*argument)
{
case 'h':
return FUZ_usage();
case 'v':
argument++;
displayLevel=4;
break;
case 'q':
argument++;
displayLevel--;
break;
case 'p': /* pause at the end */
argument++;
pause = 1;
break;
case 'i':
argument++;
nbTests=0; duration=0;
while ((*argument>='0') && (*argument<='9'))
{
nbTests *= 10;
nbTests += *argument - '0';
argument++;
}
break;
case 'T':
argument++;
nbTests = 0; duration = 0;
for (;;)
{
switch(*argument)
{
case 'm': duration *= 60; argument++; continue;
case 's':
case 'n': argument++; continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': duration *= 10; duration += *argument++ - '0'; continue;
}
break;
}
break;
case 's':
argument++;
seed=0;
seedset=1;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9'))
{
testNb *= 10;
testNb += *argument - '0';
argument++;
}
break;
case 'P': /* compressibility % */
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
default:
;
return FUZ_usage();
}
}
}
}
/* Get Seed */
printf("Starting lz4sg tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
printf("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
DISPLAYLEVEL(1, "Seed = %u Compressibility = %i%%\n", seed, proba);
if (nbTests<=0) nbTests=1;
if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
if (result) return 1;
return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
}
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
size_t result, cSize;
U32 testNb=0;
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_generateSynthetic(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
// Basic tests
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH);
if (ZSTD_isError(result)) goto _output_error;
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
{
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++)
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongSrcSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
int main(int argc, const char** argv)
{
int i,
cLevel=1,
cLevelLast=-10000,
legacy_format=0,
forceStdout=0,
main_pause=0,
multiple_inputs=0,
all_arguments_are_files=0,
operationResult=0;
operationMode_e mode = om_auto;
const char* input_filename = NULL;
const char* output_filename= NULL;
const char* dictionary_filename = NULL;
char* dynNameSpace = NULL;
const char** inFileNames = (const char**) calloc(argc, sizeof(char*));
unsigned ifnIdx=0;
const char nullOutput[] = NULL_OUTPUT;
const char extension[] = LZ4_EXTENSION;
size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
const char* const exeName = lastNameFromPath(argv[0]);
#ifdef UTIL_HAS_CREATEFILELIST
const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
unsigned fileNamesNb, recursive=0;
#endif
/* Init */
if (inFileNames==NULL) {
DISPLAY("Allocation error : not enough memory \n");
return 1;
}
inFileNames[0] = stdinmark;
LZ4IO_setOverwrite(0);
/* predefined behaviors, based on binary/link name */
if (exeNameMatch(exeName, LZ4CAT)) {
mode = om_decompress;
LZ4IO_setOverwrite(1);
LZ4IO_setRemoveSrcFile(0);
forceStdout=1;
output_filename=stdoutmark;
displayLevel=1;
multiple_inputs=1;
}
if (exeNameMatch(exeName, UNLZ4)) { mode = om_decompress; }
if (exeNameMatch(exeName, LZ4_LEGACY)) { g_lz4c_legacy_commands=1; }
/* command switches */
for(i=1; i<argc; i++) {
const char* argument = argv[i];
if(!argument) continue; /* Protection if argument empty */
/* Short commands (note : aggregated short commands are allowed) */
if (!all_arguments_are_files && argument[0]=='-') {
/* '-' means stdin/stdout */
if (argument[1]==0) {
if (!input_filename) input_filename=stdinmark;
else output_filename=stdoutmark;
continue;
}
/* long commands (--long-word) */
if (argument[1]=='-') {
if (!strcmp(argument, "--")) { all_arguments_are_files = 1; continue; }
if (!strcmp(argument, "--compress")) { mode = om_compress; continue; }
if ((!strcmp(argument, "--decompress"))
|| (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; }
if (!strcmp(argument, "--test")) { mode = om_test; continue; }
if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; }
if ((!strcmp(argument, "--stdout"))
|| (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; }
if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; }
if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; }
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(1); continue; }
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; }
if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */
if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; }
if (longCommandWArg(&argument, "--fast")) {
/* Parse optional acceleration factor */
if (*argument == '=') {
U32 fastLevel;
++argument;
fastLevel = readU32FromChar(&argument);
if (fastLevel) {
cLevel = -(int)fastLevel;
} else {
badusage(exeName);
}
} else if (*argument != 0) {
/* Invalid character following --fast */
badusage(exeName);
} else {
cLevel = -1; /* default for --fast */
}
continue;
}
}
while (argument[1]!=0) {
argument ++;
if (g_lz4c_legacy_commands) {
/* Legacy commands (-c0, -c1, -hc, -y) */
if (!strcmp(argument, "c0")) { cLevel=0; argument++; continue; } /* -c0 (fast compression) */
if (!strcmp(argument, "c1")) { cLevel=9; argument++; continue; } /* -c1 (high compression) */
if (!strcmp(argument, "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */
if (!strcmp(argument, "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */
if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
}
if ((*argument>='0') && (*argument<='9')) {
cLevel = readU32FromChar(&argument);
argument--;
continue;
}
switch(argument[0])
{
/* Display help */
case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup; /* Version */
case 'h': usage_advanced(exeName); goto _cleanup;
case 'H': usage_longhelp(exeName); goto _cleanup;
case 'e':
argument++;
cLevelLast = readU32FromChar(&argument);
argument--;
break;
/* Compression (default) */
case 'z': mode = om_compress; break;
case 'D':
if (argument[1] == '\0') {
/* path is next arg */
if (i + 1 == argc) {
/* there is no next arg */
badusage(exeName);
}
dictionary_filename = argv[++i];
} else {
/* path follows immediately */
dictionary_filename = argument + 1;
}
/* skip to end of argument so that we jump to parsing next argument */
argument += strlen(argument) - 1;
break;
/* Use Legacy format (ex : Linux kernel compression) */
case 'l': legacy_format = 1; blockSize = 8 MB; break;
/* Decoding */
case 'd': mode = om_decompress; break;
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; output_filename=stdoutmark; break;
/* Test integrity */
case 't': mode = om_test; break;
/* Overwrite */
case 'f': LZ4IO_setOverwrite(1); break;
/* Verbose mode */
case 'v': displayLevel++; break;
/* Quiet mode */
case 'q': if (displayLevel) displayLevel--; break;
/* keep source file (default anyway, so useless) (for xz/lzma compatibility) */
case 'k': LZ4IO_setRemoveSrcFile(0); break;
/* Modify Block Properties */
case 'B':
while (argument[1]!=0) {
int exitBlockProperties=0;
switch(argument[1])
{
case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break;
case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */
default :
if (argument[1] < '0' || argument[1] > '9') {
exitBlockProperties=1;
break;
} else {
unsigned B;
argument++;
B = readU32FromChar(&argument);
argument--;
if (B < 4) badusage(exeName);
if (B <= 7) {
blockSize = LZ4IO_setBlockSizeID(B);
BMK_setBlockSize(blockSize);
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
} else {
if (B < 32) badusage(exeName);
BMK_setBlockSize(B);
if (B >= 1024) {
DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10));
} else {
DISPLAYLEVEL(2, "bench: using blocks of size %u bytes \n", (U32)(B));
}
}
break;
}
}
if (exitBlockProperties) break;
}
break;
/* Benchmark */
case 'b': mode = om_bench; multiple_inputs=1;
break;
/* hidden command : benchmark files, but do not fuse result */
case 'S': BMK_setBenchSeparately(1);
break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
case 'r': recursive=1;
#endif
/* fall-through */
/* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */
case 'm': multiple_inputs=1;
break;
/* Modify Nb Seconds (benchmark only) */
case 'i':
{ unsigned iters;
argument++;
iters = readU32FromChar(&argument);
argument--;
BMK_setNotificationLevel(displayLevel);
BMK_setNbSeconds(iters); /* notification if displayLevel >= 3 */
}
break;
/* Pause at the end (hidden option) */
case 'p': main_pause=1; break;
/* Unrecognised command */
default : badusage(exeName);
}
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration)
{
unsigned testResult = 0;
unsigned testNb = 0;
void* srcBuffer = NULL;
void* compressedBuffer = NULL;
void* decodedBuffer = NULL;
U32 coreRand = seed;
LZ4F_decompressionContext_t dCtx = NULL;
LZ4F_compressionContext_t cCtx = NULL;
size_t result;
const U32 startTime = FUZ_GetMilliStart();
XXH64_state_t xxh64;
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
// backup all allocated addresses, from which we will later select buffers
const size_t max_buf_size = 131 KB;
size_t num_buf_size_distribution_deviations = 0;
LZ4SG_in_t sg_in_buf_potential [2*MAX_SG_BUFFERS];
LZ4SG_out_t sg_out_buf_potential[2*MAX_SG_BUFFERS];
LZ4SG_in_t sg_cin [MAX_SG_BUFFERS];
LZ4SG_out_t sg_cout[MAX_SG_BUFFERS];
LZ4SG_in_t sg_din [MAX_SG_BUFFERS];
LZ4SG_out_t sg_dout[MAX_SG_BUFFERS];
size_t sg_cin_len, sg_cout_len, sg_din_len, sg_dout_len;
const size_t maxDstSize = LZ4_SG_compressBound(srcDataLength, NELEMS(sg_cin), NELEMS(sg_cout));
unsigned int i;
for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
sg_in_buf_potential [i].sg_base = malloc(max_buf_size);
sg_in_buf_potential [i].sg_len = max_buf_size;
sg_out_buf_potential[i].sg_base = malloc(max_buf_size);
sg_out_buf_potential[i].sg_len = max_buf_size;
}
/* Init */
duration *= 1000;
/* Create buffers */
result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
srcBuffer = malloc(srcDataLength);
CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
const size_t compressedBufferLength = maxDstSize;
compressedBuffer = malloc(compressedBufferLength);
CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
/* jump to requested testNb */
for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer
/* main fuzzer test loop */
for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++)
{
U32 randState = coreRand ^ prime1;
(void)FUZ_rand(&coreRand); /* update seed */
srand48(FUZ_rand(&randState));
DISPLAYUPDATE(2, "\r%5u ", testNb);
const size_t max_src_buf_size = (4 MB > srcDataLength) ? srcDataLength : 4 MB;
unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(max_src_buf_size-1) - 1)) + 1;
const size_t min_src_size = 20;
const size_t min_first_dest_buf_size = 21;
const size_t min_src_buf_size = 1;
const size_t min_dst_buf_size = 10;
size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + min_src_size;
size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
size_t cSize;
size_t dstSize;
size_t dstSizeBound;
U64 crcOrig, crcDecoded;
unsigned int test_selection = FUZ_rand(&randState);
//TODO: enable lz4f_compress_compatibility_test with LZ4_SG_decompress
int lz4f_compress_compatibility_test = 0;//(test_selection % 4) == 0;
if (!lz4f_compress_compatibility_test)
{
// SGL compress
unsigned int buffer_selection = FUZ_rand(&randState);
if ((buffer_selection & 0xF) == 1)
{
// SG compress single source and single target buffers
sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
sg_cin[0].sg_len = srcSize;
sg_cin_len = 1;
sg_cout[0].sg_base = compressedBuffer;
sg_cout[0].sg_len = compressedBufferLength;
sg_cout_len = 1;
dstSizeBound = dstSize = compressedBufferLength;
}
else
{
// SG compress random number and size source and target buffers
sg_cin_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);
sg_cout_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS);
// single source buffer
if (1 == sg_cin_len) {
sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart;
sg_cin[0].sg_len = srcSize;
DISPLAYUPDATE(4, "INFO: single source buf size %i\n", (int)srcSize);
}
else {
// multiple source buffers
if (srcSize > sg_cin_len*max_buf_size/2) {
srcSize = sg_cin_len*max_buf_size/2;
num_buf_size_distribution_deviations++;
DISPLAYUPDATE(4, "NOTE: source buffer total size deviation %i\n", (int)num_buf_size_distribution_deviations);
}
size_t exact_src_size = 0;
unsigned int buf_size_mean = srcSize / sg_cin_len;
for (i = 0; i < sg_cin_len; i++) {
size_t buf_size = rnd_exponential(buf_size_mean, min_src_buf_size, max_buf_size);
DISPLAYUPDATE(4, "INFO: source buf %i size %i\n", i, (int)buf_size);
if (srcStart+exact_src_size+buf_size > srcDataLength) {
buf_size = srcDataLength-(srcStart+exact_src_size);
}
sg_cin[i].sg_base = sg_in_buf_potential[i*2+1].sg_base;
sg_cin[i].sg_len = buf_size;
memcpy((void *)sg_cin[i].sg_base, (BYTE*)srcBuffer+srcStart+exact_src_size, buf_size);
exact_src_size += buf_size;
if (srcStart+exact_src_size == srcDataLength) {
num_buf_size_distribution_deviations++;
sg_cin_len = i+1;
DISPLAYUPDATE(4, "NOTE: final source buffer size deviation %i (buffers number limited to %i)\n", (int)num_buf_size_distribution_deviations, (int)sg_cin_len);
}
}
srcSize = exact_src_size;
}
// we can now derive the required limit for output
dstSizeBound = LZ4_SG_compressBound(srcSize, sg_cin_len, sg_cout_len);
// single target buffer
if (1 == sg_cout_len) {
sg_cout[0].sg_base = compressedBuffer;
sg_cout[0].sg_len = compressedBufferLength;
}
else {
// multiple target buffers
int finalBufferTruncated = 0;
dstSize = 0;
unsigned int buf_size_mean = dstSizeBound / sg_cout_len;
for (i = 0; i < sg_cout_len; i++) {
const size_t min_buf_size = (i == 0) ? min_first_dest_buf_size : min_dst_buf_size;
size_t buf_size = rnd_exponential(buf_size_mean, min_buf_size, max_buf_size);
DISPLAYUPDATE(4, "INFO: target buf %i size %i\n", (int)i, (int)buf_size);
if (dstSize+buf_size > dstSizeBound) {
buf_size = dstSizeBound-dstSize;
finalBufferTruncated = 1;
}
dstSize += buf_size;
sg_cout[i].sg_base = sg_out_buf_potential[i*2+1].sg_base;
sg_cout[i].sg_len = buf_size;
if (finalBufferTruncated) {
num_buf_size_distribution_deviations++;
if (buf_size < min_buf_size) {
// merge truncated with previous?
if (i > 0) {
sg_cout[i-1].sg_len += buf_size;
if (sg_cout[i-1].sg_len > max_buf_size) {
// skip, too much hassle
DISPLAYUPDATE(4, "NOTE: unable to truncate final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
sg_cout_len = 0; break;
}
}
else {
// can this happen?
DISPLAYUPDATE(4, "NOTE: unable to truncate first and final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations);
sg_cout_len = 0; break;
}
sg_cout_len = i;
}
else {
sg_cout_len = i+1;
}
DISPLAYUPDATE(4, "NOTE: final target buffer size truncated (%i), buffers number limited to %i, final's size is now %i (deviations %i)\n",
(int)buf_size, (int)sg_cout_len, (int)sg_cout[sg_cout_len-1].sg_len, (int)num_buf_size_distribution_deviations);
}
}
// skip/abort condition
if (0 == sg_cout_len) continue;
}
if ((buffer_selection & 0xF) == 0) {
//TODO: select a random input and output buffer and split it in two,
// feeding consecutive addresses as consecutive entries in SGL
}
}
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
size_t sourceSizeOut = srcSize;
result = LZ4_SG_compress(&sg_cin[0], sg_cin_len, &sg_cout[0], sg_cout_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL);
if (((result == 0) || (sourceSizeOut != srcSize)) && (dstSize < dstSizeBound)) {
// forgive compression failure when output total size is lower than bound
num_buf_size_distribution_deviations++;
DISPLAYUPDATE(4, "NOTE: dstSize %i < %i dstSizeBound, compression attempt failed, not totally unexpected (deviations %i), skipping\n",
(int)dstSize, (int)dstSizeBound, (int)num_buf_size_distribution_deviations);
continue;
}
CHECK(result <= 0, "Compression failed (error %i)", (int)result);
CHECK(sourceSizeOut != srcSize, "Compression stopped at %i out of %i", (int)sourceSizeOut, (int)srcSize);
cSize = result;
}
else
{
// LZ4F compression - use it in order to verify SGL decompress compatibility with it
DISPLAYUPDATE(4, "INFO: LZ4F compression\n");
// alternative
// size_t dstMaxSize = LZ4F_compressFrameBound(srcSize, prefsPtr);
// DISPLAYLEVEL(3, "compressFrame srcSize %zu dstMaxSize %zu\n",
// srcSize, dstMaxSize);
// cSize = LZ4F_compressFrame(compressedBuffer, dstMaxSize, (char*)srcBuffer + srcStart, srcSize, prefsPtr);
// CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
unsigned BSId = 4 + (FUZ_rand(&randState) & 3);
unsigned BMId = FUZ_rand(&randState) & 1;
unsigned CCflag = FUZ_rand(&randState) & 1;
unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
LZ4F_preferences_t prefs;
LZ4F_compressOptions_t cOptions;
LZ4F_preferences_t* prefsPtr = &prefs;
memset(&prefs, 0, sizeof(prefs));
memset(&cOptions, 0, sizeof(cOptions));
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag;
prefs.frameInfo.contentSize = frameContentSize;
prefs.autoFlush = autoflush;
prefs.compressionLevel = FUZ_rand(&randState) % 5;
if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
const BYTE* ip = (const BYTE*)srcBuffer + srcStart;
const BYTE* const iend = ip + srcSize;
BYTE* op = (BYTE*)compressedBuffer;
BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL);
unsigned maxBits = FUZ_highbit((U32)srcSize);
result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result);
op += result;
while (ip < iend)
{
unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
size_t oSize = LZ4F_compressBound(iSize, prefsPtr);
unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1);
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
DISPLAYLEVEL(3, "compressUpdate ip %d iSize %zu oSize %zu forceFlush %d\n",
(int)(ip-((const BYTE*)srcBuffer + srcStart)), iSize, oSize, forceFlush);
result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
op += result;
ip += iSize;
if (forceFlush)
{
result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
op += result;
}
}
result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result);
op += result;
cSize = op-(BYTE*)compressedBuffer;
}
//DECOMPRESS
test_selection = FUZ_rand(&randState);
if (lz4f_compress_compatibility_test || ((test_selection % 2) == 0))
{
//TODO: SGL decompress with random buffer sizes
// SGL decompress with same buffer sizes used for compression
// prepare din with cout's data
sg_din_len = sg_cout_len;
for (i = 0; i < sg_din_len; i++) {
sg_din[i].sg_len = sg_cout[i].sg_len;
if (sg_cout[i].sg_len <= max_buf_size) {
// enough room to copy - do it
sg_din[i].sg_base = sg_in_buf_potential[i*2+0].sg_base;
if (sg_din[i].sg_base != sg_cout[i].sg_base) {
memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, sg_cout[i].sg_len);
}
}
else {
// this is probably single output buffer - skip copy, use directly
sg_din[i].sg_base = sg_cout[i].sg_base;
}
}
// prepare dout to receive decompressed data
sg_dout_len = sg_cin_len;
for (i = 0; i < sg_dout_len; i++) {
sg_dout[i].sg_len = sg_cin[i].sg_len;
if (sg_cin[i].sg_len <= max_buf_size) {
// enough room to decompress into independent buffer
sg_dout[i].sg_base = sg_out_buf_potential[i*2+0].sg_base;
}
else {
// this is probably single input buffer, use an external output buffer
sg_dout[i].sg_base = decodedBuffer;
}
}
size_t sourceSizeOut = cSize;
size_t maxOutputSize = srcSize;
int decomp_result = LZ4_SG_decompress(&sg_din[0], sg_din_len, &sg_dout[0], sg_dout_len, &sourceSizeOut, maxOutputSize);
CHECK(decomp_result <= 0, "SG decompression failed (error %i)", (int)decomp_result);
CHECK(decomp_result != (int)srcSize, "SG decompression stopped at %i", (int)decomp_result);
// verify result checksum
size_t total_checked = 0;
XXH64_reset(&xxh64, 1);
for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
size_t cur_size = sg_dout[i].sg_len;
size_t rem = decomp_result - total_checked;
if (rem < cur_size) cur_size = rem;
total_checked += cur_size;
XXH64_update(&xxh64, sg_dout[i].sg_base, cur_size);
}
crcDecoded = XXH64_digest(&xxh64);
if (crcDecoded != crcOrig) {
DISPLAYLEVEL(1, "checked %i out of %i (crcDecoded %08x, crcOrig %08x)\n",
(int)total_checked, decomp_result, (unsigned)crcDecoded, (unsigned)crcOrig);
// locate error if any
total_checked = 0;
for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) {
size_t cur_size = sg_dout[i].sg_len;
size_t rem = decomp_result - total_checked;
if (rem < cur_size) cur_size = rem;
total_checked += cur_size;
U64 crc_in = XXH64(sg_cin [i].sg_base, cur_size, 1);
U64 crc_out = XXH64(sg_dout[i].sg_base, cur_size, 1);
if (crc_in != crc_out) {
locateBuffDiff(sg_cin[i].sg_base, sg_dout[i].sg_base, cur_size);
break;
}
}
DISPLAYLEVEL(1, "checked %i out of %i\n",
(int)total_checked, decomp_result);
}
CHECK(crcDecoded != crcOrig, "Decompression corruption");
}
else
{
// prepare compressedBuffer from SGL
size_t total_copied = 0;
for (i = 0; i < sg_cout_len; i++) {
size_t buf_size_bytes = cSize - total_copied;
if (buf_size_bytes == 0) break;
if (buf_size_bytes > sg_cout[i].sg_len) buf_size_bytes = sg_cout[i].sg_len;
if (((char *)compressedBuffer)+total_copied != sg_cout[i].sg_base) {
memcpy(((char *)compressedBuffer)+total_copied, sg_cout[i].sg_base, buf_size_bytes);
}
total_copied += buf_size_bytes;
}
LZ4F_decompressOptions_t dOptions;
memset(&dOptions, 0, sizeof(dOptions));
const BYTE* ip = (const BYTE*)compressedBuffer;
const BYTE* const iend = ip + cSize;
BYTE* op = (BYTE*)decodedBuffer;
BYTE* const oend = op + srcDataLength;
size_t totalOut = 0;
unsigned maxBits = FUZ_highbit((U32)cSize);
XXH64_reset(&xxh64, 1);
if (maxBits < 3) maxBits = 3;
while (ip < iend)
{
unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
if (oSize > (size_t)(oend-op)) oSize = oend-op;
dOptions.stableDst = FUZ_rand(&randState) & 1;
result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid)
locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s ip %d)",
(int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result), (int)(ip-(const BYTE*)compressedBuffer));
XXH64_update(&xxh64, op, (U32)oSize);
totalOut += oSize;
op += oSize;
ip += iSize;
}
CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
if (totalOut) /* otherwise, it's a skippable frame */
{
crcDecoded = XXH64_digest(&xxh64);
if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
CHECK(crcDecoded != crcOrig, "Decompression corruption");
}
}
}
DISPLAYLEVEL(2, "\rAll tests completed \n");
_end:
LZ4F_freeDecompressionContext(dCtx);
LZ4F_freeCompressionContext(cCtx);
free(srcBuffer);
free(compressedBuffer);
free(decodedBuffer);
for (i = 0; i < NELEMS(sg_in_buf_potential); i++) {
free((void *)(sg_in_buf_potential [i].sg_base));
free( sg_out_buf_potential[i].sg_base);
}
if (num_buf_size_distribution_deviations > 0) {
DISPLAYLEVEL(2, "NOTE: %i buffer size deviations \n", (int)num_buf_size_distribution_deviations);
}
if (pause)
{
DISPLAY("press enter to finish \n");
(void)getchar();
}
return testResult;
_output_error:
testResult = 1;
goto _end;
// unreachable
return -1;
#undef CHECK
}
int main(int argCount, const char* argv[])
{
int argNb,
forceStdout=0,
followLinks=0,
main_pause=0,
nextEntryIsDictionary=0,
operationResult=0,
nextArgumentIsOutFileName=0,
nextArgumentIsMaxDict=0,
nextArgumentIsDictID=0,
nextArgumentsAreFiles=0,
ultra=0,
lastCommand = 0,
nbThreads = 1,
setRealTimePrio = 0,
separateFiles = 0,
ldmFlag = 0;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0;
const char* programName = argv[0];
const char* outFileName = NULL;
const char* dictFileName = NULL;
const char* suffix = ZSTD_EXTENSION;
unsigned maxDictSize = g_defaultMaxDictSize;
unsigned dictID = 0;
int dictCLevel = g_defaultDictCLevel;
unsigned dictSelect = g_defaultSelectivityLevel;
#ifdef UTIL_HAS_CREATEFILELIST
const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
unsigned fileNamesNb;
#endif
#ifndef ZSTD_NODICT
ZDICT_cover_params_t coverParams = defaultCoverParams();
int cover = 1;
#endif
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
(void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
(void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
g_displayOut = stderr;
programName = lastNameFromPath(programName);
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbThreads=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }
if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */
if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz */
if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like xz */
if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like unxz */
memset(&compressionParams, 0, sizeof(compressionParams));
/* command switches */
for (argNb=1; argNb<argCount; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
if (nextArgumentsAreFiles==0) {
/* "-" means stdin/stdout */
if (!strcmp(argument, "-")){
if (!filenameIdx) {
filenameIdx=1, filenameTable[0]=stdinmark;
outFileName=stdoutmark;
g_displayLevel-=(g_displayLevel==2);
continue;
} }
/* Decode commands (note : aggregated commands are allowed) */
if (argument[0]=='-') {
if (argument[1]=='-') {
/* long commands (--long-word) */
if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; }
if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
#ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
#endif
#ifdef ZSTD_LZMACOMPRESS
if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); continue; }
if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); continue; }
#endif
#ifdef ZSTD_LZ4COMPRESS
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; }
#endif
/* long commands with arguments */
#ifndef ZSTD_NODICT
if (longCommandWArg(&argument, "--train-cover")) {
operation = zom_train;
outFileName = g_defaultDictName;
cover = 1;
/* Allow optional arguments following an = */
if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
continue;
}
if (longCommandWArg(&argument, "--train-legacy")) {
operation = zom_train;
outFileName = g_defaultDictName;
cover = 0;
/* Allow optional arguments following an = */
if (*argument == 0) { continue; }
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
else if (!parseLegacyParameters(argument, &dictSelect)) { CLEAN_RETURN(badusage(programName)); }
continue;
}
#endif
if (longCommandWArg(&argument, "--threads=")) { nbThreads = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
if (longCommandWArg(&argument, "--long")) {
unsigned ldmWindowLog = 0;
ldmFlag = 1;
/* Parse optional window log */
if (*argument == '=') {
++argument;
ldmWindowLog = readU32FromChar(&argument);
} else if (*argument != 0) {
/* Invalid character following --long */
CLEAN_RETURN(badusage(programName));
}
/* Only set windowLog if not already set by --zstd */
if (compressionParams.windowLog == 0)
compressionParams.windowLog = ldmWindowLog;
continue;
}
/* fall-through, will trigger bad_usage() later on */
}
argument++;
while (argument[0]!=0) {
if (lastCommand) {
DISPLAY("error : command must be followed by argument \n");
CLEAN_RETURN(1);
}
#ifndef ZSTD_NOCOMPRESS
/* compression Level */
if ((*argument>='0') && (*argument<='9')) {
dictCLevel = cLevel = readU32FromChar(&argument);
continue;
}
#endif
switch(argument[0])
{
/* Display help */
case 'V': g_displayOut=stdout; printVersion(); CLEAN_RETURN(0); /* Version Only */
case 'H':
case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
/* Compress */
case 'z': operation=zom_compress; argument++; break;
/* Decoding */
case 'd':
#ifndef ZSTD_NOBENCH
if (operation==zom_bench) { BMK_setDecodeOnlyMode(1); argument++; break; } /* benchmark decode (hidden option) */
#endif
operation=zom_decompress; argument++; break;
/* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
/* Use file content as dictionary */
case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
/* Overwrite */
case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break;
/* Verbose mode */
case 'v': g_displayLevel++; argument++; break;
/* Quiet mode */
case 'q': g_displayLevel--; argument++; break;
/* keep source file (default) */
case 'k': FIO_setRemoveSrcFile(0); argument++; break;
/* Checksum */
case 'C': FIO_setChecksumFlag(2); argument++; break;
/* test compressed file */
case 't': operation=zom_test; argument++; break;
/* destination file name */
case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
/* limit decompression memory */
case 'M':
argument++;
memLimit = readU32FromChar(&argument);
break;
case 'l': operation=zom_list; argument++; break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
case 'r': recursive=1; argument++; break;
#endif
#ifndef ZSTD_NOBENCH
/* Benchmark */
case 'b':
operation=zom_bench;
argument++;
break;
/* range bench (benchmark only) */
case 'e':
/* compression Level */
argument++;
cLevelLast = readU32FromChar(&argument);
break;
/* Modify Nb Iterations (benchmark only) */
case 'i':
argument++;
bench_nbSeconds = readU32FromChar(&argument);
break;
/* cut input into blocks (benchmark only) */
case 'B':
argument++;
blockSize = readU32FromChar(&argument);
break;
/* benchmark files separately (hidden option) */
case 'S':
argument++;
separateFiles = 1;
break;
#endif /* ZSTD_NOBENCH */
/* nb of threads (hidden option) */
case 'T':
argument++;
nbThreads = readU32FromChar(&argument);
break;
/* Dictionary Selection level */
case 's':
argument++;
dictSelect = readU32FromChar(&argument);
break;
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
case 'p': argument++;
#ifndef ZSTD_NOBENCH
if ((*argument>='0') && (*argument<='9')) {
BMK_setAdditionalParam(readU32FromChar(&argument));
} else
#endif
main_pause=1;
break;
/* unknown command */
default : CLEAN_RETURN(badusage(programName));
}
}
continue;
} /* if (argument[0]=='-') */
if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */
nextArgumentIsMaxDict = 0;
lastCommand = 0;
maxDictSize = readU32FromChar(&argument);
continue;
}
if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */
nextArgumentIsDictID = 0;
lastCommand = 0;
dictID = readU32FromChar(&argument);
continue;
}
} /* if (nextArgumentIsAFile==0) */
if (nextEntryIsDictionary) {
nextEntryIsDictionary = 0;
lastCommand = 0;
dictFileName = argument;
continue;
}
if (nextArgumentIsOutFileName) {
nextArgumentIsOutFileName = 0;
lastCommand = 0;
outFileName = argument;
if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
continue;
}
/* add filename to list */
filenameTable[filenameIdx++] = argument;
}
if (lastCommand) { /* forgotten argument */
DISPLAY("error : command must be followed by argument \n");
CLEAN_RETURN(1);
}
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
if (nbThreads == 0) {
/* try to guess */
nbThreads = UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbThreads);
}
g_utilDisplayLevel = g_displayLevel;
if (!followLinks) {
unsigned u;
for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
if (UTIL_isLink(filenameTable[u])) {
DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
} else {
filenameTable[fileNamesNb++] = filenameTable[u];
}
}
filenameIdx = fileNamesNb;
}
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks);
if (extendedFileList) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
free((void*)filenameTable);
filenameTable = extendedFileList;
filenameIdx = fileNamesNb;
}
}
#endif
if (operation == zom_list) {
#ifndef ZSTD_NODECOMPRESS
int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
CLEAN_RETURN(ret);
#else
DISPLAY("file information is not supported \n");
CLEAN_RETURN(1);
#endif
}
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(g_displayLevel);
BMK_setSeparateFiles(separateFiles);
BMK_setBlockSize(blockSize);
BMK_setNbThreads(nbThreads);
BMK_setRealTime(setRealTimePrio);
BMK_setNbSeconds(bench_nbSeconds);
BMK_setLdmFlag(ldmFlag);
BMK_setLdmMinMatch(g_ldmMinMatch);
BMK_setLdmHashLog(g_ldmHashLog);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog);
}
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
}
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
#else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
#endif
goto _end;
}
/* Check if dictionary builder is selected */
if (operation==zom_train) {
#ifndef ZSTD_NODICT
ZDICT_params_t zParams;
zParams.compressionLevel = dictCLevel;
zParams.notificationLevel = g_displayLevel;
zParams.dictID = dictID;
if (cover) {
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = nbThreads;
coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize);
} else {
ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0);
}
#endif
goto _end;
}
#ifndef ZSTD_NODECOMPRESS
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
#endif
/* No input filename ==> use stdin and stdout */
filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */
if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark; /* when input is stdin, default output is stdout */
/* Check if input/output defined as console; trigger an error in this case */
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !strcmp(filenameTable[0], stdinmark) && !forceStdout && operation!=zom_decompress)
CLEAN_RETURN(badusage(programName));
#ifndef ZSTD_NOCOMPRESS
/* check compression level limits */
{ int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
if (cLevel > maxCLevel) {
DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
cLevel = maxCLevel;
} }
#endif
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1;
/* IO Stream/File */
FIO_setNotificationLevel(g_displayLevel);
if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS
FIO_setNbThreads(nbThreads);
FIO_setBlockSize((U32)blockSize);
FIO_setLdmFlag(ldmFlag);
FIO_setLdmHashLog(g_ldmHashLog);
FIO_setLdmMinMatch(g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
}
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
}
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams);
#else
(void)suffix;
DISPLAY("Compression not supported\n");
#endif
} else { /* decompression or test */
#ifndef ZSTD_NODECOMPRESS
if (memLimit == 0) {
if (compressionParams.windowLog == 0)
memLimit = (U32)1 << g_defaultMaxWindowLog;
else {
memLimit = (U32)1 << (compressionParams.windowLog & 31);
}
}
FIO_setMemLimit(memLimit);
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
else
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
#else
DISPLAY("Decompression not supported\n");
#endif
}
_end:
if (main_pause) waitEnter();
#ifdef UTIL_HAS_CREATEFILELIST
if (extendedFileList)
UTIL_freeFileList(extendedFileList, fileNamesBuf);
else
#endif
free((void*)filenameTable);
return operationResult;
}
static void FUZ_tests (U32 seed, U32 totalTest, U32 startTestNb)
{
BYTE* bufferP0 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP1 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP15 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP90 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferP100 = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferDst = (BYTE*) malloc (BUFFERSIZE+64);
BYTE* bufferVerif = (BYTE*) malloc (BUFFERSIZE+64);
size_t bufferDstSize = BUFFERSIZE+64;
unsigned testNb, maxSV, tableLog;
const size_t maxTestSizeMask = 0x1FFFF;
U32 rootSeed = seed;
U32 time = FUZ_GetMilliStart();
generateNoise (bufferP0, BUFFERSIZE, &rootSeed);
generate (bufferP1 , BUFFERSIZE, 0.01, &rootSeed);
generate (bufferP15 , BUFFERSIZE, 0.15, &rootSeed);
generate (bufferP90 , BUFFERSIZE, 0.90, &rootSeed);
memset(bufferP100, (BYTE)FUZ_rand(&rootSeed), BUFFERSIZE);
if (startTestNb)
{
U32 i;
for (i=0; i<startTestNb; i++)
FUZ_rand (&rootSeed);
}
for (testNb=startTestNb; testNb<totalTest; testNb++)
{
BYTE* bufferTest;
int tag=0;
U32 roundSeed = rootSeed ^ 0xEDA5B371;
FUZ_rand(&rootSeed);
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
{
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
{
/* determine test sample */
size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U32 hashOrig;
if (FUZ_rand(&roundSeed) & 7) bufferTest = bufferP15 + offset;
else
{
switch(FUZ_rand(&roundSeed) & 3)
{
case 0: bufferTest = bufferP0 + offset; break;
case 1: bufferTest = bufferP1 + offset; break;
case 2: bufferTest = bufferP90 + offset; break;
default : bufferTest = bufferP100 + offset; break;
}
}
DISPLAYLEVEL (4,"%3i ", tag++);;
hashOrig = XXH32 (bufferTest, sizeOrig, 0);
/* compress test */
sizeCompressed = FSE_compress (bufferDst, bufferDstSize, bufferTest, sizeOrig);
CHECK(FSE_isError(sizeCompressed), "Compression failed !");
if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */
{
/* failed compression test*/
{
size_t errorCode;
void* tooSmallDBuffer = malloc(sizeCompressed-1); /* overflows detected with Valgrind */
CHECK(tooSmallDBuffer==NULL, "Not enough memory for tooSmallDBuffer test");
errorCode = FSE_compress (tooSmallDBuffer, sizeCompressed-1, bufferTest, sizeOrig);
CHECK(errorCode!=0, "Compression should have failed : destination buffer too small");
free(tooSmallDBuffer);
}
/* decompression test */
{
U32 hashEnd;
BYTE saved = (bufferVerif[sizeOrig] = 254);
size_t result = FSE_decompress (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "Output buffer overrun (bufferVerif) : write beyond specified end");
CHECK(FSE_isError(result), "Decompression failed");
hashEnd = XXH32 (bufferVerif, sizeOrig, 0);
CHECK(hashEnd != hashOrig, "Decompressed data corrupted");
}
}
}
/* Attempt header decoding on bogus data */
{
short count[256];
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
maxSV = 255;
result = FSE_readNCount (count, &maxSV, &tableLog, bufferTest, FSE_NCOUNTBOUND);
if (!FSE_isError(result)) /* an error would be normal */
{
int checkCount;
CHECK(result > FSE_NCOUNTBOUND, "FSE_readHeader() reads too far (buffer overflow)");
CHECK(maxSV > 255, "count table overflow (%u)", maxSV+1);
checkCount = FUZ_checkCount(count, tableLog, maxSV);
CHECK(checkCount==-1, "symbol distribution corrupted");
}
}
/* Attempt decompression on bogus data */
{
size_t maxDstSize = FUZ_rand (&roundSeed) & maxTestSizeMask;
size_t sizeCompressed = FUZ_rand (&roundSeed) & maxTestSizeMask;
BYTE saved = (bufferDst[maxDstSize] = 253);
size_t result;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);;
result = FSE_decompress (bufferDst, maxDstSize, bufferTest, sizeCompressed);
CHECK(!FSE_isError(result) && (result > maxDstSize), "Decompression overran output buffer");
CHECK(bufferDst[maxDstSize] != saved, "Output buffer bufferDst corrupted");
}
}
/* exit */
free (bufferP0);
free (bufferP1);
free (bufferP15);
free (bufferP90);
free (bufferP100);
free (bufferDst);
free (bufferVerif);
}
int main (int argc, char** argv)
{
U32 seed, startTestNb=0, pause=0, totalTest = FUZ_NB_TESTS;
int argNb;
seed = FUZ_GetMilliStart() % 10000;
DISPLAYLEVEL (1, "FSE (%2i bits) automated test\n", (int)sizeof(void*)*8);
for (argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if (argument[0]=='-')
{
argument++;
while (argument[0]!=0)
{
switch (argument[0])
{
/* seed setting */
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
/* total tests */
case 'i':
argument++;
totalTest=0;
while ((*argument>='0') && (*argument<='9'))
{
totalTest *= 10;
totalTest += *argument - '0';
argument++;
}
break;
/* jump to test nb */
case 't':
argument++;
startTestNb=0;
while ((*argument>='0') && (*argument<='9'))
{
startTestNb *= 10;
startTestNb += *argument - '0';
argument++;
}
break;
/* verbose mode */
case 'v':
argument++;
displayLevel=4;
break;
/* pause (hidden) */
case 'p':
argument++;
pause=1;
break;
default:
return badUsage(argv[0]);
}
}
}
}
if (startTestNb == 0) unitTest();
DISPLAY("Fuzzer seed : %u \n", seed);
FUZ_tests (seed, totalTest, startTestNb);
DISPLAY ("\rAll %u tests passed \n", totalTest);
if (pause)
{
int unused;
DISPLAY("press enter ...\n");
unused = getchar();
(void)unused;
}
return 0;
}
static int badusage(const char* programName)
{
DISPLAYLEVEL(1, "Incorrect parameters\n");
if (displayLevel >= 1) usage(programName);
return 1;
}
static int basicUnitTests(U32 seed, double compressibility)
{
int testResult = 0;
void* CNBuffer;
void* compressedBuffer;
void* decodedBuffer;
U32 randState = seed;
size_t result, cSize;
U32 testNb=0;
/* Create compressible test buffer */
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
if (!CNBuffer || !compressedBuffer || !decodedBuffer)
{
DISPLAY("Not enough memory, aborting\n");
testResult = 1;
goto _end;
}
RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState);
/* Basic tests */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (ZSTD_isError(result)) goto _output_error;
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
{
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++)
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Dictionary and Duplication tests */
{
ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
const size_t dictSize = 500;
size_t cSizeOrig;
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
result = ZSTD_compressBegin(ctxOrig, 2);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
cSize = 0;
result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeCCtx(ctxOrig); /* if ctxOrig is read, will produce segfault */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
cSizeOrig = cSize;
cSize = 0;
result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
if (ZSTD_isError(result)) goto _output_error;
cSize += result;
if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
ZSTD_freeCCtx(ctxDuplicated);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
result = ZSTD_decompress_usingDict(dctx,
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
compressedBuffer, cSize,
CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
ZSTD_freeDCtx(dctx);
DISPLAYLEVEL(4, "OK \n");
}
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
if (!ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* block API tests */
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t blockSize = 100 KB;
const size_t dictSize = 16 KB;
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* dictionary block compression */
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
result = ZSTD_compressBegin(cctx, 5);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(cctx, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
if (ZSTD_isError(result)) goto _output_error;
ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize);
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
/* long rle test */
{
size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error;
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result!=sampleSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
}
_end:
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
int main(int argc, char** argv)
{
int argNb;
int proba = CDG_COMPRESSIBILITY_DEFAULT;
U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT;
// Check command line
programName = argv[0];
for(argNb=1; argNb<argc; argNb++)
{
char* argument = argv[argNb];
if(!argument) continue; // Protection if argument empty
// Decode command (note : aggregated commands are allowed)
if (*argument=='-')
{
if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
argument++;
while (*argument!=0)
{
switch(*argument)
{
case 'h':
return CDG_usage();
case 'g':
argument++;
size=0;
while ((*argument>='0') && (*argument<='9'))
{
size *= 10;
size += *argument - '0';
argument++;
}
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
if (*argument=='B') { argument++; }
break;
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
{
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 'p':
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default: ;
}
}
}
}
// Get Seed
DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
CDG_generate(size, &seed, ((double)proba) / 100);
return 0;
}
static void FUZ_tests (const U32 startSeed, U32 totalTest, U32 startTestNb)
{
size_t bufferDstSize = BUFFERSIZE*sizeof(U16) + 64;
U16* bufferP8 = (U16*) malloc (bufferDstSize);
void* bufferDst = malloc (bufferDstSize);
U16* bufferVerif = (U16*) malloc (bufferDstSize);
unsigned testNb;
const size_t maxTestSizeMask = 0x1FFFF;
U32 time = FUZ_GetMilliStart();
U32 seed = startSeed;
generateU16 (bufferP8, BUFFERSIZE, 0.08, seed);
if (startTestNb)
{
U32 i;
for (i=0; i<startTestNb; i++)
FUZ_rand (&seed);
}
for (testNb=startTestNb; testNb<totalTest; testNb++)
{
U16* bufferTest;
int tag=0;
U32 roundSeed = seed ^ 0xEDA5B371;
FUZ_rand(&seed);
DISPLAYLEVEL (4, "\r test %5u ", testNb);
if (FUZ_GetMilliSpan (time) > FUZ_UPDATERATE)
{
DISPLAY ("\r test %5u ", testNb);
time = FUZ_GetMilliStart();
}
/* Compression / Decompression tests */
{
size_t sizeOrig = (FUZ_rand (&roundSeed) & maxTestSizeMask) + 1;
size_t offset = (FUZ_rand(&roundSeed) % (BUFFERSIZE - 64 - maxTestSizeMask));
size_t sizeCompressed;
U64 hashOrig;
bufferTest = bufferP8 + offset;
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
hashOrig = XXH64 (bufferTest, sizeOrig * sizeof(U16), 0);
sizeCompressed = FSE_compressU16 (bufferDst, bufferDstSize, bufferTest, sizeOrig, FSE_MAX_SYMBOL_VALUE, 12);
CHECK(FSE_isError(sizeCompressed), "\r test %5u : FSE_compressU16 failed !", testNb);
if (sizeCompressed > 1) /* don't check uncompressed & rle corner cases */
{
U64 hashEnd;
U16 saved = (bufferVerif[sizeOrig] = 1024 + 250);
size_t dstSize;
size_t result;
/* basic decompression test : should work */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
result = FSE_decompressU16 (bufferVerif, sizeOrig, bufferDst, sizeCompressed);
CHECK(bufferVerif[sizeOrig] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);
/* larger output buffer than necessary : should work */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
result = FSE_decompressU16 (bufferVerif, sizeOrig + (FUZ_rand(&roundSeed) & 31) + 1, bufferDst, sizeCompressed);
CHECK(FSE_isError(result), "\r test %5u : FSE_decompressU16 failed : %s ! (origSize = %u shorts, cSize = %u bytes)", testNb, FSE_getErrorName(result), (U32)sizeOrig, (U32)sizeCompressed);
hashEnd = XXH64 (bufferVerif, result * sizeof(U16), 0);
CHECK(hashEnd != hashOrig, "\r test %5u : Decompressed data corrupted !!", testNb);
/* smaller output buffer than required : should fail */
DISPLAYLEVEL (4,"\b\b\b\b%3i ", tag++);
dstSize = (FUZ_rand(&roundSeed) & 31) + 1;
if (dstSize >= sizeOrig) dstSize = 1;
dstSize = sizeOrig - dstSize;
saved = (bufferVerif[dstSize] = 1024 + 250);
result = FSE_decompressU16 (bufferVerif, dstSize, bufferDst, sizeCompressed);
CHECK(bufferVerif[dstSize] != saved, "\r test %5u : FSE_decompressU16 overrun output buffer (write beyond specified end) !", testNb);
CHECK(!FSE_isError(result), "\r test %5u : FSE_decompressU16 should have failed ! (origSize = %u shorts, dstSize = %u bytes)", testNb, (U32)sizeOrig, (U32)dstSize);
}
}
}
/* clean */
free (bufferP8);
free (bufferDst);
free (bufferVerif);
}
