unsigned long long FIOv07_decompressFrame(dRessv07_t ress,
FILE* foutput, FILE* finput)
{
U64 frameSize = 0;
size_t readSize = 4;
MEM_writeLE32(ress.srcBuffer, ZSTDv07_MAGICNUMBER);
ZBUFFv07_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) {
/* Decode */
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFFv07_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFFv07_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv07_getErrorName(toRead));
readSize -= inSize;
/* Write block */
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); }
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error");
}
return frameSize;
}
static void FIOv07_freeDResources(dRessv07_t ress)
{
size_t const errorCode = ZBUFFv07_freeDCtx(ress.dctx);
if (ZBUFFv07_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv07_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
}
static dRessv07_t FIOv07_createDResources(void)
{
dRessv07_t ress;
/* init */
ress.dctx = ZBUFFv07_createDCtx();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
ress.dictBuffer = NULL; ress.dictBufferSize=0;
/* Allocate Memory */
ress.srcBufferSize = ZBUFFv07_recommendedDInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFFv07_recommendedDOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
return ress;
}
static void get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
{
if (!strcmp (input_filename, stdinmark))
{
DISPLAYLEVEL(4,"Using stdin for input\n");
*pfinput = stdin;
SET_BINARY_MODE(stdin);
}
else
{
*pfinput = fopen(input_filename, "rb");
}
if (!strcmp (output_filename, stdoutmark))
{
DISPLAYLEVEL(4,"Using stdout for output\n");
*pfoutput = stdout;
SET_BINARY_MODE(stdout);
}
else
{
/* Check if destination file already exists */
*pfoutput=0;
if (strcmp(output_filename,nulmark)) *pfoutput = fopen( output_filename, "rb" );
if (*pfoutput!=0)
{
fclose(*pfoutput);
if (!g_overwrite)
{
char ch;
if (g_displayLevel <= 1) /* No interaction possible */
EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
ch = (char)getchar();
if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
}
}
*pfoutput = fopen( output_filename, "wb" );
}
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
}
unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
{
size_t const outBuffSize = 512 KB;
BYTE* outBuff = (BYTE*)malloc(outBuffSize);
size_t inBuffSize = 128 KB + 8;
BYTE inBuff[128 KB + 8];
BYTE* op = outBuff;
BYTE* const oend = outBuff + outBuffSize;
U64 filesize = 0;
size_t toRead;
size_t sizeCheck;
ZSTDv01_Dctx* dctx = ZSTDv01_createDCtx();
/* init */
if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
/* restore header, already read from input */
MEM_writeLE32(inBuff, ZSTDv01_magicNumberLE);
sizeCheck = ZSTDv01_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv01_magicNumberLE)); /* Decode frame header */
if (ZSTDv01_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header");
/* Main decompression Loop */
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
while (toRead){
size_t readSize, decodedSize;
/* Fill input buffer */
if (toRead > inBuffSize)
EXM_THROW(43, "too large block");
readSize = fread(inBuff, 1, toRead, finput);
if (readSize != toRead)
EXM_THROW(44, "Read error");
/* Decode block */
decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) { /* not a header */
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
filesize += decodedSize;
op += decodedSize;
if (op==oend) op = outBuff;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
}
/* prepare for next Block */
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
}
/* release resources */
free(outBuff);
free(dctx);
return filesize;
}
/*
Compressed format : MAGICNUMBER - STREAMDESCRIPTOR - ( BLOCKHEADER - COMPRESSEDBLOCK ) - STREAMCRC
MAGICNUMBER - 4 bytes - Designates compression algo
STREAMDESCRIPTOR - 1 byte
bits 0-3 : max block size, 2^value from 0 to 0xA; min 0=>1KB, max 0x6=>64KB, typical 5=>32 KB
bits 4-7 = 0 : reserved;
BLOCKHEADER - 1-5 bytes
1st byte :
bits 6-7 : blockType (compressed, raw, rle, crc (end of Frame)
bit 5 : full block
** if not full block **
2nd & 3rd byte : regenerated size of block (big endian); note : 0 = 64 KB
** if blockType==compressed **
next 2 bytes : compressed size of block
COMPRESSEDBLOCK
the compressed data itself.
STREAMCRC - 3 bytes (including 1-byte blockheader)
22 bits (xxh32() >> 5) checksum of the original data, big endian
*/
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
char* in_buff;
char* out_buff;
FILE* finput;
FILE* foutput;
size_t sizeCheck;
size_t inputBlockSize = FIO_blockID_to_blockSize(g_blockSizeId);
XXH32_state_t xxhState;
typedef size_t (*compressor_t) (void* dst, size_t dstSize, const void* src, size_t srcSize);
compressor_t compressor;
unsigned magicNumber;
/* Init */
XXH32_reset (&xxhState, FSE_CHECKSUM_SEED);
get_fileHandle(input_filename, output_filename, &finput, &foutput);
switch (g_compressor)
{
case FIO_fse:
compressor = FSE_compress;
magicNumber = FIO_magicNumber_fse;
break;
case FIO_huff0:
compressor = HUF_compress;
magicNumber = FIO_magicNumber_huff0;
break;
case FIO_zlibh:
compressor = FIO_ZLIBH_compress;
magicNumber = FIO_magicNumber_zlibh;
break;
default :
EXM_THROW(20, "unknown compressor selection");
}
/* Allocate Memory */
if (inputBlockSize==0) EXM_THROW(0, "impossible problem, to please static analyzer");
in_buff = (char*)malloc(inputBlockSize);
out_buff = (char*)malloc(FSE_compressBound(inputBlockSize) + 5);
if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
/* Write Frame Header */
FIO_writeLE32(out_buff, magicNumber);
out_buff[4] = (char)g_blockSizeId; /* Max Block Size descriptor */
sizeCheck = fwrite(out_buff, 1, FIO_FRAMEHEADERSIZE, foutput);
if (sizeCheck!=FIO_FRAMEHEADERSIZE) EXM_THROW(22, "Write error : cannot write header");
compressedfilesize += FIO_FRAMEHEADERSIZE;
/* Main compression loop */
while (1)
{
/* Fill input Buffer */
size_t cSize;
size_t inSize = fread(in_buff, (size_t)1, (size_t)inputBlockSize, finput);
if (inSize==0) break;
filesize += inSize;
XXH32_update(&xxhState, in_buff, inSize);
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */
cSize = compressor(out_buff + FIO_maxBlockHeaderSize, FSE_compressBound(inputBlockSize), in_buff, inSize);
if (FSE_isError(cSize)) EXM_THROW(23, "Compression error : %s ", FSE_getErrorName(cSize));
/* Write cBlock */
switch(cSize)
{
size_t headerSize;
case 0: /* raw */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_raw << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
sizeCheck = fwrite(out_buff, 1, headerSize, foutput);
if (sizeCheck!=headerSize) EXM_THROW(24, "Write error : cannot write block header");
sizeCheck = fwrite(in_buff, 1, inSize, foutput);
if (sizeCheck!=(size_t)(inSize)) EXM_THROW(25, "Write error : cannot write block");
compressedfilesize += inSize + headerSize;
break;
case 1: /* rle */
if (inSize == inputBlockSize)
{
out_buff[0] = (BYTE)((bt_rle << 6) + BIT5);
headerSize = 1;
}
else
{
out_buff[2] = (BYTE)inSize;
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[0] = (BYTE)(bt_raw << 6);
headerSize = 3;
}
out_buff[headerSize] = in_buff[0];
sizeCheck = fwrite(out_buff, 1, headerSize+1, foutput);
if (sizeCheck!=(headerSize+1)) EXM_THROW(26, "Write error : cannot write rle block");
compressedfilesize += headerSize + 1;
break;
default : /* compressed */
if (inSize == inputBlockSize)
{
out_buff[2] = (BYTE)((bt_compressed << 6) + BIT5);
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = 3;
}
else
{
out_buff[0] = (BYTE)(bt_compressed << 6);
out_buff[1] = (BYTE)(inSize >> 8);
out_buff[2] = (BYTE)inSize;
out_buff[3] = (BYTE)(cSize >> 8);
out_buff[4] = (BYTE)cSize;
headerSize = FIO_maxBlockHeaderSize;
}
sizeCheck = fwrite(out_buff+(FIO_maxBlockHeaderSize-headerSize), 1, headerSize+cSize, foutput);
if (sizeCheck!=(headerSize+cSize)) EXM_THROW(27, "Write error : cannot write rle block");
compressedfilesize += headerSize + cSize;
break;
}