HRESULT compress_replacement(ISequentialOutStream *outStream, uint8_t *data, size_t len, UInt64 *outSizeProcessed)
{
CLibbscProps props;
// C=f
props.paramSortingContexts = LIBBSC_CONTEXTS_FOLLOWING;
// E=2
// LIBBSC_CODER_QLFC_ADAPTIVE
props.features |= PARAM_CODER_QLFC_ADAPTIVE;
// B=20
props.paramBlockSize = 20 * 1024 * 1024;
UInt32 nReduceSize = (UInt32)len;
UInt32 kMinBlockSize = 1 * 1024 * 1024;
if (nReduceSize < props.paramBlockSize) {
if (nReduceSize < kMinBlockSize)
props.paramBlockSize = kMinBlockSize;
else
props.paramBlockSize = nReduceSize;
}
int segmentedBlock[256] = { 0 };
if (bsc_init(props.paramFeatures()) != LIBBSC_NO_ERROR)
{
fprintf(stderr, "\nInternal program error, please contact the author!\n");
return E_FAIL;
}
#ifdef LIBBSC_OPENMP
int numThreads = 1;
if (paramEnableParallelProcessing)
{
numThreads = omp_get_max_threads();
if (numThreads <= nBlocks) paramEnableMultiThreading = 0;
if (numThreads >= nBlocks) numThreads = nBlocks;
}
#endif
int segmentationStart = 0, segmentationEnd = 0;
#ifdef LIBBSC_OPENMP
#pragma omp parallel num_threads(numThreads) if(numThreads > 1)
#endif
{
unsigned char * buffer = (unsigned char *)bsc_malloc(props.paramBlockSize + LIBBSC_HEADER_SIZE);
unsigned char * buffer2 = (unsigned char *)bsc_malloc(props.paramBlockSize + LIBBSC_HEADER_SIZE);
if (buffer == NULL || buffer2 == NULL)
return E_OUTOFMEMORY;
BSC_FILEOFFSET blockOffset = 0;
BSC_FILEOFFSET prevBlockOffset = 0;
BSC_FILEOFFSET readPos = 0;
//HRESULT res = SZ_OK;
for (;;)
{
UInt32 dataSize = 0;
#ifdef LIBBSC_OPENMP
#pragma omp critical(input)
#endif
{
{
UInt32 currentBlockSize = props.paramBlockSize;
if (props.paramEnableSegmentation())
{
if (segmentationEnd - segmentationStart > 1) currentBlockSize = segmentedBlock[segmentationStart];
}
if (blockOffset < readPos) {
UInt32 moveOffset = (UInt32)(blockOffset - prevBlockOffset);
UInt32 remained = (UInt32)(readPos - blockOffset);
memmove(buffer, buffer + moveOffset, remained);
dataSize = (UInt32)min(props.paramBlockSize - remained, nReduceSize - readPos);
memcpy(buffer + remained, data + readPos, dataSize);
readPos += dataSize;
dataSize += remained;
}
else {
dataSize = (UInt32)min(props.paramBlockSize, nReduceSize - readPos);
memcpy(buffer, data + readPos, dataSize);
readPos += dataSize;
}
if (dataSize <= 0)
break;
if (props.paramEnableSegmentation()) {
if (dataSize > currentBlockSize)
dataSize = currentBlockSize;
}
if (props.paramEnableSegmentation())
{
bool bSegmentation = false;
if (segmentationStart == segmentationEnd) bSegmentation = true;
if ((segmentationEnd - segmentationStart == 1) &&
((int)dataSize != segmentedBlock[segmentationStart]))
bSegmentation = true;
if (bSegmentation)
{
segmentationStart = 0; segmentationEnd = bsc_detect_segments(buffer, dataSize, segmentedBlock, 256, props.paramFeatures());
if (segmentationEnd <= LIBBSC_NO_ERROR)
{
switch (segmentationEnd)
{
case LIBBSC_NOT_ENOUGH_MEMORY:
return E_OUTOFMEMORY;
default:
return E_FAIL;
}
}
}
int newDataSize = segmentedBlock[segmentationStart++];
if ((int)dataSize != newDataSize)
{
// BSC_FILEOFFSET pos = blockOffset + newDataSize;
// BSC_FSEEK(fInput, pos, SEEK_SET);
dataSize = newDataSize;
}
}
}
}
if (dataSize == 0) break;
memcpy(buffer2, buffer, dataSize);
signed char recordSize = 1;
if (props.paramEnableReordering())
{
recordSize = (signed char)bsc_detect_recordsize(buffer2, dataSize, props.paramFeatures());
if ((int)recordSize < LIBBSC_NO_ERROR)
{
switch (recordSize)
{
case LIBBSC_NOT_ENOUGH_MEMORY:
return E_OUTOFMEMORY;
default:
return E_FAIL;
}
}
if (recordSize > 1)
{
int result = bsc_reorder_forward(buffer2, dataSize, recordSize, props.paramFeatures());
if (result != LIBBSC_NO_ERROR)
{
switch (result)
{
case LIBBSC_NOT_ENOUGH_MEMORY:
return E_OUTOFMEMORY;
default:
return E_FAIL;
}
}
}
}
signed char sortingContexts = props.paramSortingContexts;
if ((int)props.paramSortingContexts == LIBBSC_CONTEXTS_AUTODETECT)
{
sortingContexts = (signed char)bsc_detect_contextsorder(buffer2, dataSize, props.paramFeatures());
if (sortingContexts < LIBBSC_NO_ERROR)
{
switch (sortingContexts)
{
case LIBBSC_NOT_ENOUGH_MEMORY:
return E_OUTOFMEMORY;
default:
return E_FAIL;
}
}
}
if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING)
{
int result = bsc_reverse_block(buffer2, dataSize, props.paramFeatures());
if (result != LIBBSC_NO_ERROR)
return E_FAIL;
}
int blockSize = bsc_compress(buffer2, buffer2, dataSize,
props.paramLZPHashSize, props.paramLZPMinLen, props.paramBlockSorter,
props.paramCoder(), props.paramFeatures());
if (blockSize == LIBBSC_NOT_COMPRESSIBLE)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(input)
#endif
{
sortingContexts = LIBBSC_CONTEXTS_FOLLOWING; recordSize = 1;
// BSC_FILEOFFSET pos = blockOffset;
// BSC_FSEEK(fInput, blockOffset, SEEK_SET);
// RINOK(read(inStream, buffer, dataSize, &dataSize));
// _inSizeProcessed = blockOffset + dataSize;
// BSC_FSEEK(fInput, pos, SEEK_SET);
// memcpy(buffer2, buffer, dataSize);
}
blockSize = bsc_store(buffer, buffer2, dataSize, props.paramFeatures());
}
if (blockSize < LIBBSC_NO_ERROR)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
{
switch (blockSize)
{
case LIBBSC_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
case LIBBSC_NOT_SUPPORTED: fprintf(stderr, "\nSpecified compression method is not supported on this platform!\n"); break;
case LIBBSC_GPU_ERROR: fprintf(stderr, "\nGeneral GPU failure! Please check README file for more information.\n"); break;
case LIBBSC_GPU_NOT_SUPPORTED: fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break;
case LIBBSC_GPU_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break;
default: fprintf(stderr, "\nInternal program error, please contact the author!\n");
}
return E_FAIL;
}
}
#ifdef LIBBSC_OPENMP
#pragma omp critical(output)
#endif
{
Byte next = 1;
RINOK(WriteStream(outStream, &next, 1));
*outSizeProcessed += 1;
// BSC_BLOCK_HEADER header = { blockOffset, recordSize, sortingContexts };
// RINOK(outStream->Write(&header, sizeof(BSC_BLOCK_HEADER), &processedSize));
write_int(outStream, blockOffset, outSizeProcessed);
write_int(outStream, recordSize, outSizeProcessed);
write_int(outStream, sortingContexts, outSizeProcessed);
RINOK(WriteStream(outStream, buffer2, blockSize));
*outSizeProcessed += blockSize;
prevBlockOffset = blockOffset;
blockOffset = blockOffset + dataSize;
}
}
bsc_free(buffer2);
bsc_free(buffer);
}
{
Byte next = 0;
RINOK(WriteStream(outStream, &next, 1));
*outSizeProcessed += 1;
}
return S_OK;
}
void Compression(char * argv[])
{
if (!paramEnableLZP)
{
paramLZPHashSize = 0;
paramLZPMinLen = 0;
}
FILE * fInput = fopen(argv[2], "rb");
if (fInput == NULL)
{
fprintf(stderr, "Can't open input file: %s!\n", argv[2]);
exit(1);
}
FILE * fOutput = fopen(argv[3], "wb");
if (fOutput == NULL)
{
fprintf(stderr, "Can't create output file: %s!\n", argv[3]);
exit(1);
}
if (BSC_FSEEK(fInput, 0, SEEK_END))
{
fprintf(stderr, "IO error on file: %s!\n", argv[2]);
exit(1);
}
BSC_FILEOFFSET fileSize = BSC_FTELL(fInput);
if (fileSize < 0)
{
fprintf(stderr, "IO error on file: %s!\n", argv[2]);
exit(1);
}
if (BSC_FSEEK(fInput, 0, SEEK_SET))
{
fprintf(stderr, "IO error on file: %s!\n", argv[2]);
exit(1);
}
if (paramBlockSize > fileSize)
{
paramBlockSize = (int)fileSize;
}
if (fwrite(bscFileSign, sizeof(bscFileSign), 1, fOutput) != 1)
{
fprintf(stderr, "IO error on file: %s!\n", argv[3]);
exit(1);
}
int nBlocks = paramBlockSize > 0 ? (int)((fileSize + paramBlockSize - 1) / paramBlockSize) : 0;
if (fwrite(&nBlocks, sizeof(nBlocks), 1, fOutput) != 1)
{
fprintf(stderr, "IO error on file: %s!\n", argv[3]);
exit(1);
}
double startTime = BSC_CLOCK();
#ifdef LIBBSC_OPENMP
int numThreads = 1;
if (paramEnableParallelProcessing)
{
numThreads = omp_get_max_threads();
if (numThreads > nBlocks)
{
numThreads = nBlocks;
}
}
#endif
int segmentationStart = 0, segmentationEnd = 0;
#ifdef LIBBSC_OPENMP
#pragma omp parallel num_threads(numThreads) if(numThreads > 1)
#endif
{
unsigned char * buffer = (unsigned char *)bsc_malloc(paramBlockSize + LIBBSC_HEADER_SIZE);
if (buffer == NULL)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
{
fprintf(stderr, "Not enough memory! Please check README file for more information.\n");
exit(2);
}
}
while (true)
{
BSC_FILEOFFSET blockOffset = 0;
int dataSize = 0;
#ifdef LIBBSC_OPENMP
#pragma omp critical(input)
#endif
{
if ((feof(fInput) == 0) && (BSC_FTELL(fInput) != fileSize))
{
#ifdef LIBBSC_OPENMP
#pragma omp master
#endif
{
double progress = (100.0 * (double)BSC_FTELL(fInput)) / fileSize;
fprintf(stdout, "\rCompressing %.55s(%02d%%)", argv[2], (int)progress);
fflush(stdout);
}
blockOffset = BSC_FTELL(fInput);
int currentBlockSize = paramBlockSize;
if (paramEnableSegmentation)
{
if (segmentationEnd - segmentationStart > 1) currentBlockSize = segmentedBlock[segmentationStart];
}
dataSize = (int)fread(buffer, 1, currentBlockSize, fInput);
if (dataSize <= 0)
{
fprintf(stderr, "\nIO error on file: %s!\n", argv[2]);
exit(1);
}
if (paramEnableSegmentation)
{
bool bSegmentation = false;
if (segmentationStart == segmentationEnd) bSegmentation = true;
if ((segmentationEnd - segmentationStart == 1) && (dataSize != segmentedBlock[segmentationStart])) bSegmentation = true;
if (bSegmentation)
{
segmentationStart = 0; segmentationEnd = bsc_detect_segments(buffer, dataSize, segmentedBlock, 256, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
if (segmentationEnd <= LIBBSC_NO_ERROR)
{
switch (segmentationEnd)
{
case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
default : fprintf(stderr, "\nInternal program error, please contact the author!\n");
}
exit(2);
}
}
int newDataSize = segmentedBlock[segmentationStart++];
if (dataSize != newDataSize)
{
BSC_FILEOFFSET pos = BSC_FTELL(fInput) - dataSize + newDataSize;
BSC_FSEEK(fInput, pos, SEEK_SET);
dataSize = newDataSize;
}
}
}
}
if (dataSize == 0) break;
char recordSize = 1;
if (paramEnableReordering)
{
recordSize = bsc_detect_recordsize(buffer, dataSize, LIBBSC_FEATURE_FASTMODE);
if (recordSize < LIBBSC_NO_ERROR)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
{
switch (recordSize)
{
case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
default : fprintf(stderr, "\nInternal program error, please contact the author!\n");
}
exit(2);
}
}
if (recordSize > 1)
{
int result = bsc_reorder_forward(buffer, dataSize, recordSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
if (result != LIBBSC_NO_ERROR)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
{
switch (result)
{
case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
default : fprintf(stderr, "\nInternal program error, please contact the author!\n");
}
exit(2);
}
}
}
}
char sortingContexts = paramSortingContexts;
if (paramSortingContexts == LIBBSC_CONTEXTS_AUTODETECT)
{
sortingContexts = bsc_detect_contextsorder(buffer, dataSize, LIBBSC_FEATURE_FASTMODE);
if (sortingContexts < LIBBSC_NO_ERROR)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
{
switch (sortingContexts)
{
case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory!\n"); break;
default : fprintf(stderr, "\nInternal program error, please contact the author!\n");
}
exit(2);
}
}
}
if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING)
{
int result = bsc_reverse_block(buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
if (result != LIBBSC_NO_ERROR)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
{
fprintf(stderr, "\nInternal program error, please contact the author!\n");
exit(2);
}
}
}
int features =
(paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE) |
(paramEnableFastMode ? LIBBSC_FEATURE_FASTMODE : LIBBSC_FEATURE_NONE) |
(paramEnableCUDA ? LIBBSC_FEATURE_CUDA : LIBBSC_FEATURE_NONE)
;
int blockSize = bsc_compress(buffer, buffer, dataSize, paramLZPHashSize, paramLZPMinLen, paramBlockSorter, features);
if (blockSize == LIBBSC_NOT_COMPRESSIBLE)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(input)
#endif
{
sortingContexts = LIBBSC_CONTEXTS_FOLLOWING; recordSize = 1;
BSC_FILEOFFSET pos = BSC_FTELL(fInput);
{
BSC_FSEEK(fInput, blockOffset, SEEK_SET);
if (dataSize != (int)fread(buffer, 1, dataSize, fInput))
{
fprintf(stderr, "\nInternal program error, please contact the author!\n");
exit(2);
}
}
BSC_FSEEK(fInput, pos, SEEK_SET);
}
blockSize = bsc_store(buffer, buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
}
if (blockSize < LIBBSC_NO_ERROR)
{
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
{
switch (blockSize)
{
case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
case LIBBSC_NOT_SUPPORTED : fprintf(stderr, "\nSpecified compression method is not supported on this platform!\n"); break;
case LIBBSC_GPU_ERROR : fprintf(stderr, "\nGeneral GPU failure, please contact the author!\n"); break;
case LIBBSC_GPU_NOT_SUPPORTED : fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break;
case LIBBSC_GPU_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break;
default : fprintf(stderr, "\nInternal program error, please contact the author!\n");
}
exit(2);
}
}
#ifdef LIBBSC_OPENMP
#pragma omp critical(output)
#endif
{
BSC_BLOCK_HEADER header = {blockOffset, recordSize, sortingContexts};
if (fwrite(&header, sizeof(BSC_BLOCK_HEADER), 1, fOutput) != 1)
{
fprintf(stderr, "\nIO error on file: %s!\n", argv[3]);
exit(1);
}
if ((int)fwrite(buffer, 1, blockSize, fOutput) != blockSize)
{
fprintf(stderr, "\nIO error on file: %s!\n", argv[3]);
exit(1);
}
}
}
bsc_free(buffer);
}
fprintf(stdout, "\r%.55s compressed %.0f into %.0f in %.3f seconds.\n", argv[2], (double)fileSize, (double)BSC_FTELL(fOutput), BSC_CLOCK() - startTime);
fclose(fInput); fclose(fOutput);
}
