HRESULT compress_replacement(ISequentialOutStream *outStream, uint8_t *data, size_t len, UInt64 *outSizeProcessed) { CLibbscProps props; // C=f props.paramSortingContexts = LIBBSC_CONTEXTS_FOLLOWING; // E=2 // LIBBSC_CODER_QLFC_ADAPTIVE props.features |= PARAM_CODER_QLFC_ADAPTIVE; // B=20 props.paramBlockSize = 20 * 1024 * 1024; UInt32 nReduceSize = (UInt32)len; UInt32 kMinBlockSize = 1 * 1024 * 1024; if (nReduceSize < props.paramBlockSize) { if (nReduceSize < kMinBlockSize) props.paramBlockSize = kMinBlockSize; else props.paramBlockSize = nReduceSize; } int segmentedBlock[256] = { 0 }; if (bsc_init(props.paramFeatures()) != LIBBSC_NO_ERROR) { fprintf(stderr, "\nInternal program error, please contact the author!\n"); return E_FAIL; } #ifdef LIBBSC_OPENMP int numThreads = 1; if (paramEnableParallelProcessing) { numThreads = omp_get_max_threads(); if (numThreads <= nBlocks) paramEnableMultiThreading = 0; if (numThreads >= nBlocks) numThreads = nBlocks; } #endif int segmentationStart = 0, segmentationEnd = 0; #ifdef LIBBSC_OPENMP #pragma omp parallel num_threads(numThreads) if(numThreads > 1) #endif { unsigned char * buffer = (unsigned char *)bsc_malloc(props.paramBlockSize + LIBBSC_HEADER_SIZE); unsigned char * buffer2 = (unsigned char *)bsc_malloc(props.paramBlockSize + LIBBSC_HEADER_SIZE); if (buffer == NULL || buffer2 == NULL) return E_OUTOFMEMORY; BSC_FILEOFFSET blockOffset = 0; BSC_FILEOFFSET prevBlockOffset = 0; BSC_FILEOFFSET readPos = 0; //HRESULT res = SZ_OK; for (;;) { UInt32 dataSize = 0; #ifdef LIBBSC_OPENMP #pragma omp critical(input) #endif { { UInt32 currentBlockSize = props.paramBlockSize; if (props.paramEnableSegmentation()) { if (segmentationEnd - segmentationStart > 1) currentBlockSize = segmentedBlock[segmentationStart]; } if (blockOffset < readPos) { UInt32 moveOffset = (UInt32)(blockOffset - prevBlockOffset); UInt32 remained = (UInt32)(readPos - blockOffset); memmove(buffer, buffer + moveOffset, remained); dataSize = (UInt32)min(props.paramBlockSize - remained, nReduceSize - readPos); memcpy(buffer + remained, data + readPos, dataSize); readPos += dataSize; dataSize += remained; } else { dataSize = (UInt32)min(props.paramBlockSize, nReduceSize - readPos); memcpy(buffer, data + readPos, dataSize); readPos += dataSize; } if (dataSize <= 0) break; if (props.paramEnableSegmentation()) { if (dataSize > currentBlockSize) dataSize = currentBlockSize; } if (props.paramEnableSegmentation()) { bool bSegmentation = false; if (segmentationStart == segmentationEnd) bSegmentation = true; if ((segmentationEnd - segmentationStart == 1) && ((int)dataSize != segmentedBlock[segmentationStart])) bSegmentation = true; if (bSegmentation) { segmentationStart = 0; segmentationEnd = bsc_detect_segments(buffer, dataSize, segmentedBlock, 256, props.paramFeatures()); if (segmentationEnd <= LIBBSC_NO_ERROR) { switch (segmentationEnd) { case LIBBSC_NOT_ENOUGH_MEMORY: return E_OUTOFMEMORY; default: return E_FAIL; } } } int newDataSize = segmentedBlock[segmentationStart++]; if ((int)dataSize != newDataSize) { // BSC_FILEOFFSET pos = blockOffset + newDataSize; // BSC_FSEEK(fInput, pos, SEEK_SET); dataSize = newDataSize; } } } } if (dataSize == 0) break; memcpy(buffer2, buffer, dataSize); signed char recordSize = 1; if (props.paramEnableReordering()) { recordSize = (signed char)bsc_detect_recordsize(buffer2, dataSize, props.paramFeatures()); if ((int)recordSize < LIBBSC_NO_ERROR) { switch (recordSize) { case LIBBSC_NOT_ENOUGH_MEMORY: return E_OUTOFMEMORY; default: return E_FAIL; } } if (recordSize > 1) { int result = bsc_reorder_forward(buffer2, dataSize, recordSize, props.paramFeatures()); if (result != LIBBSC_NO_ERROR) { switch (result) { case LIBBSC_NOT_ENOUGH_MEMORY: return E_OUTOFMEMORY; default: return E_FAIL; } } } } signed char sortingContexts = props.paramSortingContexts; if ((int)props.paramSortingContexts == LIBBSC_CONTEXTS_AUTODETECT) { sortingContexts = (signed char)bsc_detect_contextsorder(buffer2, dataSize, props.paramFeatures()); if (sortingContexts < LIBBSC_NO_ERROR) { switch (sortingContexts) { case LIBBSC_NOT_ENOUGH_MEMORY: return E_OUTOFMEMORY; default: return E_FAIL; } } } if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING) { int result = bsc_reverse_block(buffer2, dataSize, props.paramFeatures()); if (result != LIBBSC_NO_ERROR) return E_FAIL; } int blockSize = bsc_compress(buffer2, buffer2, dataSize, props.paramLZPHashSize, props.paramLZPMinLen, props.paramBlockSorter, props.paramCoder(), props.paramFeatures()); if (blockSize == LIBBSC_NOT_COMPRESSIBLE) { #ifdef LIBBSC_OPENMP #pragma omp critical(input) #endif { sortingContexts = LIBBSC_CONTEXTS_FOLLOWING; recordSize = 1; // BSC_FILEOFFSET pos = blockOffset; // BSC_FSEEK(fInput, blockOffset, SEEK_SET); // RINOK(read(inStream, buffer, dataSize, &dataSize)); // _inSizeProcessed = blockOffset + dataSize; // BSC_FSEEK(fInput, pos, SEEK_SET); // memcpy(buffer2, buffer, dataSize); } blockSize = bsc_store(buffer, buffer2, dataSize, props.paramFeatures()); } if (blockSize < LIBBSC_NO_ERROR) { #ifdef LIBBSC_OPENMP #pragma omp critical(print) #endif { switch (blockSize) { case LIBBSC_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break; case LIBBSC_NOT_SUPPORTED: fprintf(stderr, "\nSpecified compression method is not supported on this platform!\n"); break; case LIBBSC_GPU_ERROR: fprintf(stderr, "\nGeneral GPU failure! Please check README file for more information.\n"); break; case LIBBSC_GPU_NOT_SUPPORTED: fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break; case LIBBSC_GPU_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break; default: fprintf(stderr, "\nInternal program error, please contact the author!\n"); } return E_FAIL; } } #ifdef LIBBSC_OPENMP #pragma omp critical(output) #endif { Byte next = 1; RINOK(WriteStream(outStream, &next, 1)); *outSizeProcessed += 1; // BSC_BLOCK_HEADER header = { blockOffset, recordSize, sortingContexts }; // RINOK(outStream->Write(&header, sizeof(BSC_BLOCK_HEADER), &processedSize)); write_int(outStream, blockOffset, outSizeProcessed); write_int(outStream, recordSize, outSizeProcessed); write_int(outStream, sortingContexts, outSizeProcessed); RINOK(WriteStream(outStream, buffer2, blockSize)); *outSizeProcessed += blockSize; prevBlockOffset = blockOffset; blockOffset = blockOffset + dataSize; } } bsc_free(buffer2); bsc_free(buffer); } { Byte next = 0; RINOK(WriteStream(outStream, &next, 1)); *outSizeProcessed += 1; } return S_OK; }
void Compression(char * argv[]) { if (!paramEnableLZP) { paramLZPHashSize = 0; paramLZPMinLen = 0; } FILE * fInput = fopen(argv[2], "rb"); if (fInput == NULL) { fprintf(stderr, "Can't open input file: %s!\n", argv[2]); exit(1); } FILE * fOutput = fopen(argv[3], "wb"); if (fOutput == NULL) { fprintf(stderr, "Can't create output file: %s!\n", argv[3]); exit(1); } if (BSC_FSEEK(fInput, 0, SEEK_END)) { fprintf(stderr, "IO error on file: %s!\n", argv[2]); exit(1); } BSC_FILEOFFSET fileSize = BSC_FTELL(fInput); if (fileSize < 0) { fprintf(stderr, "IO error on file: %s!\n", argv[2]); exit(1); } if (BSC_FSEEK(fInput, 0, SEEK_SET)) { fprintf(stderr, "IO error on file: %s!\n", argv[2]); exit(1); } if (paramBlockSize > fileSize) { paramBlockSize = (int)fileSize; } if (fwrite(bscFileSign, sizeof(bscFileSign), 1, fOutput) != 1) { fprintf(stderr, "IO error on file: %s!\n", argv[3]); exit(1); } int nBlocks = paramBlockSize > 0 ? (int)((fileSize + paramBlockSize - 1) / paramBlockSize) : 0; if (fwrite(&nBlocks, sizeof(nBlocks), 1, fOutput) != 1) { fprintf(stderr, "IO error on file: %s!\n", argv[3]); exit(1); } double startTime = BSC_CLOCK(); #ifdef LIBBSC_OPENMP int numThreads = 1; if (paramEnableParallelProcessing) { numThreads = omp_get_max_threads(); if (numThreads > nBlocks) { numThreads = nBlocks; } } #endif int segmentationStart = 0, segmentationEnd = 0; #ifdef LIBBSC_OPENMP #pragma omp parallel num_threads(numThreads) if(numThreads > 1) #endif { unsigned char * buffer = (unsigned char *)bsc_malloc(paramBlockSize + LIBBSC_HEADER_SIZE); if (buffer == NULL) { #ifdef LIBBSC_OPENMP #pragma omp critical(print) #endif { fprintf(stderr, "Not enough memory! Please check README file for more information.\n"); exit(2); } } while (true) { BSC_FILEOFFSET blockOffset = 0; int dataSize = 0; #ifdef LIBBSC_OPENMP #pragma omp critical(input) #endif { if ((feof(fInput) == 0) && (BSC_FTELL(fInput) != fileSize)) { #ifdef LIBBSC_OPENMP #pragma omp master #endif { double progress = (100.0 * (double)BSC_FTELL(fInput)) / fileSize; fprintf(stdout, "\rCompressing %.55s(%02d%%)", argv[2], (int)progress); fflush(stdout); } blockOffset = BSC_FTELL(fInput); int currentBlockSize = paramBlockSize; if (paramEnableSegmentation) { if (segmentationEnd - segmentationStart > 1) currentBlockSize = segmentedBlock[segmentationStart]; } dataSize = (int)fread(buffer, 1, currentBlockSize, fInput); if (dataSize <= 0) { fprintf(stderr, "\nIO error on file: %s!\n", argv[2]); exit(1); } if (paramEnableSegmentation) { bool bSegmentation = false; if (segmentationStart == segmentationEnd) bSegmentation = true; if ((segmentationEnd - segmentationStart == 1) && (dataSize != segmentedBlock[segmentationStart])) bSegmentation = true; if (bSegmentation) { segmentationStart = 0; segmentationEnd = bsc_detect_segments(buffer, dataSize, segmentedBlock, 256, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE); if (segmentationEnd <= LIBBSC_NO_ERROR) { switch (segmentationEnd) { case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break; default : fprintf(stderr, "\nInternal program error, please contact the author!\n"); } exit(2); } } int newDataSize = segmentedBlock[segmentationStart++]; if (dataSize != newDataSize) { BSC_FILEOFFSET pos = BSC_FTELL(fInput) - dataSize + newDataSize; BSC_FSEEK(fInput, pos, SEEK_SET); dataSize = newDataSize; } } } } if (dataSize == 0) break; char recordSize = 1; if (paramEnableReordering) { recordSize = bsc_detect_recordsize(buffer, dataSize, LIBBSC_FEATURE_FASTMODE); if (recordSize < LIBBSC_NO_ERROR) { #ifdef LIBBSC_OPENMP #pragma omp critical(print) #endif { switch (recordSize) { case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break; default : fprintf(stderr, "\nInternal program error, please contact the author!\n"); } exit(2); } } if (recordSize > 1) { int result = bsc_reorder_forward(buffer, dataSize, recordSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE); if (result != LIBBSC_NO_ERROR) { #ifdef LIBBSC_OPENMP #pragma omp critical(print) #endif { switch (result) { case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break; default : fprintf(stderr, "\nInternal program error, please contact the author!\n"); } exit(2); } } } } char sortingContexts = paramSortingContexts; if (paramSortingContexts == LIBBSC_CONTEXTS_AUTODETECT) { sortingContexts = bsc_detect_contextsorder(buffer, dataSize, LIBBSC_FEATURE_FASTMODE); if (sortingContexts < LIBBSC_NO_ERROR) { #ifdef LIBBSC_OPENMP #pragma omp critical(print) #endif { switch (sortingContexts) { case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory!\n"); break; default : fprintf(stderr, "\nInternal program error, please contact the author!\n"); } exit(2); } } } if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING) { int result = bsc_reverse_block(buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE); if (result != LIBBSC_NO_ERROR) { #ifdef LIBBSC_OPENMP #pragma omp critical(print) #endif { fprintf(stderr, "\nInternal program error, please contact the author!\n"); exit(2); } } } int features = (paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE) | (paramEnableFastMode ? LIBBSC_FEATURE_FASTMODE : LIBBSC_FEATURE_NONE) | (paramEnableCUDA ? LIBBSC_FEATURE_CUDA : LIBBSC_FEATURE_NONE) ; int blockSize = bsc_compress(buffer, buffer, dataSize, paramLZPHashSize, paramLZPMinLen, paramBlockSorter, features); if (blockSize == LIBBSC_NOT_COMPRESSIBLE) { #ifdef LIBBSC_OPENMP #pragma omp critical(input) #endif { sortingContexts = LIBBSC_CONTEXTS_FOLLOWING; recordSize = 1; BSC_FILEOFFSET pos = BSC_FTELL(fInput); { BSC_FSEEK(fInput, blockOffset, SEEK_SET); if (dataSize != (int)fread(buffer, 1, dataSize, fInput)) { fprintf(stderr, "\nInternal program error, please contact the author!\n"); exit(2); } } BSC_FSEEK(fInput, pos, SEEK_SET); } blockSize = bsc_store(buffer, buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE); } if (blockSize < LIBBSC_NO_ERROR) { #ifdef LIBBSC_OPENMP #pragma omp critical(print) #endif { switch (blockSize) { case LIBBSC_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break; case LIBBSC_NOT_SUPPORTED : fprintf(stderr, "\nSpecified compression method is not supported on this platform!\n"); break; case LIBBSC_GPU_ERROR : fprintf(stderr, "\nGeneral GPU failure, please contact the author!\n"); break; case LIBBSC_GPU_NOT_SUPPORTED : fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break; case LIBBSC_GPU_NOT_ENOUGH_MEMORY : fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break; default : fprintf(stderr, "\nInternal program error, please contact the author!\n"); } exit(2); } } #ifdef LIBBSC_OPENMP #pragma omp critical(output) #endif { BSC_BLOCK_HEADER header = {blockOffset, recordSize, sortingContexts}; if (fwrite(&header, sizeof(BSC_BLOCK_HEADER), 1, fOutput) != 1) { fprintf(stderr, "\nIO error on file: %s!\n", argv[3]); exit(1); } if ((int)fwrite(buffer, 1, blockSize, fOutput) != blockSize) { fprintf(stderr, "\nIO error on file: %s!\n", argv[3]); exit(1); } } } bsc_free(buffer); } fprintf(stdout, "\r%.55s compressed %.0f into %.0f in %.3f seconds.\n", argv[2], (double)fileSize, (double)BSC_FTELL(fOutput), BSC_CLOCK() - startTime); fclose(fInput); fclose(fOutput); }