コード例 #1
0
ファイル: PackjpgCoder.cpp プロジェクト: StephanBusch/EcoZip
HRESULT uncompress_replacement(ISequentialInStream * inStream, ISequentialOutStream *outStream, UInt64 *inSizeProcessed, UInt64 *outSizeProcessed)
{
  UInt32 _inSize;
  CLibbscProps props;

  // C=f
  props.paramSortingContexts = LIBBSC_CONTEXTS_FOLLOWING;

  // E=2
  // LIBBSC_CODER_QLFC_ADAPTIVE
  props.features |= PARAM_CODER_QLFC_ADAPTIVE;

  // B=20
  props.paramBlockSize = 20 * 1024 * 1024;

  if (bsc_init(props.paramFeatures()) != LIBBSC_NO_ERROR)
  {
    //fprintf(stderr, "\nInternal program error, please contact the author!\n");
    return E_FAIL;
  }

#ifdef LIBBSC_OPENMP

  int numThreads = 1;
  if (paramEnableParallelProcessing)
  {
    numThreads = omp_get_max_threads();
    if (numThreads <= nBlocks) paramEnableMultiThreading = 0;
    if (numThreads >= nBlocks) numThreads = nBlocks;
  }

#pragma omp parallel num_threads(numThreads) if(numThreads > 1)
#endif
  {
    int bufferSize = -1; unsigned char * buffer = NULL;

    for (;;)
    {
      BSC_FILEOFFSET  blockOffset = 0;

      signed char     sortingContexts = 0;
      signed char     recordSize = 0;
      int             blockSize = 0;
      int             dataSize = 0;

#ifdef LIBBSC_OPENMP
#pragma omp critical(input)
#endif
      {
        Byte next = 0;
        if (!prepare_ex_data(inStream, 1))
          break;
        _inSize = 1;
        next = ex_data[ex_next_offset++];
        *inSizeProcessed += 1;
        if (next == 0)
          break;

        BSC_BLOCK_HEADER header = { 0, 0, 0 };
        header.blockOffset = read_int(inStream, inSizeProcessed);
        header.recordSize = (char)read_int(inStream, inSizeProcessed);
        header.sortingContexts = (char)read_int(inStream, inSizeProcessed);

        recordSize = header.recordSize;
        if (recordSize < 1)
        {
          fprintf(stderr, "\nThis is not bsc archive or invalid compression method!\n");
          return E_FAIL;
        }

        sortingContexts = header.sortingContexts;
        if ((sortingContexts != LIBBSC_CONTEXTS_FOLLOWING) && (sortingContexts != LIBBSC_CONTEXTS_PRECEDING))
        {
          fprintf(stderr, "\nThis is not bsc archive or invalid compression method!\n");
          return E_FAIL;
        }

        blockOffset = (BSC_FILEOFFSET)header.blockOffset;

        unsigned char *bscBlockHeader;

        _inSize = LIBBSC_HEADER_SIZE;
        if (!prepare_ex_data(inStream, _inSize))
          break;
        bscBlockHeader = ex_data + ex_next_offset;
        ex_next_offset += _inSize;
        *inSizeProcessed += _inSize;

        if (bsc_block_info(bscBlockHeader, LIBBSC_HEADER_SIZE, &blockSize, &dataSize, props.paramFeatures()) != LIBBSC_NO_ERROR)
        {
          fprintf(stderr, "\nThis is not bsc archive or invalid compression method!\n");
          return E_FAIL;
        }

        if ((blockSize > bufferSize) || (dataSize > bufferSize))
        {
          if (blockSize > bufferSize) bufferSize = blockSize;
          if (dataSize > bufferSize) bufferSize = dataSize;

          if (buffer != NULL) bsc_free(buffer); buffer = (unsigned char *)bsc_malloc(bufferSize);
        }

        if (buffer == NULL)
        {
          fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n");
          return E_FAIL;
        }

        memcpy(buffer, bscBlockHeader, LIBBSC_HEADER_SIZE);

        _inSize = blockSize - LIBBSC_HEADER_SIZE;
        if (!prepare_ex_data(inStream, _inSize))
          break;
        memcpy(buffer + LIBBSC_HEADER_SIZE, ex_data + ex_next_offset, _inSize);
        ex_next_offset += _inSize;
        *inSizeProcessed += _inSize;
      }


      if (dataSize == 0) break;

      int result = bsc_decompress(buffer, blockSize, buffer, dataSize, props.paramFeatures());
      if (result < LIBBSC_NO_ERROR)
      {
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
        {
          switch (result)
          {
          case LIBBSC_DATA_CORRUPT: fprintf(stderr, "\nThe compressed data is corrupted!\n"); break;
          case LIBBSC_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
          case LIBBSC_GPU_ERROR: fprintf(stderr, "\nGeneral GPU failure! Please check README file for more information.\n"); break;
          case LIBBSC_GPU_NOT_SUPPORTED: fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break;
          case LIBBSC_GPU_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break;

          default: fprintf(stderr, "\nInternal program error, please contact the author!\n");
          }
          return E_FAIL;
        }
      }

      if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING)
      {
        result = bsc_reverse_block(buffer, dataSize, props.paramFeatures());
        if (result != LIBBSC_NO_ERROR)
        {
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
          {
            fprintf(stderr, "\nInternal program error, please contact the author!\n");
            return E_FAIL;
          }
        }
      }

      if (recordSize > 1)
      {
        result = bsc_reorder_reverse(buffer, dataSize, recordSize, props.paramFeatures());
        if (result != LIBBSC_NO_ERROR)
        {
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
          {
            switch (result)
            {
            case LIBBSC_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
            default: fprintf(stderr, "\nInternal program error, please contact the author!\n");
            }
            return E_FAIL;
          }
        }
      }

#ifdef LIBBSC_OPENMP
#pragma omp critical(output)
#endif
      {
// 				if (BSC_FSEEK(fOutput, blockOffset, SEEK_SET))
// 				{
// 					fprintf(stderr, "\nIO error on file: %s!\n", argv[3]);
//          return E_FAIL;
// 				}

        UInt32 processedSize = 0;
        RINOK(WriteStream(outStream, buffer, dataSize));
        *outSizeProcessed += dataSize;
      }
    }
  }
  return S_OK;
}
コード例 #2
0
ファイル: bsc.cpp プロジェクト: seanjensengrey/libbsc
void Decompression(char * argv[])
{
    FILE * fInput = fopen(argv[2], "rb");
    if (fInput == NULL)
    {
        fprintf(stderr, "Can't open input file: %s!\n", argv[2]);
        exit(1);
    }

    FILE * fOutput = fopen(argv[3], "wb");
    if (fOutput == NULL)
    {
        fprintf(stderr, "Can't create output file: %s!\n", argv[3]);
        exit(1);
    }

    if (BSC_FSEEK(fInput, 0, SEEK_END))
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[2]);
        exit(1);
    }

    BSC_FILEOFFSET fileSize = BSC_FTELL(fInput);
    if (fileSize < 0)
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[2]);
        exit(1);
    }

    if (BSC_FSEEK(fInput, 0, SEEK_SET))
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[2]);
        exit(1);
    }

    unsigned char inputFileSign[sizeof(bscFileSign)];

    if (fread(inputFileSign, sizeof(bscFileSign), 1, fInput) != 1)
    {
        fprintf(stderr, "This is not bsc archive!\n");
        exit(1);
    }

    if (memcmp(inputFileSign, bscFileSign, sizeof(bscFileSign)) != 0)
    {
        fprintf(stderr, "This is not bsc archive or invalid compression method!\n");
        exit(2);
    }

    int nBlocks = 0;
    if (fread(&nBlocks, sizeof(nBlocks), 1, fInput) != 1)
    {
        fprintf(stderr, "This is not bsc archive!\n");
        exit(1);
    }

    double startTime = BSC_CLOCK();

#ifdef LIBBSC_OPENMP

    int numThreads = 1;
    if (paramEnableParallelProcessing)
    {
        numThreads = omp_get_max_threads();
        if (numThreads > nBlocks)
        {
            numThreads = nBlocks;
        }
    }

    #pragma omp parallel num_threads(numThreads) if(numThreads > 1)
#endif
    {
        int bufferSize = -1; unsigned char * buffer = NULL;

        while (true)
        {
            BSC_FILEOFFSET  blockOffset     = 0;

            char            sortingContexts = 0;
            char            recordSize      = 0;
            int             blockSize       = 0;
            int             dataSize        = 0;

#ifdef LIBBSC_OPENMP
            #pragma omp critical(input)
#endif
            {
                if ((feof(fInput) == 0) && (BSC_FTELL(fInput) != fileSize))
                {
#ifdef LIBBSC_OPENMP
                    #pragma omp master
#endif
                    {
                        double progress = (100.0 * (double)BSC_FTELL(fInput)) / fileSize;
                        fprintf(stdout, "\rDecompressing %.55s(%02d%%)", argv[2], (int)progress);
                        fflush(stdout);
                    }

                    BSC_BLOCK_HEADER header = {0, 0, 0};
                    if (fread(&header, sizeof(BSC_BLOCK_HEADER), 1, fInput) != 1)
                    {
                        fprintf(stderr, "\nUnexpected end of file: %s!\n", argv[2]);
                        exit(1);
                    }

                    recordSize = header.recordSize;
                    if (recordSize < 1)
                    {
                        fprintf(stderr, "\nThis is not bsc archive or invalid compression method!\n");
                        exit(2);
                    }

                    sortingContexts = header.sortingContexts;
                    if ((sortingContexts != LIBBSC_CONTEXTS_FOLLOWING) && (sortingContexts != LIBBSC_CONTEXTS_PRECEDING))
                    {
                        fprintf(stderr, "\nThis is not bsc archive or invalid compression method!\n");
                        exit(2);
                    }

                    blockOffset = (BSC_FILEOFFSET)header.blockOffset;

                    unsigned char bscBlockHeader[LIBBSC_HEADER_SIZE];

                    if (fread(bscBlockHeader, LIBBSC_HEADER_SIZE, 1, fInput) != 1)
                    {
                        fprintf(stderr, "\nUnexpected end of file: %s!\n", argv[2]);
                        exit(1);
                    }

                    if (bsc_block_info(bscBlockHeader, LIBBSC_HEADER_SIZE, &blockSize, &dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE) != LIBBSC_NO_ERROR)
                    {
                        fprintf(stderr, "\nThis is not bsc archive or invalid compression method!\n");
                        exit(2);
                    }

                    if ((blockSize > bufferSize) || (dataSize > bufferSize))
                    {
                        if (blockSize > bufferSize) bufferSize = blockSize;
                        if (dataSize  > bufferSize) bufferSize = dataSize;

                        if (buffer != NULL) bsc_free(buffer); buffer = (unsigned char *)bsc_malloc(bufferSize);
                    }

                    if (buffer == NULL)
                    {
                        fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n");
                        exit(2);
                    }

                    memcpy(buffer, bscBlockHeader, LIBBSC_HEADER_SIZE);

                    if (fread(buffer + LIBBSC_HEADER_SIZE, blockSize - LIBBSC_HEADER_SIZE, 1, fInput) != 1)
                    {
                        fprintf(stderr, "\nUnexpected end of file: %s!\n", argv[2]);
                        exit(1);
                    }
                }
            }

            if (dataSize == 0) break;

            int result = bsc_decompress(buffer, blockSize, buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
            if (result < LIBBSC_NO_ERROR)
            {
#ifdef LIBBSC_OPENMP
                #pragma omp critical(print)
#endif
                {
                    switch (result)
                    {
                        case LIBBSC_DATA_CORRUPT            : fprintf(stderr, "\nThe compressed data is corrupted!\n"); break;
                        case LIBBSC_NOT_ENOUGH_MEMORY       : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
                        case LIBBSC_GPU_ERROR               : fprintf(stderr, "\nGeneral GPU failure, please contact the author!\n"); break;
                        case LIBBSC_GPU_NOT_SUPPORTED       : fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break;
                        case LIBBSC_GPU_NOT_ENOUGH_MEMORY   : fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break;

                        default                             : fprintf(stderr, "\nInternal program error, please contact the author!\n");
                    }
                    exit(2);
                }
            }

            if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING)
            {
                result = bsc_reverse_block(buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
                if (result != LIBBSC_NO_ERROR)
                {
#ifdef LIBBSC_OPENMP
                    #pragma omp critical(print)
#endif
                    {
                        fprintf(stderr, "\nInternal program error, please contact the author!\n");
                        exit(2);
                    }
                }
            }

            if (recordSize > 1)
            {
                result = bsc_reorder_reverse(buffer, dataSize, recordSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
                if (result != LIBBSC_NO_ERROR)
                {
#ifdef LIBBSC_OPENMP
                    #pragma omp critical(print)
#endif
                    {
                        switch (result)
                        {
                            case LIBBSC_NOT_ENOUGH_MEMORY   : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
                            default                         : fprintf(stderr, "\nInternal program error, please contact the author!\n");
                        }
                        exit(2);
                    }
                }
            }

#ifdef LIBBSC_OPENMP
            #pragma omp critical(output)
#endif
            {
                if (BSC_FSEEK(fOutput, blockOffset, SEEK_SET))
                {
                    fprintf(stderr, "\nIO error on file: %s!\n", argv[3]);
                    exit(1);
                }

                if ((int)fwrite(buffer, 1, dataSize, fOutput) != dataSize)
                {
                    fprintf(stderr, "\nIO error on file: %s!\n", argv[3]);
                    exit(1);
                }
            }
        }

        if (buffer != NULL) bsc_free(buffer);
    }

    if (BSC_FSEEK(fOutput, 0, SEEK_END))
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[3]);
        exit(1);
    }

    fprintf(stdout, "\r%.55s decompressed %.0f into %.0f in %.3f seconds.\n", argv[2], (double)fileSize, (double)BSC_FTELL(fOutput), BSC_CLOCK() - startTime);

    fclose(fInput); fclose(fOutput);
}
コード例 #3
0
ファイル: PackjpgCoder.cpp プロジェクト: StephanBusch/EcoZip
HRESULT compress_replacement(ISequentialOutStream *outStream, uint8_t *data, size_t len, UInt64 *outSizeProcessed)
{
  CLibbscProps props;

  // C=f
  props.paramSortingContexts = LIBBSC_CONTEXTS_FOLLOWING;

  // E=2
  // LIBBSC_CODER_QLFC_ADAPTIVE
  props.features |= PARAM_CODER_QLFC_ADAPTIVE;

  // B=20
  props.paramBlockSize = 20 * 1024 * 1024;

  UInt32 nReduceSize = (UInt32)len;

  UInt32 kMinBlockSize = 1 * 1024 * 1024;

  if (nReduceSize < props.paramBlockSize) {
    if (nReduceSize < kMinBlockSize)
      props.paramBlockSize = kMinBlockSize;
    else
      props.paramBlockSize = nReduceSize;
  }

  int segmentedBlock[256] = { 0 };

  if (bsc_init(props.paramFeatures()) != LIBBSC_NO_ERROR)
  {
    fprintf(stderr, "\nInternal program error, please contact the author!\n");
    return E_FAIL;
  }

#ifdef LIBBSC_OPENMP
  int numThreads = 1;
  if (paramEnableParallelProcessing)
  {
    numThreads = omp_get_max_threads();
    if (numThreads <= nBlocks) paramEnableMultiThreading = 0;
    if (numThreads >= nBlocks) numThreads = nBlocks;
  }
#endif

  int segmentationStart = 0, segmentationEnd = 0;

#ifdef LIBBSC_OPENMP
#pragma omp parallel num_threads(numThreads) if(numThreads > 1)
#endif
  {
    unsigned char * buffer = (unsigned char *)bsc_malloc(props.paramBlockSize + LIBBSC_HEADER_SIZE);
    unsigned char * buffer2 = (unsigned char *)bsc_malloc(props.paramBlockSize + LIBBSC_HEADER_SIZE);
    if (buffer == NULL || buffer2 == NULL)
      return E_OUTOFMEMORY;

    BSC_FILEOFFSET  blockOffset = 0;
    BSC_FILEOFFSET  prevBlockOffset = 0;
    BSC_FILEOFFSET  readPos = 0;
    //HRESULT res = SZ_OK;
    for (;;)
    {
      UInt32          dataSize = 0;

#ifdef LIBBSC_OPENMP
#pragma omp critical(input)
#endif
      {
        {
          UInt32 currentBlockSize = props.paramBlockSize;
          if (props.paramEnableSegmentation())
          {
            if (segmentationEnd - segmentationStart > 1) currentBlockSize = segmentedBlock[segmentationStart];
          }

          if (blockOffset < readPos) {
            UInt32 moveOffset = (UInt32)(blockOffset - prevBlockOffset);
            UInt32 remained = (UInt32)(readPos - blockOffset);
            memmove(buffer, buffer + moveOffset, remained);
            dataSize = (UInt32)min(props.paramBlockSize - remained, nReduceSize - readPos);
            memcpy(buffer + remained, data + readPos, dataSize);
            readPos += dataSize;
            dataSize += remained;
          }
          else {
            dataSize = (UInt32)min(props.paramBlockSize, nReduceSize - readPos);
            memcpy(buffer, data + readPos, dataSize);
            readPos += dataSize;
          }
          if (dataSize <= 0)
            break;
          if (props.paramEnableSegmentation()) {
            if (dataSize > currentBlockSize)
              dataSize = currentBlockSize;
          }

          if (props.paramEnableSegmentation())
          {
            bool bSegmentation = false;

            if (segmentationStart == segmentationEnd) bSegmentation = true;
            if ((segmentationEnd - segmentationStart == 1) &&
              ((int)dataSize != segmentedBlock[segmentationStart]))
              bSegmentation = true;

            if (bSegmentation)
            {
              segmentationStart = 0; segmentationEnd = bsc_detect_segments(buffer, dataSize, segmentedBlock, 256, props.paramFeatures());
              if (segmentationEnd <= LIBBSC_NO_ERROR)
              {
                switch (segmentationEnd)
                {
                case LIBBSC_NOT_ENOUGH_MEMORY:
                  return E_OUTOFMEMORY;
                default:
                  return E_FAIL;
                }
              }
            }

            int newDataSize = segmentedBlock[segmentationStart++];
            if ((int)dataSize != newDataSize)
            {
              //                   BSC_FILEOFFSET pos = blockOffset + newDataSize;
              //                   BSC_FSEEK(fInput, pos, SEEK_SET);
              dataSize = newDataSize;
            }
          }
        }
      }

      if (dataSize == 0) break;

      memcpy(buffer2, buffer, dataSize);

      signed char recordSize = 1;
      if (props.paramEnableReordering())
      {
        recordSize = (signed char)bsc_detect_recordsize(buffer2, dataSize, props.paramFeatures());
        if ((int)recordSize < LIBBSC_NO_ERROR)
        {
          switch (recordSize)
          {
          case LIBBSC_NOT_ENOUGH_MEMORY:
            return E_OUTOFMEMORY;
          default:
            return E_FAIL;
          }
        }
        if (recordSize > 1)
        {
          int result = bsc_reorder_forward(buffer2, dataSize, recordSize, props.paramFeatures());
          if (result != LIBBSC_NO_ERROR)
          {
            switch (result)
            {
            case LIBBSC_NOT_ENOUGH_MEMORY:
              return E_OUTOFMEMORY;
            default:
              return E_FAIL;
            }
          }
        }
      }

      signed char sortingContexts = props.paramSortingContexts;
      if ((int)props.paramSortingContexts == LIBBSC_CONTEXTS_AUTODETECT)
      {
        sortingContexts = (signed char)bsc_detect_contextsorder(buffer2, dataSize, props.paramFeatures());
        if (sortingContexts < LIBBSC_NO_ERROR)
        {
          switch (sortingContexts)
          {
          case LIBBSC_NOT_ENOUGH_MEMORY:
            return E_OUTOFMEMORY;
          default:
            return E_FAIL;
          }
        }
      }
      if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING)
      {
        int result = bsc_reverse_block(buffer2, dataSize, props.paramFeatures());
        if (result != LIBBSC_NO_ERROR)
          return E_FAIL;
      }

      int blockSize = bsc_compress(buffer2, buffer2, dataSize,
        props.paramLZPHashSize, props.paramLZPMinLen, props.paramBlockSorter,
        props.paramCoder(), props.paramFeatures());
      if (blockSize == LIBBSC_NOT_COMPRESSIBLE)
      {
#ifdef LIBBSC_OPENMP
#pragma omp critical(input)
#endif
        {
          sortingContexts = LIBBSC_CONTEXTS_FOLLOWING; recordSize = 1;

//           BSC_FILEOFFSET pos = blockOffset;
//           BSC_FSEEK(fInput, blockOffset, SEEK_SET);
//           RINOK(read(inStream, buffer, dataSize, &dataSize));
//           _inSizeProcessed = blockOffset + dataSize;
//           BSC_FSEEK(fInput, pos, SEEK_SET);
//           memcpy(buffer2, buffer, dataSize);
        }

        blockSize = bsc_store(buffer, buffer2, dataSize, props.paramFeatures());
      }
      if (blockSize < LIBBSC_NO_ERROR)
      {
#ifdef LIBBSC_OPENMP
#pragma omp critical(print)
#endif
        {
          switch (blockSize)
          {
          case LIBBSC_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
          case LIBBSC_NOT_SUPPORTED: fprintf(stderr, "\nSpecified compression method is not supported on this platform!\n"); break;
          case LIBBSC_GPU_ERROR: fprintf(stderr, "\nGeneral GPU failure! Please check README file for more information.\n"); break;
          case LIBBSC_GPU_NOT_SUPPORTED: fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break;
          case LIBBSC_GPU_NOT_ENOUGH_MEMORY: fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break;

          default: fprintf(stderr, "\nInternal program error, please contact the author!\n");
          }
          return E_FAIL;
        }
      }

#ifdef LIBBSC_OPENMP
#pragma omp critical(output)
#endif
      {
        Byte next = 1;
        RINOK(WriteStream(outStream, &next, 1));
        *outSizeProcessed += 1;

//         BSC_BLOCK_HEADER header = { blockOffset, recordSize, sortingContexts };
//         RINOK(outStream->Write(&header, sizeof(BSC_BLOCK_HEADER), &processedSize));

        write_int(outStream, blockOffset, outSizeProcessed);
        write_int(outStream, recordSize, outSizeProcessed);
        write_int(outStream, sortingContexts, outSizeProcessed);

        RINOK(WriteStream(outStream, buffer2, blockSize));
        *outSizeProcessed += blockSize;

        prevBlockOffset = blockOffset;
        blockOffset = blockOffset + dataSize;
      }
    }

    bsc_free(buffer2);
    bsc_free(buffer);
  }

  {
    Byte next = 0;
    RINOK(WriteStream(outStream, &next, 1));
    *outSizeProcessed += 1;
  }
  return S_OK;
}
コード例 #4
0
ファイル: bsc.cpp プロジェクト: seanjensengrey/libbsc
void Compression(char * argv[])
{
    if (!paramEnableLZP)
    {
        paramLZPHashSize = 0;
        paramLZPMinLen = 0;
    }

    FILE * fInput = fopen(argv[2], "rb");
    if (fInput == NULL)
    {
        fprintf(stderr, "Can't open input file: %s!\n", argv[2]);
        exit(1);
    }

    FILE * fOutput = fopen(argv[3], "wb");
    if (fOutput == NULL)
    {
        fprintf(stderr, "Can't create output file: %s!\n", argv[3]);
        exit(1);
    }

    if (BSC_FSEEK(fInput, 0, SEEK_END))
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[2]);
        exit(1);
    }

    BSC_FILEOFFSET fileSize = BSC_FTELL(fInput);
    if (fileSize < 0)
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[2]);
        exit(1);
    }

    if (BSC_FSEEK(fInput, 0, SEEK_SET))
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[2]);
        exit(1);
    }

    if (paramBlockSize > fileSize)
    {
        paramBlockSize = (int)fileSize;
    }

    if (fwrite(bscFileSign, sizeof(bscFileSign), 1, fOutput) != 1)
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[3]);
        exit(1);
    }

    int nBlocks = paramBlockSize > 0 ? (int)((fileSize + paramBlockSize - 1) / paramBlockSize) : 0;
    if (fwrite(&nBlocks, sizeof(nBlocks), 1, fOutput) != 1)
    {
        fprintf(stderr, "IO error on file: %s!\n", argv[3]);
        exit(1);
    }

    double startTime = BSC_CLOCK();

#ifdef LIBBSC_OPENMP

    int numThreads = 1;
    if (paramEnableParallelProcessing)
    {
        numThreads = omp_get_max_threads();
        if (numThreads > nBlocks)
        {
            numThreads = nBlocks;
        }
    }

#endif

    int segmentationStart = 0, segmentationEnd = 0;

#ifdef LIBBSC_OPENMP
    #pragma omp parallel num_threads(numThreads) if(numThreads > 1)
#endif
    {
        unsigned char * buffer = (unsigned char *)bsc_malloc(paramBlockSize + LIBBSC_HEADER_SIZE);
        if (buffer == NULL)
        {
#ifdef LIBBSC_OPENMP
            #pragma omp critical(print)
#endif
            {

                fprintf(stderr, "Not enough memory! Please check README file for more information.\n");
                exit(2);
            }
        }

        while (true)
        {
            BSC_FILEOFFSET  blockOffset     = 0;
            int             dataSize        = 0;

#ifdef LIBBSC_OPENMP
            #pragma omp critical(input)
#endif
            {
                if ((feof(fInput) == 0) && (BSC_FTELL(fInput) != fileSize))
                {
#ifdef LIBBSC_OPENMP
                    #pragma omp master
#endif
                    {
                        double progress = (100.0 * (double)BSC_FTELL(fInput)) / fileSize;
                        fprintf(stdout, "\rCompressing %.55s(%02d%%)", argv[2], (int)progress);
                        fflush(stdout);
                    }

                    blockOffset = BSC_FTELL(fInput);

                    int currentBlockSize = paramBlockSize;
                    if (paramEnableSegmentation)
                    {
                        if (segmentationEnd - segmentationStart > 1) currentBlockSize = segmentedBlock[segmentationStart];
                    }

                    dataSize = (int)fread(buffer, 1, currentBlockSize, fInput);
                    if (dataSize <= 0)
                    {
                        fprintf(stderr, "\nIO error on file: %s!\n", argv[2]);
                        exit(1);
                    }

                    if (paramEnableSegmentation)
                    {
                        bool bSegmentation = false;

                        if (segmentationStart == segmentationEnd) bSegmentation = true;
                        if ((segmentationEnd - segmentationStart == 1) && (dataSize != segmentedBlock[segmentationStart])) bSegmentation = true;

                        if (bSegmentation)
                        {
                            segmentationStart = 0; segmentationEnd = bsc_detect_segments(buffer, dataSize, segmentedBlock, 256, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
                            if (segmentationEnd <= LIBBSC_NO_ERROR)
                            {
                                switch (segmentationEnd)
                                {
                                    case LIBBSC_NOT_ENOUGH_MEMORY   : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
                                    default                         : fprintf(stderr, "\nInternal program error, please contact the author!\n");
                                }
                                exit(2);
                            }
                        }

                        int newDataSize = segmentedBlock[segmentationStart++];
                        if (dataSize != newDataSize)
                        {
                            BSC_FILEOFFSET pos = BSC_FTELL(fInput) - dataSize + newDataSize;
                            BSC_FSEEK(fInput, pos, SEEK_SET);
                            dataSize = newDataSize;
                        }
                    }
                }
            }

            if (dataSize == 0) break;

            char recordSize = 1;
            if (paramEnableReordering)
            {
                recordSize = bsc_detect_recordsize(buffer, dataSize, LIBBSC_FEATURE_FASTMODE);
                if (recordSize < LIBBSC_NO_ERROR)
                {
#ifdef LIBBSC_OPENMP
                    #pragma omp critical(print)
#endif
                    {
                        switch (recordSize)
                        {
                            case LIBBSC_NOT_ENOUGH_MEMORY   : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
                            default                         : fprintf(stderr, "\nInternal program error, please contact the author!\n");
                        }
                        exit(2);
                    }
                }
                if (recordSize > 1)
                {
                    int result = bsc_reorder_forward(buffer, dataSize, recordSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
                    if (result != LIBBSC_NO_ERROR)
                    {
#ifdef LIBBSC_OPENMP
                        #pragma omp critical(print)
#endif
                        {
                            switch (result)
                            {
                                case LIBBSC_NOT_ENOUGH_MEMORY   : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
                                default                         : fprintf(stderr, "\nInternal program error, please contact the author!\n");
                            }
                            exit(2);
                        }
                    }
                }
            }

            char sortingContexts = paramSortingContexts;
            if (paramSortingContexts == LIBBSC_CONTEXTS_AUTODETECT)
            {
                sortingContexts = bsc_detect_contextsorder(buffer, dataSize, LIBBSC_FEATURE_FASTMODE);
                if (sortingContexts < LIBBSC_NO_ERROR)
                {
#ifdef LIBBSC_OPENMP
                    #pragma omp critical(print)
#endif
                    {
                        switch (sortingContexts)
                        {
                            case LIBBSC_NOT_ENOUGH_MEMORY   : fprintf(stderr, "\nNot enough memory!\n"); break;
                            default                         : fprintf(stderr, "\nInternal program error, please contact the author!\n");
                        }
                        exit(2);
                    }
                }
            }
            if (sortingContexts == LIBBSC_CONTEXTS_PRECEDING)
            {
                int result = bsc_reverse_block(buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
                if (result != LIBBSC_NO_ERROR)
                {
#ifdef LIBBSC_OPENMP
                    #pragma omp critical(print)
#endif
                    {
                        fprintf(stderr, "\nInternal program error, please contact the author!\n");
                        exit(2);
                    }
                }
            }

            int features =
                (paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE) |
                (paramEnableFastMode       ? LIBBSC_FEATURE_FASTMODE       : LIBBSC_FEATURE_NONE) |
                (paramEnableCUDA           ? LIBBSC_FEATURE_CUDA           : LIBBSC_FEATURE_NONE)
            ;

            int blockSize = bsc_compress(buffer, buffer, dataSize, paramLZPHashSize, paramLZPMinLen, paramBlockSorter, features);
            if (blockSize == LIBBSC_NOT_COMPRESSIBLE)
            {
#ifdef LIBBSC_OPENMP
                #pragma omp critical(input)
#endif
                {
                    sortingContexts = LIBBSC_CONTEXTS_FOLLOWING; recordSize = 1;

                    BSC_FILEOFFSET pos = BSC_FTELL(fInput);
                    {
                        BSC_FSEEK(fInput, blockOffset, SEEK_SET);
                        if (dataSize != (int)fread(buffer, 1, dataSize, fInput))
                        {
                            fprintf(stderr, "\nInternal program error, please contact the author!\n");
                            exit(2);
                        }
                    }
                    BSC_FSEEK(fInput, pos, SEEK_SET);
                }

                blockSize = bsc_store(buffer, buffer, dataSize, paramEnableMultiThreading ? LIBBSC_FEATURE_MULTITHREADING : LIBBSC_FEATURE_NONE);
            }
            if (blockSize < LIBBSC_NO_ERROR)
            {
#ifdef LIBBSC_OPENMP
                #pragma omp critical(print)
#endif
                {
                    switch (blockSize)
                    {
                        case LIBBSC_NOT_ENOUGH_MEMORY       : fprintf(stderr, "\nNot enough memory! Please check README file for more information.\n"); break;
                        case LIBBSC_NOT_SUPPORTED           : fprintf(stderr, "\nSpecified compression method is not supported on this platform!\n"); break;
                        case LIBBSC_GPU_ERROR               : fprintf(stderr, "\nGeneral GPU failure, please contact the author!\n"); break;
                        case LIBBSC_GPU_NOT_SUPPORTED       : fprintf(stderr, "\nYour GPU is not supported! Please check README file for more information.\n"); break;
                        case LIBBSC_GPU_NOT_ENOUGH_MEMORY   : fprintf(stderr, "\nNot enough GPU memory! Please check README file for more information.\n"); break;

                        default                             : fprintf(stderr, "\nInternal program error, please contact the author!\n");
                    }
                    exit(2);
                }
            }

#ifdef LIBBSC_OPENMP
            #pragma omp critical(output)
#endif
            {
                BSC_BLOCK_HEADER header = {blockOffset, recordSize, sortingContexts};

                if (fwrite(&header, sizeof(BSC_BLOCK_HEADER), 1, fOutput) != 1)
                {
                    fprintf(stderr, "\nIO error on file: %s!\n", argv[3]);
                    exit(1);
                }

                if ((int)fwrite(buffer, 1, blockSize, fOutput) != blockSize)
                {
                    fprintf(stderr, "\nIO error on file: %s!\n", argv[3]);
                    exit(1);
                }
            }

        }

        bsc_free(buffer);
    }

    fprintf(stdout, "\r%.55s compressed %.0f into %.0f in %.3f seconds.\n", argv[2], (double)fileSize, (double)BSC_FTELL(fOutput), BSC_CLOCK() - startTime);

    fclose(fInput); fclose(fOutput);
}