void test_decompress(FILE* outFp, FILE* inpFp)
{
    static char decBuf[DECODE_RING_BUFFER];
    int   decOffset    = 0;
    LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
    LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;

    for(;;) {
        int cmpBytes = 0;
        char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];

        {
            const size_t r0 = read_int32(inpFp, &cmpBytes);
            if(r0 != 1 || cmpBytes <= 0) break;

            const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes);
            if(r1 != (size_t) cmpBytes) break;
        }

        {
            char* const decPtr = &decBuf[decOffset];
            const int decBytes = LZ4_decompress_safe_continue(
                lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
            if(decBytes <= 0) break;
            decOffset += decBytes;
            write_bin(outFp, decPtr, decBytes);

            // Wraparound the ringbuffer offset
            if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0;
        }
    }
}
Exemplo n.º 2
0
void LZ4Segment<T>::_decompress_block_to_bytes(const size_t block_index, std::vector<char>& decompressed_data,
                                               const size_t write_offset) const {
  const auto decompressed_block_size = block_index + 1 != _lz4_blocks.size() ? _block_size : _last_block_size;
  auto& compressed_block = _lz4_blocks[block_index];
  const auto compressed_block_size = compressed_block.size();

  int decompressed_result;
  if (_dictionary.empty()) {
    /**
     * If the dictionary is empty, we either have only a single block or had not enough data for a dictionary.
     * When decoding without a dictionary LZ4 needs a stream decode pointer (which would be used to decode the
     * following blocks).
     * A new decoder needs to be created for every block (in the case of multiple blocks being compressed without a
     * dictionary) since the blocks were compressed independently.
     * This decoder needs to be reset via LZ4_setStreamDecode since LZ4 reuses the previous state instead.
     */
    LZ4_streamDecode_t lz4_stream_decoder;
    auto lz4_stream_decoder_ptr = std::make_unique<LZ4_streamDecode_t>(lz4_stream_decoder);
    const auto reset_decoder_status = LZ4_setStreamDecode(lz4_stream_decoder_ptr.get(), nullptr, 0);
    Assert(reset_decoder_status == 1, "LZ4 decompression failed to reset stream decoder.");

    decompressed_result = LZ4_decompress_safe_continue(
        lz4_stream_decoder_ptr.get(), compressed_block.data(), decompressed_data.data() + write_offset,
        static_cast<int>(compressed_block_size), static_cast<int>(decompressed_block_size));
  } else {
    decompressed_result = LZ4_decompress_safe_usingDict(
        compressed_block.data(), decompressed_data.data() + write_offset, static_cast<int>(compressed_block_size),
        static_cast<int>(decompressed_block_size), _dictionary.data(), static_cast<int>(_dictionary.size()));
  }

  Assert(decompressed_result > 0, "LZ4 stream decompression failed");
  DebugAssert(static_cast<size_t>(decompressed_result) == decompressed_block_size,
              "Decompressed LZ4 block has different size than the initial source data.");
}
Exemplo n.º 3
0
 void ReadBlock()
 {
     char m_lz4buf[LZ4Size];
     uint32_t sz;
     if( fread( &sz, 1, sizeof( sz ), m_file ) == sizeof( sz ) )
     {
         fread( m_lz4buf, 1, sz, m_file );
         m_lastBlock = LZ4_decompress_safe_continue( m_stream, m_lz4buf, m_second, sz, BufSize );
     }
     else
     {
         m_lastBlock = 0;
     }
 }
Exemplo n.º 4
0
Arquivo: lz4.c Projeto: NekBox/NekBox
void lz4_unpack(void * in, const size_t * const compSize, void * const out, int * const size, const int * const ierr) {
  size_t offset=0;
  size_t offset_in=0;
  decBufIndex=0;
  LZ4_resetStream(lz4Stream);
  LZ4_setStreamDecode(lz4StreamDecode,NULL,0);
  for(;;) {
    char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
    int cmpBytes = 0;
    /*printf("offset_in+sizeof(cmpBytes): %d\n",offset_in+sizeof(cmpBytes));*/
    /*printf("compSize: %d\n",*compSize);*/
    int tmp=offset_in+sizeof(cmpBytes);
    /*printf("tmp: %d\n", tmp);*/
    /*printf("compSize: %d\n", *compSize);*/
    if(tmp > (int) *compSize) {
      break;
    }
    memcpy(&cmpBytes, in+offset_in, sizeof(cmpBytes));
    /*printf("cmpBytes: %d\n",cmpBytes);*/
    if(cmpBytes <= 0) {
      /*printf("cmpBytes %d\n", cmpBytes);*/
      break;
    }

    /*printf("lz4 cmpBytes %d\n", cmpBytes);*/
    memcpy(&cmpBuf, in+offset_in+sizeof(cmpBytes), cmpBytes);
    const size_t readCount1=0;
    offset_in=offset_in+sizeof(cmpBytes)+cmpBytes;
    /*printf("new offset_in: %zu\n", offset_in);*/

    {
      char* decPtr = decBuf[decBufIndex];
      int decBytes = LZ4_decompress_safe_continue(
          lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
      if(decBytes <= 0) {
        /*printf("lz4 decBytes: %d\n", decBytes);*/
        break;
      }
      memcpy(out+offset, decPtr, (size_t) decBytes);
      offset=offset+(size_t) decBytes;
    }
    decBufIndex = (decBufIndex + 1) % 2;
  }
  *size=offset;
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
    LZ4_streamDecode_t lz4StreamDecode_body;
    LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;

    char decBuf[2][BLOCK_BYTES];
    int  decBufIndex = 0;

    LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);

    for(;;) {
        char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
        int  cmpBytes = 0;

        {
            const size_t readCount0 = read_int(inpFp, &cmpBytes);
            if(readCount0 != 1 || cmpBytes <= 0) {
                break;
            }

            const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes);
            if(readCount1 != (size_t) cmpBytes) {
                break;
            }
        }

        {
            char* const decPtr = decBuf[decBufIndex];
            const int decBytes = LZ4_decompress_safe_continue(
                lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
            if(decBytes <= 0) {
                break;
            }
            write_bin(outFp, decPtr, (size_t) decBytes);
        }

        decBufIndex = (decBufIndex + 1) % 2;
    }
}
Exemplo n.º 6
0
bool LZ4Decompressor::FillPage0()
{
  // swap pages
  std::swap(m_Page[0], m_Page[1]);

  int32_t compSize = 0;

  bool success = true;

  success &= m_Read->Read(compSize);
  success &= m_Read->Read(m_CompressBuffer, compSize);

  if(!success)
  {
    FreeAlignedBuffer(m_Page[0]);
    FreeAlignedBuffer(m_Page[1]);
    FreeAlignedBuffer(m_CompressBuffer);
    m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
    return false;
  }

  int32_t decompSize = LZ4_decompress_safe_continue(&m_LZ4Decomp, (const char *)m_CompressBuffer,
                                                    (char *)m_Page[0], compSize, lz4BlockSize);

  if(decompSize < 0)
  {
    RDCERR("Error decompressing: %i", decompSize);
    FreeAlignedBuffer(m_Page[0]);
    FreeAlignedBuffer(m_Page[1]);
    FreeAlignedBuffer(m_CompressBuffer);
    m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
    return false;
  }

  m_PageOffset = 0;
  m_PageLength = decompSize;

  return success;
}
void decompressFile(FILE* outFp, FILE* inpFp)
{
    LZ4_streamDecode_t lz4StreamDecode_body;
    LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;

    char decBuf[2][BLOCK_BYTES];
    int  decBufIndex = 0;

    LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);

    while(1)
    {
        char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
        int  cmpBytes = 0;

        {
            const size_t readCount0 = intRead(inpFp, &cmpBytes);
            if((readCount0 != 1) || (cmpBytes <= 0))
            {
                break;
            }
            const size_t readCount1 = binRead(inpFp, cmpBuf, (size_t) cmpBytes);
            if(readCount1 != (size_t) cmpBytes)
            {
                break;
            }
        }
        {
            char* const decPtr = decBuf[decBufIndex];
            const int decBytes = LZ4_decompress_safe_continue(lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
            if(decBytes <= 0)
            {
                break;
            }
            binWrite(outFp, decPtr, (size_t) decBytes);
        }
        decBufIndex = (decBufIndex + 1) % 2;
    }
}
static void test_decompress(
    FILE* outFp,
    FILE* inpFp,
    size_t messageMaxBytes,
    size_t ringBufferBytes)
{
    LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
    char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
    char* const decBuf = (char*) malloc(ringBufferBytes);
    int decOffset = 0;

    for ( ; ; )
    {
        uint16_t cmpBytes = 0;

        if (read_uint16(inpFp, &cmpBytes) != 1) break;
        if (cmpBytes <= 0) break;
        if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;

        {
            char* const decPtr = &decBuf[decOffset];
            const int decBytes = LZ4_decompress_safe_continue(
                lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
            if (decBytes <= 0) break;
            write_bin(outFp, decPtr, decBytes);

            // Add and wraparound the ringbuffer offset
            decOffset += decBytes;
            if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
        }
    }

    free(decBuf);
    free(cmpBuf);
    LZ4_freeStreamDecode(lz4StreamDecode);
}
Exemplo n.º 9
0
int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {

#ifdef HAVE_LZ4
        _cleanup_free_ char *buf = NULL, *out = NULL;
        size_t buf_size = 0;
        LZ4_streamDecode_t lz4_data = {};
        le32_t header;
        size_t total_in = sizeof(header), total_out = 0;

        assert(fdf >= 0);
        assert(fdt >= 0);

        out = malloc(4*LZ4_BUFSIZE);
        if (!out)
                return log_oom();

        for (;;) {
                ssize_t n, m;
                int r;

                n = read(fdf, &header, sizeof(header));
                if (n < 0)
                        return -errno;
                if (n != sizeof(header))
                        return errno ? -errno : -EIO;

                m = le32toh(header);
                if (m == 0)
                        break;

                /* We refuse to use a bigger decompression buffer than
                 * the one used for compression by 4 times. This means
                 * that compression buffer size can be enlarged 4
                 * times. This can be changed, but old binaries might
                 * not accept buffers compressed by newer binaries then.
                 */
                if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
                        log_error("Compressed stream block too big: %zd bytes", m);
                        return -EBADMSG;
                }

                total_in += sizeof(header) + m;

                if (!GREEDY_REALLOC(buf, buf_size, m))
                        return log_oom();

                errno = 0;
                n = loop_read(fdf, buf, m, false);
                if (n < 0)
                        return n;
                if (n != m)
                        return errno ? -errno : -EIO;

                r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
                if (r <= 0)
                        log_error("LZ4 decompression failed.");

                total_out += r;

                if (max_bytes != -1 && total_out > (size_t) max_bytes) {
                        log_debug("Decompressed stream longer than %zd bytes", max_bytes);
                        return -EFBIG;
                }

                n = loop_write(fdt, out, r, false);
                if (n < 0)
                        return n;
        }

        log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
                  total_in, total_out,
                  (double) total_out / total_in * 100);

        return 0;
#else
        log_error("Cannot decompress file. Compiled without LZ4 support.");
        return -EPROTONOSUPPORT;
#endif
}
Exemplo n.º 10
0
void Lz4::Put(const void *ptr_, int size)
{
	LLOG("Put " << size);

	ASSERT(compress >= 0);
	
	if(error)
		return;
	
	const char *ptr = reinterpret_cast<const char *>(ptr_);

	if(compress) {
		while(size > 0) {
			if(error)
				return;
			int n = BLOCK_BYTES - pos;
			if(size >= n) {
				memcpy(block[cbi] + pos, ptr, n); // TODO: Avoid unnecessary memcpy
				pos = BLOCK_BYTES;
				FlushOut();
				size -= n;
				ptr += n;
			}
			else {
				memcpy(block[cbi] + pos, ptr, size);
				pos += size;
				break;
			}
		}
	}
	else {
		while(size > 0) { // TODO might try to avoid memcpy
			if(pos < 4) { // Get length first
				int n = min(size, 4 - pos);
				memcpy(~buffer + pos, ptr, n);
				pos += n;
				ptr += n;
				size -= n;
			}
			if(pos < 4)
				return;
			int count = Peek32le(~buffer);
			int need = count - (pos - 4);
			if(size >= need) { // we have enough data to finish the block
				memcpy(~buffer + pos, ptr, need);
				int out_count = LZ4_decompress_safe_continue(lz4StreamDecode, ~buffer + 4, block[cbi], count, BLOCK_BYTES);
				if(out_count <= 0) {
					error = true;
					return;
				}
				WhenOut(block[cbi], out_count);
				cbi = !cbi;
				ptr += need;
				size -= need;
				pos = 0;
			}
			else {
				memcpy(~buffer + pos, ptr, size);
				pos += size;
				break;
			}
		}
	}
}