void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DECODE_RING_BUFFER];
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
for(;;) {
int cmpBytes = 0;
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
if(r0 != 1 || cmpBytes <= 0) break;
const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes) break;
}
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0) break;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0;
}
}
}
void LZ4Segment<T>::_decompress_block_to_bytes(const size_t block_index, std::vector<char>& decompressed_data,
const size_t write_offset) const {
const auto decompressed_block_size = block_index + 1 != _lz4_blocks.size() ? _block_size : _last_block_size;
auto& compressed_block = _lz4_blocks[block_index];
const auto compressed_block_size = compressed_block.size();
int decompressed_result;
if (_dictionary.empty()) {
/**
* If the dictionary is empty, we either have only a single block or had not enough data for a dictionary.
* When decoding without a dictionary LZ4 needs a stream decode pointer (which would be used to decode the
* following blocks).
* A new decoder needs to be created for every block (in the case of multiple blocks being compressed without a
* dictionary) since the blocks were compressed independently.
* This decoder needs to be reset via LZ4_setStreamDecode since LZ4 reuses the previous state instead.
*/
LZ4_streamDecode_t lz4_stream_decoder;
auto lz4_stream_decoder_ptr = std::make_unique<LZ4_streamDecode_t>(lz4_stream_decoder);
const auto reset_decoder_status = LZ4_setStreamDecode(lz4_stream_decoder_ptr.get(), nullptr, 0);
Assert(reset_decoder_status == 1, "LZ4 decompression failed to reset stream decoder.");
decompressed_result = LZ4_decompress_safe_continue(
lz4_stream_decoder_ptr.get(), compressed_block.data(), decompressed_data.data() + write_offset,
static_cast<int>(compressed_block_size), static_cast<int>(decompressed_block_size));
} else {
decompressed_result = LZ4_decompress_safe_usingDict(
compressed_block.data(), decompressed_data.data() + write_offset, static_cast<int>(compressed_block_size),
static_cast<int>(decompressed_block_size), _dictionary.data(), static_cast<int>(_dictionary.size()));
}
Assert(decompressed_result > 0, "LZ4 stream decompression failed");
DebugAssert(static_cast<size_t>(decompressed_result) == decompressed_block_size,
"Decompressed LZ4 block has different size than the initial source data.");
}
void ReadBlock()
{
char m_lz4buf[LZ4Size];
uint32_t sz;
if( fread( &sz, 1, sizeof( sz ), m_file ) == sizeof( sz ) )
{
fread( m_lz4buf, 1, sz, m_file );
m_lastBlock = LZ4_decompress_safe_continue( m_stream, m_lz4buf, m_second, sz, BufSize );
}
else
{
m_lastBlock = 0;
}
}
void lz4_unpack(void * in, const size_t * const compSize, void * const out, int * const size, const int * const ierr) {
size_t offset=0;
size_t offset_in=0;
decBufIndex=0;
LZ4_resetStream(lz4Stream);
LZ4_setStreamDecode(lz4StreamDecode,NULL,0);
for(;;) {
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
/*printf("offset_in+sizeof(cmpBytes): %d\n",offset_in+sizeof(cmpBytes));*/
/*printf("compSize: %d\n",*compSize);*/
int tmp=offset_in+sizeof(cmpBytes);
/*printf("tmp: %d\n", tmp);*/
/*printf("compSize: %d\n", *compSize);*/
if(tmp > (int) *compSize) {
break;
}
memcpy(&cmpBytes, in+offset_in, sizeof(cmpBytes));
/*printf("cmpBytes: %d\n",cmpBytes);*/
if(cmpBytes <= 0) {
/*printf("cmpBytes %d\n", cmpBytes);*/
break;
}
/*printf("lz4 cmpBytes %d\n", cmpBytes);*/
memcpy(&cmpBuf, in+offset_in+sizeof(cmpBytes), cmpBytes);
const size_t readCount1=0;
offset_in=offset_in+sizeof(cmpBytes)+cmpBytes;
/*printf("new offset_in: %zu\n", offset_in);*/
{
char* decPtr = decBuf[decBufIndex];
int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0) {
/*printf("lz4 decBytes: %d\n", decBytes);*/
break;
}
memcpy(out+offset, decPtr, (size_t) decBytes);
offset=offset+(size_t) decBytes;
}
decBufIndex = (decBufIndex + 1) % 2;
}
*size=offset;
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
LZ4_streamDecode_t lz4StreamDecode_body;
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
char decBuf[2][BLOCK_BYTES];
int decBufIndex = 0;
LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);
for(;;) {
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
{
const size_t readCount0 = read_int(inpFp, &cmpBytes);
if(readCount0 != 1 || cmpBytes <= 0) {
break;
}
const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes);
if(readCount1 != (size_t) cmpBytes) {
break;
}
}
{
char* const decPtr = decBuf[decBufIndex];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0) {
break;
}
write_bin(outFp, decPtr, (size_t) decBytes);
}
decBufIndex = (decBufIndex + 1) % 2;
}
}
bool LZ4Decompressor::FillPage0()
{
// swap pages
std::swap(m_Page[0], m_Page[1]);
int32_t compSize = 0;
bool success = true;
success &= m_Read->Read(compSize);
success &= m_Read->Read(m_CompressBuffer, compSize);
if(!success)
{
FreeAlignedBuffer(m_Page[0]);
FreeAlignedBuffer(m_Page[1]);
FreeAlignedBuffer(m_CompressBuffer);
m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
return false;
}
int32_t decompSize = LZ4_decompress_safe_continue(&m_LZ4Decomp, (const char *)m_CompressBuffer,
(char *)m_Page[0], compSize, lz4BlockSize);
if(decompSize < 0)
{
RDCERR("Error decompressing: %i", decompSize);
FreeAlignedBuffer(m_Page[0]);
FreeAlignedBuffer(m_Page[1]);
FreeAlignedBuffer(m_CompressBuffer);
m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
return false;
}
m_PageOffset = 0;
m_PageLength = decompSize;
return success;
}
void decompressFile(FILE* outFp, FILE* inpFp)
{
LZ4_streamDecode_t lz4StreamDecode_body;
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
char decBuf[2][BLOCK_BYTES];
int decBufIndex = 0;
LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);
while(1)
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
{
const size_t readCount0 = intRead(inpFp, &cmpBytes);
if((readCount0 != 1) || (cmpBytes <= 0))
{
break;
}
const size_t readCount1 = binRead(inpFp, cmpBuf, (size_t) cmpBytes);
if(readCount1 != (size_t) cmpBytes)
{
break;
}
}
{
char* const decPtr = decBuf[decBufIndex];
const int decBytes = LZ4_decompress_safe_continue(lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0)
{
break;
}
binWrite(outFp, decPtr, (size_t) decBytes);
}
decBufIndex = (decBufIndex + 1) % 2;
}
}
static void test_decompress(
FILE* outFp,
FILE* inpFp,
size_t messageMaxBytes,
size_t ringBufferBytes)
{
LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
char* const decBuf = (char*) malloc(ringBufferBytes);
int decOffset = 0;
for ( ; ; )
{
uint16_t cmpBytes = 0;
if (read_uint16(inpFp, &cmpBytes) != 1) break;
if (cmpBytes <= 0) break;
if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
if (decBytes <= 0) break;
write_bin(outFp, decPtr, decBytes);
// Add and wraparound the ringbuffer offset
decOffset += decBytes;
if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
}
}
free(decBuf);
free(cmpBuf);
LZ4_freeStreamDecode(lz4StreamDecode);
}
int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
#ifdef HAVE_LZ4
_cleanup_free_ char *buf = NULL, *out = NULL;
size_t buf_size = 0;
LZ4_streamDecode_t lz4_data = {};
le32_t header;
size_t total_in = sizeof(header), total_out = 0;
assert(fdf >= 0);
assert(fdt >= 0);
out = malloc(4*LZ4_BUFSIZE);
if (!out)
return log_oom();
for (;;) {
ssize_t n, m;
int r;
n = read(fdf, &header, sizeof(header));
if (n < 0)
return -errno;
if (n != sizeof(header))
return errno ? -errno : -EIO;
m = le32toh(header);
if (m == 0)
break;
/* We refuse to use a bigger decompression buffer than
* the one used for compression by 4 times. This means
* that compression buffer size can be enlarged 4
* times. This can be changed, but old binaries might
* not accept buffers compressed by newer binaries then.
*/
if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
log_error("Compressed stream block too big: %zd bytes", m);
return -EBADMSG;
}
total_in += sizeof(header) + m;
if (!GREEDY_REALLOC(buf, buf_size, m))
return log_oom();
errno = 0;
n = loop_read(fdf, buf, m, false);
if (n < 0)
return n;
if (n != m)
return errno ? -errno : -EIO;
r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
if (r <= 0)
log_error("LZ4 decompression failed.");
total_out += r;
if (max_bytes != -1 && total_out > (size_t) max_bytes) {
log_debug("Decompressed stream longer than %zd bytes", max_bytes);
return -EFBIG;
}
n = loop_write(fdt, out, r, false);
if (n < 0)
return n;
}
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out,
(double) total_out / total_in * 100);
return 0;
#else
log_error("Cannot decompress file. Compiled without LZ4 support.");
return -EPROTONOSUPPORT;
#endif
}
void Lz4::Put(const void *ptr_, int size)
{
LLOG("Put " << size);
ASSERT(compress >= 0);
if(error)
return;
const char *ptr = reinterpret_cast<const char *>(ptr_);
if(compress) {
while(size > 0) {
if(error)
return;
int n = BLOCK_BYTES - pos;
if(size >= n) {
memcpy(block[cbi] + pos, ptr, n); // TODO: Avoid unnecessary memcpy
pos = BLOCK_BYTES;
FlushOut();
size -= n;
ptr += n;
}
else {
memcpy(block[cbi] + pos, ptr, size);
pos += size;
break;
}
}
}
else {
while(size > 0) { // TODO might try to avoid memcpy
if(pos < 4) { // Get length first
int n = min(size, 4 - pos);
memcpy(~buffer + pos, ptr, n);
pos += n;
ptr += n;
size -= n;
}
if(pos < 4)
return;
int count = Peek32le(~buffer);
int need = count - (pos - 4);
if(size >= need) { // we have enough data to finish the block
memcpy(~buffer + pos, ptr, need);
int out_count = LZ4_decompress_safe_continue(lz4StreamDecode, ~buffer + 4, block[cbi], count, BLOCK_BYTES);
if(out_count <= 0) {
error = true;
return;
}
WhenOut(block[cbi], out_count);
cbi = !cbi;
ptr += need;
size -= need;
pos = 0;
}
else {
memcpy(~buffer + pos, ptr, size);
pos += size;
break;
}
}
}
}
