bool LZ4Compressor::FlushPage0()
{
// if we encountered a stream error this will be NULL
if(!m_CompressBuffer)
return false;
// m_PageOffset is the amount written, usually equal to lz4BlockSize except the last block.
int32_t compSize =
LZ4_compress_fast_continue(&m_LZ4Comp, (const char *)m_Page[0], (char *)m_CompressBuffer,
(int)m_PageOffset, (int)LZ4_COMPRESSBOUND(lz4BlockSize), 1);
if(compSize < 0)
{
RDCERR("Error compressing: %i", compSize);
FreeAlignedBuffer(m_Page[0]);
FreeAlignedBuffer(m_Page[1]);
FreeAlignedBuffer(m_CompressBuffer);
m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
return false;
}
bool success = true;
success &= m_Write->Write(compSize);
success &= m_Write->Write(m_CompressBuffer, compSize);
// swap pages
std::swap(m_Page[0], m_Page[1]);
// start writing to the start of the page again
m_PageOffset = 0;
return success;
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DECODE_RING_BUFFER];
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
for(;;) {
int cmpBytes = 0;
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
if(r0 != 1 || cmpBytes <= 0) break;
const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes) break;
}
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0) break;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0;
}
}
}
void compress_data(Data *d) {
int slen = d->datalen;
int olen = LZ4_COMPRESSBOUND(slen);
char cbuf[olen];
#if LZ4_VERSION_MINOR >= 7
int ocnt = LZ4_compress_default((const char*)d->data, cbuf, slen, olen);
#else
/* In this API version, $cbuf is required to be large enough to handle
* up to LZ4_COMPRESSBOUND($slen) bytes. Since we already allocate
* this large of a buffer anyway, we just drop $olen.
*
* This is required for building against the liblz4 version in Debian
* Jessie.
*/
int ocnt = LZ4_compress((const char*)d->data, cbuf, slen);
#endif
if(ocnt == 0) /* should never happen */
LERROR(EXIT_FAILURE, 0, "LZ4_compress_default() failed");
free(d->data);
d->data = malloc(ocnt);
GUARD_MALLOC(d->data);
d->datalen = (size_t)ocnt;
memcpy(d->data, (const void*)cbuf, d->datalen);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_stream_t lz4Stream_body;
LZ4_stream_t* lz4Stream = &lz4Stream_body;
char inpBuf[2][BLOCK_BYTES];
int inpBufIndex = 0;
LZ4_resetStream(lz4Stream);
for(;;) {
char* const inpPtr = inpBuf[inpBufIndex];
const int inpBytes = (int) read_bin(inpFp, inpPtr, BLOCK_BYTES);
if(0 == inpBytes) {
break;
}
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf), 1);
if(cmpBytes <= 0) {
break;
}
write_int(outFp, cmpBytes);
write_bin(outFp, cmpBuf, (size_t) cmpBytes);
}
inpBufIndex = (inpBufIndex + 1) % 2;
}
write_int(outFp, 0);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_stream_t lz4Stream_body = { 0 };
LZ4_stream_t* lz4Stream = &lz4Stream_body;
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
for(;;) {
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
{
char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
const int cmpBytes = LZ4_compress_continue(lz4Stream, inpPtr, cmpBuf, inpBytes);
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
inpOffset += inpBytes;
// Wraparound the ringbuffer offset
if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) inpOffset = 0;
}
}
write_int32(outFp, 0);
}
LZ4Compressor::LZ4Compressor(StreamWriter *write, Ownership own) : Compressor(write, own)
{
m_Page[0] = AllocAlignedBuffer(lz4BlockSize);
m_Page[1] = AllocAlignedBuffer(lz4BlockSize);
m_CompressBuffer = AllocAlignedBuffer(LZ4_COMPRESSBOUND(lz4BlockSize));
m_PageOffset = 0;
LZ4_resetStream(&m_LZ4Comp);
}
Lz4::Lz4()
{
block[0] = new char[2 * BLOCK_BYTES];
block[1] = block[0] + BLOCK_BYTES;
buffer.Alloc(LZ4_COMPRESSBOUND(BLOCK_BYTES) + 4);
compress = -1;
WhenOut = callback(this, &Lz4::PutOut);
}
static PyObject *create_hc_stream(PyObject *self, PyObject *args, PyObject *keywds)
{
(void)self;
int block_size = 0;
int compression_level = 9;
static char *kwlist[] = {"block_size", "compression_level", NULL};
if (!PyArg_ParseTupleAndKeywords(args, keywds, "i|i", kwlist,
&block_size,
&compression_level)) {
return NULL;
}
if (block_size > LZ4_MAX_INPUT_SIZE) {
PyErr_Format(PyExc_ValueError, "block size is %d bytes, which is larger than the maximum supported size of %d bytes", block_size, LZ4_MAX_INPUT_SIZE);
return NULL;
}
if (block_size <= 0) {
PyErr_Format(PyExc_ValueError, "block size is %d bytes, which is invalid", block_size);
return NULL;
}
struct compression_stream *stream = (struct compression_stream *)PyMem_Malloc(sizeof(struct compression_stream));
if (!stream) {
return PyErr_NoMemory();
}
stream->block_size = block_size;
stream->input_buffer_index = 0;
stream->input_buffer[0] = (char *)PyMem_Malloc(block_size);
stream->input_buffer[1] = (char *)PyMem_Malloc(block_size);
stream->compressed_buffer_max_size = LZ4_COMPRESSBOUND(block_size);
stream->compressed_buffer = (char *)PyMem_Malloc(stream->compressed_buffer_max_size);
stream->stream = LZ4_createStreamHC();
if (!stream->input_buffer[0] ||
!stream->input_buffer[1] ||
!stream->compressed_buffer ||
!stream->stream) {
PyMem_Free(stream->input_buffer[0]);
PyMem_Free(stream->input_buffer[1]);
PyMem_Free(stream->compressed_buffer);
if (stream->stream) {
LZ4_freeStreamHC(stream->stream);
}
PyMem_Free(stream);
return PyErr_NoMemory();
}
LZ4_resetStreamHC(stream->stream, compression_level);
return PyCapsule_New(stream, NULL, NULL);
}
LZ4Decompressor::LZ4Decompressor(StreamReader *read, Ownership own) : Decompressor(read, own)
{
m_Page[0] = AllocAlignedBuffer(lz4BlockSize);
m_Page[1] = AllocAlignedBuffer(lz4BlockSize);
m_CompressBuffer = AllocAlignedBuffer(LZ4_COMPRESSBOUND(lz4BlockSize));
m_PageOffset = 0;
m_PageLength = 0;
LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0);
}
/*
* lz4_pre_size --
* WiredTiger LZ4 destination buffer sizing for compression.
*/
static int
lz4_pre_size(WT_COMPRESSOR *compressor, WT_SESSION *session,
uint8_t *src, size_t src_len,
size_t *result_lenp)
{
(void)compressor;
(void)session;
(void)src;
/*
* LZ4 can use more space than the input data size, use the library
* calculation of that overhead (plus our overhead) to be safe.
*/
*result_lenp = LZ4_COMPRESSBOUND(src_len) + sizeof(size_t);
return (0);
}
static void test_compress(
FILE* outFp,
FILE* inpFp,
size_t messageMaxBytes,
size_t ringBufferBytes)
{
LZ4_stream_t* const lz4Stream = LZ4_createStream();
const size_t cmpBufBytes = LZ4_COMPRESSBOUND(messageMaxBytes);
char* const cmpBuf = (char*) malloc(cmpBufBytes);
char* const inpBuf = (char*) malloc(ringBufferBytes);
int inpOffset = 0;
for ( ; ; )
{
char* const inpPtr = &inpBuf[inpOffset];
#if 0
// Read random length data to the ring buffer.
const int randomLength = (rand() % messageMaxBytes) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
#else
// Read line to the ring buffer.
int inpBytes = 0;
if (!fgets(inpPtr, (int) messageMaxBytes, inpFp))
break;
inpBytes = (int) strlen(inpPtr);
#endif
{
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1);
if (cmpBytes <= 0) break;
write_uint16(outFp, (uint16_t) cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
// Add and wraparound the ringbuffer offset
inpOffset += inpBytes;
if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0;
}
}
write_uint16(outFp, 0);
free(inpBuf);
free(cmpBuf);
LZ4_freeStream(lz4Stream);
}
void lz4_pack(const void * const in, const int * const size, void * const out, int * const compSize, const int * const ierr) {
int i=0;
size_t ccount=0; // Bytes in the chunk to be compressed. This is BLOCK_BYTES besides for the last chunk
LZ4_resetStream(lz4Stream);
if(*size<0 || out==NULL || *size<0 ) {
printf("Error in Nek LZ4 compression\n");
return;
}
/*printf("size: %d\n", *size);*/
size_t lcount=(size_t) *size; // Bytes left to be compressed
/*printf("lcount: %zu\n", lcount);*/
size_t offset=0;
*compSize=0;
while (lcount > 0) {
if(lcount < BLOCK_BYTES) {
ccount=lcount;
}
else {
ccount=BLOCK_BYTES;
}
/*printf("offset: %zu\n",offset);*/
/*printf("ccount: %zu\n",ccount);*/
char* const inpPtr = inpBuf[inpBufIndex];
memcpy(inpPtr, in+offset, ccount);
/*printf("i: %d\n",i);*/
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpPtr, cmpBuf, ccount, sizeof(cmpBuf), 1);
if(cmpBytes <= 0) {
break;
}
/*printf("cmpBytes: %d\n",cmpBytes);*/
/*printf("compSize: %d\n",*compSize);*/
memcpy(out+*compSize, &cmpBytes, sizeof(cmpBytes));
memcpy(out+*compSize+sizeof(cmpBytes), &cmpBuf, cmpBytes);
*compSize=*compSize+cmpBytes+sizeof(cmpBytes);
}
inpBufIndex = (inpBufIndex + 1) % 2;
lcount=lcount-ccount;
offset=offset+ccount;
/*printf("----------------------\n");*/
i=i+1;
}
/*printf("compSize in C: %d\n", *compSize);*/
}
static SquashStatus
squash_lz4_compress_buffer_unsafe (SquashCodec* codec,
size_t* compressed_size,
uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_size)],
size_t uncompressed_size,
const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_size)],
SquashOptions* options) {
int level = squash_codec_get_option_int_index (codec, options, SQUASH_LZ4_OPT_LEVEL);
#if INT_MAX < SIZE_MAX
if (SQUASH_UNLIKELY(INT_MAX < uncompressed_size) ||
SQUASH_UNLIKELY(INT_MAX < *compressed_size))
return squash_error (SQUASH_RANGE);
#endif
assert (*compressed_size >= LZ4_COMPRESSBOUND(uncompressed_size));
int lz4_r;
if (level == 7) {
lz4_r = LZ4_compress ((char*) uncompressed,
(char*) compressed,
(int) uncompressed_size);
} else if (level < 7) {
lz4_r = LZ4_compress_fast ((const char*) uncompressed,
(char*) compressed,
(int) uncompressed_size,
(int) *compressed_size,
squash_lz4_level_to_fast_mode (level));
} else {
lz4_r = LZ4_compressHC2 ((char*) uncompressed,
(char*) compressed,
(int) uncompressed_size,
squash_lz4_level_to_hc_level (level));
}
#if SIZE_MAX < INT_MAX
if (SQUASH_UNLIKELY(SIZE_MAX < lz4_r))
return squash_error (SQUASH_RANGE);
#endif
*compressed_size = lz4_r;
return (lz4_r == 0) ? SQUASH_BUFFER_FULL : SQUASH_OK;
}
void lz4_unpack(void * in, const size_t * const compSize, void * const out, int * const size, const int * const ierr) {
size_t offset=0;
size_t offset_in=0;
decBufIndex=0;
LZ4_resetStream(lz4Stream);
LZ4_setStreamDecode(lz4StreamDecode,NULL,0);
for(;;) {
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
/*printf("offset_in+sizeof(cmpBytes): %d\n",offset_in+sizeof(cmpBytes));*/
/*printf("compSize: %d\n",*compSize);*/
int tmp=offset_in+sizeof(cmpBytes);
/*printf("tmp: %d\n", tmp);*/
/*printf("compSize: %d\n", *compSize);*/
if(tmp > (int) *compSize) {
break;
}
memcpy(&cmpBytes, in+offset_in, sizeof(cmpBytes));
/*printf("cmpBytes: %d\n",cmpBytes);*/
if(cmpBytes <= 0) {
/*printf("cmpBytes %d\n", cmpBytes);*/
break;
}
/*printf("lz4 cmpBytes %d\n", cmpBytes);*/
memcpy(&cmpBuf, in+offset_in+sizeof(cmpBytes), cmpBytes);
const size_t readCount1=0;
offset_in=offset_in+sizeof(cmpBytes)+cmpBytes;
/*printf("new offset_in: %zu\n", offset_in);*/
{
char* decPtr = decBuf[decBufIndex];
int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0) {
/*printf("lz4 decBytes: %d\n", decBytes);*/
break;
}
memcpy(out+offset, decPtr, (size_t) decBytes);
offset=offset+(size_t) decBytes;
}
decBufIndex = (decBufIndex + 1) % 2;
}
*size=offset;
}
static mrb_value
mrb_LZ4_compress_default(mrb_state *mrb, mrb_value self)
{
char *source;
mrb_int source_size;
mrb_get_args(mrb, "s", &source, &source_size);
int maxDestSize = LZ4_COMPRESSBOUND(source_size);
if (likely(maxDestSize)) {
mrb_value dest = mrb_str_buf_new(mrb, maxDestSize);
int destSize = LZ4_compress_default(source, RSTRING_PTR(dest), source_size, RSTRING_CAPA(dest));
if (likely(destSize)) {
return mrb_str_resize(mrb, dest, destSize);
} else {
mrb_raise(mrb, E_LZ4_ERROR, "cannot compress");
}
} else {
mrb_raise(mrb, E_ARGUMENT_ERROR, "source_size is too large");
}
}
void save_thread_func(void * lpParam){
char line_buffer[128];
char * cmp_frame_buff ;
IplImage * dummy_image ;
int i = 0 ;
unsigned long timestamp ;
FILE *seqFile;
cmp_frame_buff = malloc(LZ4_MESSAGE_MAX_BYTES);
if(cmp_frame_buff == NULL) printf("Cannot allocate sequence output file");
sprintf(path, "%s/sequence.lz4", path_base);
seqFile = fopen(path, "wb");
//start LZ4 compression
LZ4_stream_t* const lz4Stream = LZ4_createStream();
const size_t cmpBufBytes = LZ4_COMPRESSBOUND(LZ4_MESSAGE_MAX_BYTES);
char* const cmpBuf = (char*) malloc(cmpBufBytes);
dummy_image = cvCreateImage(cvSize(640, 480), IPL_DEPTH_8U, 1);
printf("Start Save ! \n");
while(thread_alive || my_frame_buffer.nb_frames_availables > 0){
if(my_frame_buffer.nb_frames_availables > 0){
if(pop_frame(dummy_image, ×tamp, &my_frame_buffer) >= 0){
memcpy(cmp_frame_buff, ×tamp, sizeof(unsigned long));
memcpy((cmp_frame_buff + sizeof(unsigned long)), dummy_image->imageData, 640*480);
const int cmpBytes = LZ4_compress_continue(lz4Stream, cmp_frame_buff, cmpBuf, MESSAGE_MAX_BYTES);
if (cmpBytes <= 0){
printf("Compression failed \n");
break;
}
fwrite(cmpBuf, 1, cmpBytes, seqFile);
i ++ ;
}
}
usleep(WRITE_DELAY_US);
}
free(cmp_frame_buff);
LZ4_freeStream(lz4Stream);
fclose(seqFile);
printf("End Save \n");
}
void decompressFile(FILE* outFp, FILE* inpFp)
{
LZ4_streamDecode_t lz4StreamDecode_body;
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
char decBuf[2][BLOCK_BYTES];
int decBufIndex = 0;
LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);
while(1)
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
{
const size_t readCount0 = intRead(inpFp, &cmpBytes);
if((readCount0 != 1) || (cmpBytes <= 0))
{
break;
}
const size_t readCount1 = binRead(inpFp, cmpBuf, (size_t) cmpBytes);
if(readCount1 != (size_t) cmpBytes)
{
break;
}
}
{
char* const decPtr = decBuf[decBufIndex];
const int decBytes = LZ4_decompress_safe_continue(lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0)
{
break;
}
binWrite(outFp, decPtr, (size_t) decBytes);
}
decBufIndex = (decBufIndex + 1) % 2;
}
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
LZ4_streamDecode_t lz4StreamDecode_body;
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
char decBuf[2][BLOCK_BYTES];
int decBufIndex = 0;
LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);
for(;;) {
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
{
const size_t readCount0 = read_int(inpFp, &cmpBytes);
if(readCount0 != 1 || cmpBytes <= 0) {
break;
}
const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes);
if(readCount1 != (size_t) cmpBytes) {
break;
}
}
{
char* const decPtr = decBuf[decBufIndex];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0) {
break;
}
write_bin(outFp, decPtr, (size_t) decBytes);
}
decBufIndex = (decBufIndex + 1) % 2;
}
}
static void test_decompress(
FILE* outFp,
FILE* inpFp,
size_t messageMaxBytes,
size_t ringBufferBytes)
{
LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
char* const decBuf = (char*) malloc(ringBufferBytes);
int decOffset = 0;
for ( ; ; )
{
uint16_t cmpBytes = 0;
if (read_uint16(inpFp, &cmpBytes) != 1) break;
if (cmpBytes <= 0) break;
if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
if (decBytes <= 0) break;
write_bin(outFp, decPtr, decBytes);
// Add and wraparound the ringbuffer offset
decOffset += decBytes;
if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
}
}
free(decBuf);
free(cmpBuf);
LZ4_freeStreamDecode(lz4StreamDecode);
}
int RemoteDesktop::Compression_Handler::CompressionBound(int s) {
return LZ4_COMPRESSBOUND(s);
}
int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
#ifdef HAVE_LZ4
_cleanup_free_ char *buf = NULL, *out = NULL;
size_t buf_size = 0;
LZ4_streamDecode_t lz4_data = {};
le32_t header;
size_t total_in = sizeof(header), total_out = 0;
assert(fdf >= 0);
assert(fdt >= 0);
out = malloc(4*LZ4_BUFSIZE);
if (!out)
return log_oom();
for (;;) {
ssize_t n, m;
int r;
n = read(fdf, &header, sizeof(header));
if (n < 0)
return -errno;
if (n != sizeof(header))
return errno ? -errno : -EIO;
m = le32toh(header);
if (m == 0)
break;
/* We refuse to use a bigger decompression buffer than
* the one used for compression by 4 times. This means
* that compression buffer size can be enlarged 4
* times. This can be changed, but old binaries might
* not accept buffers compressed by newer binaries then.
*/
if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
log_error("Compressed stream block too big: %zd bytes", m);
return -EBADMSG;
}
total_in += sizeof(header) + m;
if (!GREEDY_REALLOC(buf, buf_size, m))
return log_oom();
errno = 0;
n = loop_read(fdf, buf, m, false);
if (n < 0)
return n;
if (n != m)
return errno ? -errno : -EIO;
r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
if (r <= 0)
log_error("LZ4 decompression failed.");
total_out += r;
if (max_bytes != -1 && total_out > (size_t) max_bytes) {
log_debug("Decompressed stream longer than %zd bytes", max_bytes);
return -EFBIG;
}
n = loop_write(fdt, out, r, false);
if (n < 0)
return n;
}
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out,
(double) total_out / total_in * 100);
return 0;
#else
log_error("Cannot decompress file. Compiled without LZ4 support.");
return -EPROTONOSUPPORT;
#endif
}
int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
#ifdef HAVE_LZ4
_cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
char *buf;
LZ4_stream_t lz4_data = {};
le32_t header;
size_t total_in = 0, total_out = sizeof(header);
ssize_t n;
assert(fdf >= 0);
assert(fdt >= 0);
buf1 = malloc(LZ4_BUFSIZE);
buf2 = malloc(LZ4_BUFSIZE);
out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
if (!buf1 || !buf2 || !out)
return log_oom();
buf = buf1;
for (;;) {
size_t m;
int r;
m = LZ4_BUFSIZE;
if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
m = max_bytes - total_in;
n = read(fdf, buf, m);
if (n < 0)
return -errno;
if (n == 0)
break;
total_in += n;
r = LZ4_compress_continue(&lz4_data, buf, out, n);
if (r == 0) {
log_error("LZ4 compression failed.");
return -EBADMSG;
}
header = htole32(r);
errno = 0;
n = write(fdt, &header, sizeof(header));
if (n < 0)
return -errno;
if (n != sizeof(header))
return errno ? -errno : -EIO;
n = loop_write(fdt, out, r, false);
if (n < 0)
return n;
total_out += sizeof(header) + r;
buf = buf == buf1 ? buf2 : buf1;
}
header = htole32(0);
n = write(fdt, &header, sizeof(header));
if (n < 0)
return -errno;
if (n != sizeof(header))
return errno ? -errno : -EIO;
log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out,
(double) total_out / total_in * 100);
return 0;
#else
return -EPROTONOSUPPORT;
#endif
}
#include <stdio.h>
#include <assert.h>
#include <stdint.h>
#include <fcntl.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <time.h>
#include <errno.h>
#include "lz4.h"
#define BLOCK_SIZE_MAX (1024 * 64)
static char in[BLOCK_SIZE_MAX];
static char out[LZ4_COMPRESSBOUND(BLOCK_SIZE_MAX)];
static char uncomp[BLOCK_SIZE_MAX];
int main01(int argc, const char * argv[])
{
struct stat buf;
FILE *inf;
int rsize, fsize;
int csize, dsize;
void *lz4state;
stat(argv[1], &buf);
fsize = (int)buf.st_size;
inf = fopen(argv[1], "r"); // dumpFile is a 512KB file
if(NULL == inf)
static size_t
squash_lz4_get_max_compressed_size (SquashCodec* codec, size_t uncompressed_size) {
return LZ4_COMPRESSBOUND(uncompressed_size);
}
static size_t lz4_filter(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void * outBuf = NULL;
size_t ret_value;
if (flags & H5Z_FLAG_REVERSE)
{
const char* rpos = (char*)*buf; /* pointer to current read position */
const uint64_t * const i64Buf = (uint64_t *) rpos;
const uint64_t origSize = (uint64_t)(be64toht(*i64Buf));/* is saved in be format */
rpos += 8; /* advance the pointer */
uint32_t *i32Buf = (uint32_t*)rpos;
uint32_t blockSize = (uint32_t)(be32toht(*i32Buf));
rpos += 4;
if(blockSize>origSize)
blockSize = origSize;
if (NULL==(outBuf = malloc(origSize)))
{
printf("cannot malloc\n");
goto error;
}
char *roBuf = (char*)outBuf; /* pointer to current write position */
uint64_t decompSize = 0;
/// start with the first block ///
while(decompSize < origSize)
{
if(origSize-decompSize < blockSize) /* the last block can be smaller than blockSize. */
blockSize = origSize-decompSize;
i32Buf = (uint32_t*)rpos;
uint32_t compressedBlockSize = be32toht(*i32Buf); /// is saved in be format
rpos += 4;
if(compressedBlockSize == blockSize) /* there was no compression */
{
memcpy(roBuf, rpos, blockSize);
}
else /* do the decompression */
{
int compressedBytes = LZ4_uncompress(rpos, roBuf, blockSize);
if(compressedBytes != compressedBlockSize)
{
printf("decompressed size not the same: %d, != %d\n", compressedBytes, compressedBlockSize);
goto error;
}
}
rpos += compressedBlockSize; /* advance the read pointer to the next block */
roBuf += blockSize; /* advance the write pointer */
decompSize += blockSize;
}
free(*buf);
*buf = outBuf;
outBuf = NULL;
ret_value = (size_t)origSize; // should always work, as orig_size cannot be > 2GB (sizeof(size_t) < 4GB)
}
else /* forward filter */
{
if (nbytes > INT32_MAX)
{
/* can only compress chunks up to 2GB */
goto error;
}
size_t blockSize;
if(cd_nelmts > 0 && cd_values[0] > 0)
{
blockSize = cd_values[0];
}
else
{
blockSize = DEFAULT_BLOCK_SIZE;
}
if(blockSize > nbytes)
{
blockSize = nbytes;
}
size_t nBlocks = (nbytes-1)/blockSize +1 ;
if (NULL==(outBuf = malloc(LZ4_COMPRESSBOUND(nbytes)
+ 4+8 + nBlocks*4)))
{
goto error;
}
char *rpos = (char*)*buf; /* pointer to current read position */
char *roBuf = (char*)outBuf; /* pointer to current write position */
/* header */
uint64_t * i64Buf = (uint64_t *) (roBuf);
i64Buf[0] = htobe64t((uint64_t)nbytes); /* Store decompressed size in be format */
roBuf += 8;
uint32_t *i32Buf = (uint32_t *) (roBuf);
i32Buf[0] = htobe32t((uint32_t)blockSize); /* Store the block size in be format */
roBuf += 4;
size_t outSize = 12; /* size of the output buffer. Header size (12 bytes) is included */
for(size_t block = 0; block < nBlocks; ++block)
{
size_t origWritten = block*blockSize;
if(nbytes - origWritten < blockSize) /* the last block may be < blockSize */
blockSize = nbytes - origWritten;
uint32_t compBlockSize = LZ4_compress(rpos, roBuf+4, blockSize); /// reserve space for compBlockSize
if(!compBlockSize)
goto error;
if(compBlockSize >= blockSize) /* compression did not save any space, do a memcpy instead */
{
compBlockSize = blockSize;
memcpy(roBuf+4, rpos, blockSize);
}
i32Buf = (uint32_t *) (roBuf);
i32Buf[0] = htobe32t((uint32_t)compBlockSize); /* write blocksize */
roBuf += 4;
rpos += blockSize; /* advance read pointer */
roBuf += compBlockSize; /* advance write pointer */
outSize += compBlockSize + 4;
}
free(*buf);
*buf = outBuf;
*buf_size = outSize;
outBuf = NULL;
ret_value = outSize;
}
done:
if(outBuf)
free(outBuf);
return ret_value;
error:
if(outBuf)
free(outBuf);
outBuf = NULL;
return 0;
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** The format of compressed block - see CompressedStream.h
*/
switch (method)
{
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress_default(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size));
else
compressed_size = header_size + LZ4_compress_HC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size),
0);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
bool JobTicket::DoJob(const wxRect &rect)
{
unsigned char *bit_array[10];
for(int i=0 ; i < 10 ; i++)
bit_array[i] = 0;
wxRect ncrect(rect);
bit_array[0] = level0_bits;
level0_bits = NULL;
if(!bit_array[0]) {
// Grab a copy of the level0 chart bits
// we could alternately subsample grabbing leveln chart bits
// directly here to speed things up...
ChartBase *pchart;
int index;
if(ChartData){
index = ChartData->FinddbIndex( m_ChartPath );
pchart = ChartData->OpenChartFromDBAndLock(index, FULL_INIT );
if(pchart && ChartData->IsChartLocked( index )){
ChartBaseBSB *pBSBChart = dynamic_cast<ChartBaseBSB*>( pchart );
if( pBSBChart ) {
bit_array[0] = (unsigned char *) malloc( ncrect.width * ncrect.height * 4 );
pBSBChart->GetChartBits( ncrect, bit_array[0], 1 );
}
ChartData->UnLockCacheChart(index);
}
else
bit_array[0] = NULL;
}
}
//OK, got the bits?
int ssize, dim;
if(!bit_array[0] )
return false;
// Fill in the rest of the private uncompressed array
dim = g_GLOptions.m_iTextureDimension;
dim /= 2;
for( int i = 1 ; i < g_mipmap_max_level+1 ; i++ ){
size_t nmalloc = wxMax(dim * dim * 3, 4*4*3);
bit_array[i] = (unsigned char *) malloc( nmalloc );
MipMap_24( 2*dim, 2*dim, bit_array[i - 1], bit_array[i] );
dim /= 2;
}
int texture_level = 0;
for( int level = level_min_request; level < g_mipmap_max_level+1 ; level++ ){
int dim = TextureDim(level);
int size = TextureTileSize(level, true);
unsigned char *tex_data = (unsigned char*)malloc(size);
if(g_raster_format == GL_COMPRESSED_RGB_S3TC_DXT1_EXT) {
// color range fit is worse quality but twice as fast
int flags = squish::kDxt1 | squish::kColourRangeFit;
if( g_GLOptions.m_bTextureCompressionCaching) {
/* use slower cluster fit since we are building the cache for
* better quality, this takes roughly 25% longer and uses about
* 10% more disk space (result doesn't compress as well with lz4) */
flags = squish::kDxt1 | squish::kColourClusterFit;
}
OCPNStopWatch sww;
squish::CompressImageRGBpow2_Flatten_Throttle_Abort( bit_array[level], dim, dim, tex_data, flags,
true, b_throttle ? throttle_func : 0, &sww, b_abort );
} else if(g_raster_format == GL_ETC1_RGB8_OES)
CompressDataETC(bit_array[level], dim, size, tex_data, b_abort);
else if(g_raster_format == GL_COMPRESSED_RGB_FXT1_3DFX) {
if(!CompressUsingGPU(bit_array[level], dim, size, tex_data, texture_level, binplace)) {
b_abort = true;
break;
}
if(binplace)
g_tex_mem_used += size;
texture_level++;
}
comp_bits_array[level] = tex_data;
if(b_abort){
for( int i = 0; i < g_mipmap_max_level+1; i++ ){
free( bit_array[i] );
bit_array[i] = 0;
}
return false;
}
}
// All done with the uncompressed data in the thread
for( int i = 0; i < g_mipmap_max_level+1; i++ ) {
free( bit_array[i] );
bit_array[i] = 0;
}
if(b_throttle)
wxThread::Sleep(1);
if(b_abort)
return false;
if(bpost_zip_compress) {
int max_compressed_size = LZ4_COMPRESSBOUND(g_tile_size);
for(int level = level_min_request; level < g_mipmap_max_level+1 ; level++){
if(b_abort)
return false;
unsigned char *compressed_data = (unsigned char *)malloc(max_compressed_size);
int csize = TextureTileSize(level, true);
char *src = (char *)comp_bits_array[level];
int compressed_size = LZ4_compressHC2( src, (char *)compressed_data, csize, 4);
// shrink buffer to actual size.
// This will greatly reduce ram usage, ratio usually 10:1
// there might be a more efficient way than realloc...
compressed_data = (unsigned char*)realloc(compressed_data, compressed_size);
compcomp_bits_array[level] = compressed_data;
compcomp_size_array[level] = compressed_size;
}
}
return true;
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** Формат сжатого блока - см. CompressedStream.h
*/
switch (method)
{
case CompressionMethod::QuickLZ:
{
#ifdef USE_QUICKLZ
compressed_buffer.resize(uncompressed_size + QUICKLZ_ADDITIONAL_SPACE);
compressed_size = qlz_compress(
working_buffer.begin(),
&compressed_buffer[0],
uncompressed_size,
qlz_state.get());
compressed_buffer[0] &= 3;
compressed_buffer_ptr = &compressed_buffer[0];
break;
#else
throw Exception("QuickLZ compression method is disabled", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
#endif
}
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
else
compressed_size = header_size + LZ4_compressHC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
uint128 checksum = CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
