void teDecodeFile(const teString & from, const teString & to, u1 localPath)
{
c8 * inBuffer = (c8*)TE_ALLOCATE(LZ4_compressBound(CHUNKSIZE));
c8 * outBuffer = (c8*)TE_ALLOCATE(CHUNKSIZE);
teDecodeFile(from, to, CHUNKSIZE, inBuffer, LZ4_compressBound(CHUNKSIZE), outBuffer, CHUNKSIZE, localPath);
TE_FREE(inBuffer);
TE_FREE(outBuffer);
}
int
lz4_buf_extra(uint64_t buflen)
{
if (buflen > LZ4_MAX_CHUNK)
buflen = LZ4_MAX_CHUNK;
return (LZ4_compressBound(buflen) - buflen + sizeof(int));
}
static VALUE rubylz4_raw_compressHC(VALUE self, VALUE input, VALUE compression_level_value)
{
Check_Type(input, RUBY_T_STRING);
const char* input_data = RSTRING_PTR(input);
int input_size = RSTRING_LEN(input);
Check_Type(compression_level_value, RUBY_T_FIXNUM);
int compression_level = FIX2INT(compression_level_value);
// do compress
int max_compressed_size = LZ4_compressBound(input_size);
VALUE output = rb_str_new(NULL, max_compressed_size);
char* output_data = RSTRING_PTR(output);
int compressed_size = LZ4_compress_HC(input_data, output_data,
input_size, max_compressed_size,
compression_level);
if (compressed_size == 0) {
rb_raise(rb_eRuntimeError, "%s", "compress failed");
} else {
rb_str_resize(output, compressed_size);
return output;
}
}
void store_table(ubyte *table, char color)
{
FILE *F;
char name[64];
if (!lz4_buf) {
lz4_buf = malloc(4 + LZ4_compressBound(COPYSIZE));
if (!lz4_buf) {
printf("Out of memory.\n");
exit(0);
}
}
sprintf(name, "%s.%c", tablename, color);
if (!(F = fopen(name, "wb"))) {
printf("Could not open %s for writing.\n", name);
exit(-1);
}
ubyte *ptr = table;
long64 total = size;
while (total > 0) {
int chunk = COPYSIZE;
if (total < chunk) chunk = total;
total -= chunk;
uint32 lz4_size = LZ4_compress((char *)ptr, lz4_buf + 4, chunk);
ptr += chunk;
*(uint32 *)lz4_buf = lz4_size;
fwrite(lz4_buf, 1, lz4_size + 4, F);
}
fclose(F);
}
size_t lz4Compress(std::shared_ptr<uint8_t> src, size_t uncompressedBytes,
std::shared_ptr<uint8_t>& dst,
bool /*highCompression*/)
{
if (uncompressedBytes > size_t(std::numeric_limits<int>::max()))
throw std::runtime_error("Input data too big for LZ4 (max ~1.9GB)");
int const inputSize = static_cast<int>(uncompressedBytes);
int const upperBound = LZ4_compressBound(inputSize);
if (upperBound < 0)
throw std::runtime_error("Input data too big for LZ4 (max ~1.9GB)");
dst.reset(new uint8_t[size_t(upperBound)], nonstd::DeleteArray<uint8_t>());
// NOTE: LZ4_compressHC compresses stronger and slower but decompresses
// faster. It causes some bad memory accesses that's why we disable it here
// for now. (e.g. processing the HeadAneurysm dataset with brick size 256 at
// brick index 8)
int compressedBytes = 0;
//if (!highCompression)
compressedBytes = LZ4_compress((const char*)src.get(),
(char*)dst.get(),
inputSize);
/*
else
compressedBytes = LZ4_compressHC((const char*)src.get(),
(char*)dst.get(),
inputSize);
*/
assert(compressedBytes >= 0);
return compressedBytes;
}
void dnCalcDelta( const snapshot_t *olds, const snapshot_t *news, snapshotDelta_t *delta ) {
unsigned char bitmask[ sizeof( snapshot_t ) / 8 ] = { 0 };
std::vector<char> stream;
char *oldp = (char*)olds;
char *newp = (char*)news;
stream.reserve( 256 * 1024 );
for ( int i = 0; i < sizeof( snapshot_t ); i++, oldp++, newp++ ) {
if ( *newp != *oldp ) {
bitmask[ i/8 ] |= ( 1 << i%8 ); /* mark changed byte */
stream.push_back( *newp ); /* put new value to the stream */
}
}
std::vector<char> buffer;
buffer.resize( LZ4_compressBound( sizeof( bitmask ) ) );
delta->bitmaskSize = LZ4_compress( (char*)&bitmask[0], &buffer[0], sizeof( bitmask ));
delta->bitmask = (char*)malloc( delta->bitmaskSize );
memcpy( delta->bitmask, &buffer[0], delta->bitmaskSize );
delta->streamSize = stream.size();
delta->stream = (char*)malloc( delta->streamSize );
memcpy( delta->stream, &stream[0], delta->streamSize );
}
static bool compress( T2 &buffer_out, const T1 &buffer_in, bool highest_compression = true )
{
static const bool verbose = false;
bool ret = false;
if( 1 )
{
// resize to worst case
buffer_out.resize( LZ4_compressBound((int)(buffer_in.size())) );
// compress
size_t compressed_size = highest_compression ?
LZ4_compressHC( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() )
:
LZ4_compress( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() );
ret = ( compressed_size > 0 && compressed_size < buffer_in.size() );
if( ret )
{
// if ok, resize properly to unused space
buffer_out.resize( compressed_size );
}
if( verbose )
{
// std::cout << moon9::echo( ret, compressed_size, buffer_in.size() );
}
}
return ret;
}
std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {
std::unique_ptr<IOBuf> clone;
if (data->isChained()) {
// LZ4 doesn't support streaming, so we have to coalesce
clone = data->clone();
clone->coalesce();
data = clone.get();
}
uint32_t extraSize = encodeSize() ? kMaxVarintLength64 : 0;
auto out = IOBuf::create(extraSize + LZ4_compressBound(data->length()));
if (encodeSize()) {
encodeVarintToIOBuf(data->length(), out.get());
}
int n;
if (highCompression_) {
n = LZ4_compressHC(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
} else {
n = LZ4_compress(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
}
CHECK_GE(n, 0);
CHECK_LE(n, out->capacity());
out->append(n);
return out;
}
/**
* By the time this function gets called, g_tw_delta_sz should be as large
* as it will ever get.
*/
static void tw_delta_alloc(tw_pe *pe) {
g_tw_delta_sz = LZ4_compressBound(g_tw_delta_sz);
pe->delta_buffer[0] = (unsigned char *)tw_calloc(TW_LOC, "Delta buffers", g_tw_delta_sz, 1);
pe->delta_buffer[1] = (unsigned char *)tw_calloc(TW_LOC, "Delta buffers", g_tw_delta_sz, 1);
pe->delta_buffer[2] = (unsigned char *)tw_calloc(TW_LOC, "Delta buffers", g_tw_delta_sz, 1);
}
bool CompressStream(Serializer& dest, Deserializer& src)
{
unsigned srcSize = src.GetSize() - src.GetPosition();
// Prepend the source and dest. data size in the stream so that we know to buffer & uncompress the right amount
if (!srcSize)
{
dest.WriteUInt(0);
dest.WriteUInt(0);
return true;
}
unsigned maxDestSize = LZ4_compressBound(srcSize);
SharedArrayPtr<unsigned char> srcBuffer(new unsigned char[srcSize]);
SharedArrayPtr<unsigned char> destBuffer(new unsigned char[maxDestSize]);
if (src.Read(srcBuffer, srcSize) != srcSize)
return false;
unsigned destSize = LZ4_compressHC((const char*)srcBuffer.Get(), (char*)destBuffer.Get(), srcSize);
bool success = true;
success &= dest.WriteUInt(srcSize);
success &= dest.WriteUInt(destSize);
success &= dest.Write(destBuffer, destSize) == destSize;
return success;
}
void* CBAWMeshWriter::compressWithLz4AndTryOnStack(const void* _input, size_t _inputSize, void* _stack, size_t _stackSize, size_t& _outComprSize) const
{
void* data = _stack;
size_t dstSize = _stackSize;
size_t compressedSize = 0;
const int lz4CompressBound = LZ4_compressBound(_inputSize);
if (lz4CompressBound) // if input is not too large
{
if (lz4CompressBound > _stackSize)
{
dstSize = asset::BlobHeaderV1::calcEncSize(lz4CompressBound);
data = _IRR_ALIGNED_MALLOC(dstSize,_IRR_SIMD_ALIGNMENT);
}
compressedSize = LZ4_compress_default((const char*)_input, (char*)data, _inputSize, dstSize);
}
if (!compressedSize) // if compression did not succeed
{
if (data != _stack)
_IRR_ALIGNED_FREE(data);
compressedSize = _inputSize;
data = const_cast<void*>(_input);
#ifdef _DEBUG
os::Printer::log("Failed to compress (lz4). Blob exported without compression.", ELL_WARNING);
#endif
}
_outComprSize = compressedSize;
return data;
}
int
lz4_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
int level, uchar_t chdr, void *data)
{
int rv;
struct lz4_params *lzdat = (struct lz4_params *)data;
int _srclen = srclen;
uchar_t *dst2;
if (lzdat->level == 1) {
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
} else if (lzdat->level == 2) {
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
if (rv == 0 || rv > *dstlen) {
return (-1);
}
dst2 = (uchar_t *)slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv));
*((int *)dst2) = htonl(rv);
rv = LZ4_compressHC((const char *)dst, (char *)(dst2 + sizeof (int)), rv);
if (rv != 0) {
rv += sizeof (int);
memcpy(dst, dst2, rv);
}
slab_free(NULL, dst2);
} else {
rv = LZ4_compressHC((const char *)src, (char *)dst, _srclen);
}
if (rv == 0) {
return (-1);
}
*dstlen = rv;
return (0);
}
void teCompressFile(const teString & from, const teString & to, u32 chunkSize, c8 * chunkInputBuffer, u32 chunkInputBufferSize, c8 * chunkOutputBuffer, u32 chunkOutputBufferSize, u1 highCompression, u1 localPath)
{
TE_ASSERT(chunkInputBufferSize >= chunkSize);
TE_ASSERT(chunkOutputBufferSize >= LZ4_compressBound(chunkSize));
teLZ4CompressionFunction compressionFunction = highCompression ? LZ4_compressHC : LZ4_compress;
IBuffer * fileInput = GetFileManager()->OpenFile(from, CFileBuffer::FWM_READ, localPath);
if(!fileInput)
return;
IBuffer * fileOutput = GetFileManager()->OpenFile(to, CFileBuffer::FWM_WRITE, localPath);
if(!fileOutput)
{
TE_SAFE_DROP(fileInput);
return;
}
fileInput->SetStreamMode(true);
fileOutput->SetStreamMode(true);
fileInput->Lock(BLT_READ);
fileInput->SetPosition(0);
fileOutput->Lock(BLT_WRITE);
fileOutput->SetPosition(0);
u32 magicNumber = ARCHIVE_MAGICNUMBER;
fileOutput->Write(&magicNumber, ARCHIVE_MAGICNUMBER_SIZE);
c8 * inBuffer = chunkInputBuffer;
c8 * outBuffer = chunkOutputBuffer;
u32 fileSize = 0;
while(true)
{
u32 inSize = teMin(chunkSize, fileInput->GetSize() - fileInput->GetPosition());
fileInput->Read(inBuffer, inSize);
fileSize += inSize;
if(!inSize)
break;
u32 outSize = compressionFunction(inBuffer, outBuffer + 4, inSize);
*(u32*)outBuffer = outSize;
fileOutput->Write(outBuffer, outSize + 4);
}
fileInput->Unlock();
fileOutput->Unlock();
TE_SAFE_DROP(fileInput);
TE_SAFE_DROP(fileOutput);
}
static VALUE rubylz4_raw_compressBound(VALUE input_size_value)
{
Check_Type(input_size_value, RUBY_T_FIXNUM);
int input_size = FIX2INT(input_size_value);
int max_compressed_size = LZ4_compressBound(input_size);
return INT2FIX(max_compressed_size);
}
static ZEND_FUNCTION(lz4_compress)
{
zval *data;
char *output;
int output_len, data_len;
zend_bool high = 0;
char *extra = NULL;
int extra_len = -1;
int offset = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|bs", &data, &high,
&extra, &extra_len) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"lz4_compress : expects parameter to be string.");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
offset = extra_len;
} else {
offset = sizeof(int);
}
data_len = Z_STRLEN_P(data);
output = (char *)emalloc(LZ4_compressBound(data_len) + offset);
if (!output) {
zend_error(E_WARNING, "lz4_compress : memory error");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
memcpy(output, extra, offset);
} else {
/* Set the data length */
memcpy(output, &data_len, offset);
}
if (high) {
output_len = LZ4_compressHC(Z_STRVAL_P(data), output + offset, data_len);
} else {
output_len = LZ4_compress(Z_STRVAL_P(data), output + offset, data_len);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
RETVAL_STRINGL(output, output_len + offset, 1);
}
efree(output);
}
compressedSnapshot_t* dnCompressSnapshot( const snapshot_t *snapshot ) {
compressedSnapshot_t *result;
char *buffer = (char*)malloc( LZ4_compressBound( sizeof( snapshot_t ) ) );
int size = LZ4_compress( (char*)snapshot, buffer, sizeof( snapshot_t ) );
result = (compressedSnapshot_t*)malloc( sizeof( result->size ) + size );
result->size = size;
memcpy( &result->data[0], buffer, size );
free( buffer );
return result;
}
int lz4compress_size( const unsigned char* data, unsigned size ){
auto max_size = LZ4_compressBound( size );
std::vector<unsigned char> buffer( max_size );
return LZ4_compress_HC(
(const char*)data, (char*)buffer.data()
, size, buffer.size()
, LZ4HC_CLEVEL_MAX //TODO: Higher?
);
}
size_t bshuf_compress_lz4_bound(const size_t size,
const size_t elem_size, size_t block_size) {
size_t bound, leftover;
if (block_size == 0) {
block_size = bshuf_default_block_size(elem_size);
}
if (block_size < 0 || block_size % BSHUF_BLOCKED_MULT) return -81;
// Note that each block gets a 4 byte header.
// Size of full blocks.
bound = (LZ4_compressBound(block_size * elem_size) + 4) * (size / block_size);
// Size of partial blocks, if any.
leftover = ((size % block_size) / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT;
if (leftover) bound += LZ4_compressBound(leftover * elem_size) + 4;
// Size of uncompressed data not fitting into any blocks.
bound += (size % BSHUF_BLOCKED_MULT) * elem_size;
return bound;
}
void LZ4CompressStream::Alloc()
{
int N = 16;
int sz = concurrent ? N * BLOCK_BYTES : BLOCK_BYTES;
buffer.Alloc(sz);
outbuf.Alloc(N * LZ4_compressBound(BLOCK_BYTES));
outsz.Alloc(N);
Stream::buffer = ~buffer;
wrlim = ~buffer + sz;
ptr = ~buffer;
}
void LZ4CompressStream::FlushOut()
{
if(ptr == (byte *)~buffer)
return;
CoWork co;
int osz = LZ4_compressBound(BLOCK_BYTES);
byte *t = ~outbuf;
int ii = 0;
for(byte *s = ~buffer; s < ptr; s += BLOCK_BYTES) {
int origsize = min((int)BLOCK_BYTES, int(ptr - s));
#ifdef _MULTITHREADED
if(concurrent)
co & [=] {
outsz[ii] = LZ4_compress_default((char *)s, (char *)t, origsize, osz);
};
else
#endif
outsz[ii] = LZ4_compress_default((char *)s, (char *)t, origsize, osz);
ii++;
t += osz;
}
if(concurrent)
co.Finish();
byte *s = ~buffer;
t = ~outbuf;
for(int i = 0; i < ii; i++) {
int origsize = min((int)BLOCK_BYTES, int(ptr - s));
int clen = outsz[i];
if(clen < 0) {
SetError();
return;
}
if(clen >= origsize || clen == 0) {
out->Put32le(0x80000000 | origsize);
out->Put(s, origsize);
}
else {
out->Put32le(clen);
out->Put(t, clen);
}
s += BLOCK_BYTES;
t += osz;
}
int origsize = int(ptr - ~buffer);
xxh.Put(~buffer, origsize);
pos += origsize;
ptr = ~buffer;
}
size_t block_writer::write_block(size_t segment_id,
size_t column_id,
char* data,
block_info block) {
DASSERT_LT(segment_id, m_index_info.nsegments);
DASSERT_LT(column_id, m_index_info.columns.size());
DASSERT_TRUE(m_output_files[segment_id] != nullptr);
// try to compress the data
size_t compress_bound = LZ4_compressBound(block.block_size);
auto compression_buffer = m_buffer_pool.get_new_buffer();
compression_buffer->resize(compress_bound);
char* cbuffer = compression_buffer->data();
size_t clen = compress_bound;
clen = LZ4_compress(data, cbuffer, block.block_size);
char* buffer_to_write = NULL;
size_t buffer_to_write_len = 0;
if (clen < COMPRESSION_DISABLE_THRESHOLD * block.block_size) {
// compression has a benefit!
block.flags |= LZ4_COMPRESSION;
block.length = clen;
buffer_to_write = cbuffer;
buffer_to_write_len = clen;
} else {
// compression has no benefit! do not compress!
// unset LZ4
block.flags &= (~(size_t)LZ4_COMPRESSION);
block.length = block.block_size;
buffer_to_write = data;
buffer_to_write_len = block.block_size;
}
size_t padding = ((buffer_to_write_len + 4095) / 4096) * 4096 - buffer_to_write_len;
ASSERT_LT(padding, 4096);
// write!
m_output_file_locks[segment_id].lock();
block.offset = m_output_bytes_written[segment_id];
m_output_bytes_written[segment_id] += buffer_to_write_len + padding;
m_index_info.columns[column_id].segment_sizes[segment_id] += block.num_elem;
m_output_files[segment_id]->write(buffer_to_write, buffer_to_write_len);
m_output_files[segment_id]->write(padding_bytes, padding);
m_blocks[segment_id][column_id].push_back(block);
m_output_file_locks[segment_id].unlock();
m_buffer_pool.release_buffer(std::move(compression_buffer));
if (!m_output_files[segment_id]->good()) {
log_and_throw_io_failure("Fail to write. Disk may be full.");
}
return buffer_to_write_len;
}
std::pair<std::unique_ptr<char[]>, size_t> compress(const void* data, size_t dataSize, int level)
{
if (dataSize == 0) return std::make_pair(std::unique_ptr<char[]>(), 0);
int csizeBound = LZ4_compressBound(dataSize);
std::unique_ptr<char[]> cdata(new char[csizeBound]);
int csize = (level == 0)
? LZ4_compress_default(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound)
: LZ4_compress_HC(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound, level);
assert(csize >= 0 && csize <= csizeBound);
return std::make_pair(std::move(cdata), csize);
}
void SDRdaemonBufferOld::updateLZ4Sizes(uint32_t frameSize)
{
uint32_t maxInputSize = LZ4_compressBound(frameSize);
if (m_lz4InBuffer) {
delete[] m_lz4InBuffer;
}
m_lz4InBuffer = new uint8_t[maxInputSize];
if (m_lz4OutBuffer) {
delete[] m_lz4OutBuffer;
}
m_lz4OutBuffer = new uint8_t[frameSize];
}
static PyObject *compress_hc(PyObject *self, PyObject *args, PyObject *keywds)
{
(void)self;
const char *source;
int source_size;
int compression_level;
static char *kwlist[] = {"source", "compression_level", NULL};
if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#i", kwlist,
&source, &source_size,
&compression_level)) {
return NULL;
}
if (source_size > LZ4_MAX_INPUT_SIZE) {
PyErr_Format(PyExc_ValueError, "input data is %d bytes, which is larger than the maximum supported size of %d bytes", source_size, LZ4_MAX_INPUT_SIZE);
return NULL;
}
int dest_size = LZ4_compressBound(source_size);
PyObject *py_dest = PyBytes_FromStringAndSize(NULL, dest_size);
if (py_dest == NULL) {
return NULL;
}
char *dest = PyBytes_AS_STRING(py_dest);
if (source_size > 0) {
int compressed_size = LZ4_compress_HC(source, dest, source_size, dest_size, compression_level);
if (compressed_size <= 0) {
Py_DECREF(py_dest);
PyErr_Format(PyExc_RuntimeError, "LZ4_compress_HC failed with code: %d", compressed_size);
return NULL;
}
/* The actual compressed size might be less than we allocated
(we allocated using a worst case guess). If the actual size is
less than 75% of what we allocated, then it's worth performing an
expensive resize operation to reclaim some space. */
if (compressed_size < (dest_size / 4) * 3) {
_PyBytes_Resize(&py_dest, compressed_size);
} else {
Py_SIZE(py_dest) = compressed_size;
}
}
return py_dest;
}
void
Compress(const std::vector<T>& data, BufferT& out)
{
ssize_t countBS = sizeof(T)*data.size();
// allocate as much memory as we need
int maxOut = LZ4_compressBound(countBS);
out.resize(maxOut);
int r = LZ4_compress((const char*)&data.front(), (char*)&out.front(), countBS);
if(r == 0) {
Util::fire_exception("lz4 compression failed");
}
// trim to really used size
out.resize(r);
}
static inline void compress(I begin,I end,O out)
{
unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
char *buf = new char[bufLen * 2];
char *buf2 = buf + bufLen;
try {
I inp(begin);
for(;;) {
unsigned int readLen = 0;
while ((readLen < ZT_COMPRESSION_BLOCK_SIZE)&&(inp != end)) {
buf[readLen++] = (char)*inp;
++inp;
}
if (!readLen)
break;
uint32_t l = hton((uint32_t)readLen);
out((const void *)&l,4); // original size
if (readLen < 32) { // don't bother compressing itty bitty blocks
l = 0; // stored
out((const void *)&l,4);
out((const void *)buf,readLen);
continue;
}
int lz4CompressedLen = LZ4_compressHC(buf,buf2,(int)readLen);
if ((lz4CompressedLen <= 0)||(lz4CompressedLen >= (int)readLen)) {
l = 0; // stored
out((const void *)&l,4);
out((const void *)buf,readLen);
continue;
}
l = hton((uint32_t)lz4CompressedLen); // lz4 only
out((const void *)&l,4);
out((const void *)buf2,(unsigned int)lz4CompressedLen);
}
delete [] buf;
} catch ( ... ) {
delete [] buf;
throw;
}
}
void LZ4CompressedWriterAdapter::flush_buffer()
{
const size_t max_compressed_buffer_size =
static_cast<size_t>(LZ4_compressBound(static_cast<int>(m_buffer_index)));
ensure_minimum_size(m_compressed_buffer, max_compressed_buffer_size);
const int compressed_buffer_size =
LZ4_compress(
reinterpret_cast<const char*>(&m_buffer[0]),
reinterpret_cast<char*>(&m_compressed_buffer[0]),
static_cast<int>(m_buffer_index));
m_file.write(static_cast<uint64>(m_buffer_index));
m_file.write(static_cast<uint64>(compressed_buffer_size));
m_file.write(&m_compressed_buffer[0], compressed_buffer_size);
m_buffer_index = 0;
}
int
G_lz4_compress(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz)
{
int err, nbytes, buf_sz;
unsigned char *buf;
/* Catch errors early */
if (src == NULL || dst == NULL)
return -1;
/* Don't do anything if either of these are true */
if (src_sz <= 0 || dst_sz <= 0)
return 0;
/* Output buffer has to be larger for single pass compression */
buf_sz = LZ4_compressBound(src_sz);
if (NULL == (buf = (unsigned char *)
G_calloc(buf_sz, sizeof(unsigned char))))
return -1;
/* Do single pass compression */
err = LZ4_compress_default((char *)src, (char *)buf, src_sz, buf_sz);
if (err <= 0) {
G_free(buf);
return -1;
}
if (err >= src_sz) {
/* compression not possible */
G_free(buf);
return -2;
}
/* bytes of compressed data is return value */
nbytes = err;
/* Copy the data from buf to dst */
for (err = 0; err < nbytes; err++)
dst[err] = buf[err];
G_free(buf);
return nbytes;
}
static ZEND_FUNCTION(horde_lz4_compress)
{
zval *data;
char *output;
int header_offset = (sizeof(headerid) + sizeof(int));
int output_len, data_len;
zend_bool high = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|b", &data, &high) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"horde_lz4_compress: uncompressed data must be a string.");
RETURN_FALSE;
}
data_len = Z_STRLEN_P(data);
output = (char *)emalloc(LZ4_compressBound(data_len) + header_offset);
if (!output) {
zend_error(E_WARNING, "horde_lz4_compress: memory error");
RETURN_FALSE;
}
*output = headerid;
memcpy(output + sizeof(headerid), &data_len, sizeof(int));
if (high) {
output_len = LZ4_compressHC(Z_STRVAL_P(data), output + header_offset, data_len);
} else {
output_len = LZ4_compress(Z_STRVAL_P(data), output + header_offset, data_len);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
RETVAL_STRINGL(output, output_len + header_offset, 1);
}
efree(output);
}
static yrmcds_error send_data(
yrmcds* c, yrmcds_command cmd, const char* key, size_t key_len,
const char* data, size_t data_len, uint32_t flags, uint32_t expire,
uint64_t cas, uint32_t* serial) {
if( c == NULL || key == NULL || key_len == 0 ||
data == NULL || data_len == 0 )
return YRMCDS_BAD_ARGUMENT;
int compressed = 0;
#ifdef LIBYRMCDS_USE_LZ4
if( (c->compress_size > 0) && (data_len > c->compress_size) ) {
if( flags & YRMCDS_FLAG_COMPRESS )
return YRMCDS_BAD_ARGUMENT;
size_t bound = (size_t)LZ4_compressBound((int)data_len);
char* new_data = (char*)malloc(bound + sizeof(uint32_t));
if( new_data == NULL )
return YRMCDS_OUT_OF_MEMORY;
uint32_t new_size =
(uint32_t)LZ4_compress(data, new_data + sizeof(uint32_t),
(int)data_len);
if( new_size == 0 ) {
free(new_data);
return YRMCDS_COMPRESS_FAILED;
}
hton32((uint32_t)data_len, new_data);
flags |= YRMCDS_FLAG_COMPRESS;
data_len = sizeof(uint32_t) + new_size;
data = new_data;
compressed = 1;
}
#endif // LIBYRMCDS_USE_LZ4
char extras[8];
hton32(flags, extras);
hton32(expire, &extras[4]);
yrmcds_error e = send_command(c, cmd, cas, serial, key_len, key,
sizeof(extras), extras, data_len, data);
if( compressed )
free((void*)data);
return e;
}
