static PyObject *
python_decompress(PyObject *self, PyObject *args) {
char *input, *output;
Py_ssize_t inlen;
long outlen;
PyObject *result;
if (!PyArg_ParseTuple(args, "s#l", &input, &inlen, &outlen))
return NULL;
if (outlen < 0) {
PyErr_SetString(PyExc_ValueError, "max_len cannot be less than 0");
return NULL;
}
output = (char *)malloc(outlen);
outlen = lzf_decompress(input, inlen, output, outlen);
if (outlen)
result = PYBYTES_FSAS(output, outlen);
else {
if (errno == EINVAL) {
PyErr_SetString(PyExc_ValueError, "error in compressed data");
free(output);
return NULL;
}
Py_XINCREF(Py_None);
result = Py_None;
}
free(output);
return result;
}
char* loadLzfStringObject()
{
unsigned int slen, clen;
char* c, *s;
if ((clen = loadLength(NULL)) == REDIS_RDB_LENERR) {
return NULL;
}
if ((slen = loadLength(NULL)) == REDIS_RDB_LENERR) {
return NULL;
}
c = malloc(clen);
if (!readBytes(c, clen)) {
free(c);
return NULL;
}
s = malloc(slen + 1);
if (lzf_decompress(c, clen, s, slen) == 0) {
free(c);
free(s);
return NULL;
}
free(c);
return s;
}
static char* rdb_read_lzf_string(int fd) {
int32_t clen, len;
char *cstr, *str;
/*
* 1. load compress length.
* 2. load raw length.
* 3. load lzf_string, and use lzf_decompress to decode.
*/
if ((clen = rdb_read_store_len(fd, NULL)) == REDIS_RDB_LENERR) {
return NULL;
}
if ((len = rdb_read_store_len(fd, NULL)) == REDIS_RDB_LENERR) {
return NULL;
}
cstr = malloc(clen);
str = malloc(len + 1);
if (!cstr || !str) {
logger(ERROR, "Exited, as malloc failed at load lzf string.\n");
exit(1);
}
int ret;
if ((ret = rdb_crc_read(fd, cstr, clen)) == 0) goto READLZFERR;
if ((ret = lzf_decompress(cstr, clen, str, len)) == 0) goto READLZFERR;
str[len] = '\0';
free(cstr);
return str;
READLZFERR:
free(cstr);
free(str);
return NULL;
}
inline int
mummy_string_decompress(mummy_string *str, char free_buffer, char *rc) {
uint32_t ucsize;
char *output;
*rc = 0;
/* not compressed */
if (0 == (str->data[0] & 0x80)) return 0;
ucsize = ntohl(*(uint32_t *)(str->data + 1));
if (NULL == (output = malloc(ucsize + 2)))
return ENOMEM;
output[0] = str->data[0] & 0x7f;
if (ucsize != lzf_decompress(
str->data + 5, str->len - 5, output + 1, ucsize + 1)) {
if (E2BIG == errno || EINVAL == errno) {
free(output);
return errno;
}
return -2;
}
*rc = 1;
if (free_buffer) free(str->data);
str->data = output;
str->len = ucsize + 1;
return 0;
}
extern int
lzf_decode
(const unsigned char *in_start, size_t in_len, unsigned char *out_start, size_t *pout_len, size_t out_max)
{
*pout_len = (size_t)lzf_decompress (in_start, (unsigned)in_len, out_start, (unsigned)out_max);
return *pout_len != 0;
}
static char *streamLZFTextToString(uchar *s, uint32 *len) {
uint32 clen;
uint32 oclen = streamIntToUInt(s, &clen, &fdum);
char *buf = malloc(oclen); /* FREE ME 035 */
int llen = *len - clen;
*len = lzf_decompress(s + clen, llen, buf, oclen);
return buf;
}
bool SFCompressLzf::Uncompress(BYTE* pDest, int& DestLen, BYTE* pSrc, int SrcLen)
{
int Size = lzf_decompress(pSrc, SrcLen, pDest, DestLen);
DestLen = Size;
//intfastlz_compress_level(int level, const void* input, int length, void* output);
return Size > 0;
}
int pl_decompress(pl_buffer *input, pl_buffer **output, const char *key, int check_compression) {
pl_buffer *result;
unsigned int data_size, real_size, *header,
header_size, crc32_ptr_compresed, i;
char *data;
if(!input)
return PL_DECOMP_INVALID;
if(check_compression && pl_check_compression(input) != PL_CHECK_COMPRESSESED) {
return PL_DECOMP_CHECK_COMP;
}
*output = NULL;
header = (unsigned int*)((char*)input->buffer + pl_header_magic_text_size);
// skip version
++header;
// get crc32 of compressed
crc32_ptr_compresed = *header++;
// get real size
real_size = *header++;
// get compressed size
data_size = *header;
header_size = PL_HEADER_ITEMS_SIZE + pl_header_magic_text_size;
// check buffer crc
if(crc32_ptr_compresed != calculate_crc32((char*)input->buffer + header_size, data_size)) {
return PL_DECOMP_WRONG_KEY;
}
// decompress
result = pl_buf_create(real_size + 16); // with dummy
if(!result) {
return PL_DECOMP_MEMORY;
}
result->length = lzf_decompress((char*)input->buffer + header_size, data_size,
result->buffer, result->allocated_size);
if(result->length == 0) {
pl_buf_destroy(result);
return PL_DECOMP_DECOMPRESS;
}
result->length = real_size;
*output = result;
return PL_DECOMP_OK;
}
static sds rdbLoadLzfStringObject(FILE*fp) {
unsigned int len, clen;
unsigned char *c = NULL;
sds val = NULL;
if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
if ((c = zmalloc(clen)) == NULL) goto err;
if ((val = sdsnewlen(NULL,len)) == NULL) goto err;
if (freadCheck(c,clen,1,fp) == 0) goto err;
if (lzf_decompress(c,clen,val,len) == 0) goto err;
zfree(c);
return val;
err:
zfree(c);
sdsfree(val);
return NULL;
}
static char* loadLzfStringObject() {
unsigned int slen, clen;
char *c, *s;
if ((clen = loadLength(NULL)) == RDB_LENERR) return NULL;
if ((slen = loadLength(NULL)) == RDB_LENERR) return NULL;
c = zmalloc(clen);
if (!readBytes(c, clen)) {
zfree(c);
return NULL;
}
s = zmalloc(slen+1);
if (lzf_decompress(c,clen,s,slen) == 0) {
zfree(c); zfree(s);
return NULL;
}
zfree(c);
return s;
}
int64_t lzbench_lzf_decompress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t, size_t, char*)
{
return lzf_decompress(inbuf, insize, outbuf, outsize);
}
bool CompressionSuite::Decompressor::Decompress(char* pOutputBuffer, const int outputBufferSize, CompressionStats* pStats)
{
assert(mHeader.Algorithm != ALG_Invalid);
int uncompressedDataSize = mHeader.UncompressedDataSize;
if (pStats)
{
pStats->CompressedDataSize = mCompressedDataSize;
pStats->UncompressedDataSize = uncompressedDataSize;
}
if (uncompressedDataSize > outputBufferSize)
return false;
double elapsedTime = 0.0;
bool result = false;
if (mHeader.Algorithm == ALG_DOBOZ)
{
doboz::Decompressor decompressor;
Timer timer;
timer.delta();
result = (decompressor.decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize) == doboz::RESULT_OK);
elapsedTime = timer.delta();
}
else if (mHeader.Algorithm == ALG_YAPPY)
{
Timer timer;
timer.delta();
unsigned char* pSrcData = (unsigned char*)mpInputBuffer;
unsigned char* pSrcDataEnd = (unsigned char*)mpInputBuffer + mCompressedDataSize;
unsigned char* pDstData = (unsigned char*)pOutputBuffer;
while (pSrcData < pSrcDataEnd)
{
unsigned short compressedBlockSize = *reinterpret_cast<unsigned short*>(pSrcData);
pSrcData += sizeof(unsigned short);
unsigned char* pSrcBlockEnd = pSrcData + compressedBlockSize;
unsigned char* pDstBlockEnd = Yappy_UnCompress(pSrcData, pSrcBlockEnd, pDstData);
pDstData = pDstBlockEnd;
pSrcData = pSrcBlockEnd;
};
elapsedTime = timer.delta();
result = ((pDstData == ((unsigned char*)pOutputBuffer + uncompressedDataSize)) &&
(pSrcData == pSrcDataEnd));
}
else if (mHeader.Algorithm == ALG_QUICKLZ)
{
qlz_state_decompress qlzState;
memset(&qlzState, 0, sizeof(qlzState));
Timer timer;
timer.delta();
unsigned int outSize = qlz_decompress(mpInputBuffer, pOutputBuffer, &qlzState);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
else if (mHeader.Algorithm == ALG_FASTLZ)
{
Timer timer;
timer.delta();
int outSize = fastlz_decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
else if (mHeader.Algorithm == ALG_LZF)
{
Timer timer;
timer.delta();
int outSize = lzf_decompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer, uncompressedDataSize);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
else if (mHeader.Algorithm == ALG_SNAPPY)
{
Timer timer;
timer.delta();
result = snappy::RawUncompress(mpInputBuffer, mCompressedDataSize, pOutputBuffer);
elapsedTime = timer.delta();
}
else if (mHeader.Algorithm == ALG_LZ4)
{
Timer timer;
timer.delta();
int outSize = LZ4_decode(mpInputBuffer, pOutputBuffer, mCompressedDataSize);
elapsedTime = timer.delta();
result = (outSize == uncompressedDataSize);
}
if (pStats)
pStats->ElapsedTime = elapsedTime;
FreeCompressedData();
return result;
}
