int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,Mode p_mode) {
switch(p_mode) {
case MODE_FASTLZ: {
if (p_src_size<16) {
uint8_t src[16];
zeromem(&src[p_src_size],16-p_src_size);
copymem(src,p_src,p_src_size);
return fastlz_compress(src,16,p_dst);
} else {
return fastlz_compress(p_src,p_src_size,p_dst);
}
} break;
case MODE_DEFLATE: {
z_stream strm;
strm.zalloc = zipio_alloc;
strm.zfree = zipio_free;
strm.opaque = Z_NULL;
int err = deflateInit(&strm,Z_DEFAULT_COMPRESSION);
if (err!=Z_OK)
return -1;
strm.avail_in=p_src_size;
int aout = deflateBound(&strm,p_src_size);;
/*if (aout>p_src_size) {
deflateEnd(&strm);
return -1;
}*/
strm.avail_out=aout;
strm.next_in=(Bytef*)p_src;
strm.next_out=p_dst;
deflate(&strm,Z_FINISH);
aout = aout - strm.avail_out;
deflateEnd(&strm);
return aout;
} break;
}
ERR_FAIL_V(-1);
}
PHP_COUCHBASE_LOCAL
int php_couchbase_compress_fastlz(const smart_str *input,
php_couchbase_comp *output)
{
cbcomp_new(output,
(size_t)((input->len * 1.05) + 1),
input->len);
return (output->compressed_len =
(fastlz_compress(input->c, input->len, output->data)));
}
void Message::compress() {
if (isCompressed())
return;
#ifdef BUILD_WITH_COMPRESSION_MINIZ
mz_ulong compressedSize = mz_compressBound(_size);
int cmp_status;
uint8_t *pCmp;
pCmp = (mz_uint8 *)malloc((size_t)compressedSize);
// last argument is speed size tradeoff: BEST_SPEED [0-9] BEST_COMPRESSION
cmp_status = mz_compress2(pCmp, &compressedSize, (const unsigned char *)_data.get(), _size, 5);
if (cmp_status != Z_OK) {
// error
free(pCmp);
}
_data = SharedPtr<char>((char*)pCmp);
_meta["um.compressed"] = toStr(_size);
_size = compressedSize;
#elif defined(BUILD_WITH_COMPRESSION_FASTLZ)
// The minimum input buffer size is 16.
if (_size < 16)
return;
// The output buffer must be at least 5% larger than the input buffer and can not be smaller than 66 bytes.
int compressedSize = _size + (double)_size * 0.06;
if (compressedSize < 66)
compressedSize = 66;
char* compressedData = (char*)malloc(compressedSize);
compressedSize = fastlz_compress(_data.get(), _size, compressedData);
// If the input is not compressible, the return value might be larger than length
if (compressedSize > _size) {
free(compressedData);
return;
}
// std::cout << _size << " -> " << compressedSize << " = " << ((float)compressedSize / (float)_size) << std::endl;
_data = SharedPtr<char>((char*)compressedData);
_meta["um.compressed"] = toStr(_size);
_size = compressedSize;
#endif
}
int MPK_Compress(byte * data, int len)
{
int elen = len > 16 ? len / 8 : 2;
static_memory buff(M_STATIC_MEMORY_I_CHANEL, len + elen);
int encodes = fastlz_compress(data, len, buff.data());
if (encodes >= 0 && encodes < len)
{
memcpy(data, buff.data(), encodes);
return encodes;
}
return -1;
}
static int lua_f_fastlz_compress ( lua_State *L )
{
if ( !lua_isstring ( L, 1 ) ) {
lua_pushnil ( L );
return 1;
}
size_t vlen = 0;
const char *value = lua_tolstring ( L, 1, &vlen );
if ( vlen < 1 || vlen > 1 * 1024 * 1024 ) { /// Max 1Mb !!!
lua_pushnil ( L );
return 1;
}
char *dst = ( char * ) &temp_buf;
if ( vlen + 20 > 4096 ) {
dst = large_malloc ( vlen + 20 );
}
if ( dst == NULL ) {
lua_pushnil ( L );
lua_pushstring ( L, "not enough memory!" );
return 2;
}
const unsigned int size_header = ( vlen );
// const unsigned int size_header = htonl ( vlen );
memcpy ( dst, &size_header, sizeof ( unsigned int ) );
int dlen = fastlz_compress ( value, vlen, dst + sizeof ( unsigned int ) );
if ( dst ) {
lua_pushlstring ( L, dst, dlen + sizeof ( unsigned int ) );
if ( dst != ( char * ) &temp_buf ) {
free ( dst );
}
return 1;
} else {
lua_pushnil ( L );
return 1;
}
}
static int yac_add_impl(char *prefix, uint prefix_len, char *key, uint len, zval *value, int ttl, int add TSRMLS_DC) /* {{{ */ {
int ret = 0, flag = Z_TYPE_P(value);
char *msg, buf[YAC_STORAGE_MAX_KEY_LEN];
if ((len + prefix_len) > YAC_STORAGE_MAX_KEY_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Key%s can not be longer than %d bytes",
prefix_len? "(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN);
return ret;
}
if (prefix_len) {
len = snprintf(buf, sizeof(buf), "%s%s", prefix, key);
key = (char *)buf;
}
switch (Z_TYPE_P(value)) {
case IS_NULL:
ret = yac_storage_update(key, len, (char *)&flag, sizeof(int), flag, add, ttl);
break;
case IS_BOOL:
case IS_LONG:
ret = yac_storage_update(key, len, (char *)&Z_LVAL_P(value), sizeof(long), flag, add, ttl);
break;
case IS_DOUBLE:
ret = yac_storage_update(key, len, (char *)&Z_DVAL_P(value), sizeof(double), flag, add, ttl);
break;
case IS_STRING:
case IS_CONSTANT:
{
if (Z_STRLEN_P(value) > YAC_G(compress_threshold) || Z_STRLEN_P(value) > YAC_STORAGE_MAX_ENTRY_LEN) {
int compressed_len;
char *compressed;
/* if longer than this, then we can not stored the length in flag */
if (Z_STRLEN_P(value) > YAC_ENTRY_MAX_ORIG_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value));
return ret;
}
compressed = emalloc(Z_STRLEN_P(value) * 1.05);
compressed_len = fastlz_compress(Z_STRVAL_P(value), Z_STRLEN_P(value), compressed);
if (!compressed_len || compressed_len > Z_STRLEN_P(value)) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed");
efree(compressed);
return ret;
}
if (compressed_len > YAC_G(compress_threshold) || compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value));
efree(compressed);
return ret;
}
flag |= YAC_ENTRY_COMPRESSED;
flag |= (Z_STRLEN_P(value) << YAC_ENTRY_ORIG_LEN_SHIT);
ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add);
efree(compressed);
} else {
ret = yac_storage_update(key, len, Z_STRVAL_P(value), Z_STRLEN_P(value), flag, ttl, add);
}
}
break;
case IS_ARRAY:
case IS_CONSTANT_ARRAY:
case IS_OBJECT:
{
smart_str buf = {0};
if (yac_serializer_php_pack(value, &buf, &msg TSRMLS_CC)) {
if (buf.len > YAC_G(compress_threshold) || buf.len > YAC_STORAGE_MAX_ENTRY_LEN) {
int compressed_len;
char *compressed;
if (buf.len > YAC_ENTRY_MAX_ORIG_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored");
return ret;
}
compressed = emalloc(buf.len * 1.05);
compressed_len = fastlz_compress(buf.c, buf.len, compressed);
if (!compressed_len || compressed_len > buf.len) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed");
efree(compressed);
return ret;
}
if (compressed_len > YAC_G(compress_threshold) || compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored");
efree(compressed);
return ret;
}
flag |= YAC_ENTRY_COMPRESSED;
flag |= (buf.len << YAC_ENTRY_ORIG_LEN_SHIT);
ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add);
efree(compressed);
} else {
ret = yac_storage_update(key, len, buf.c, buf.len, flag, ttl, add);
}
smart_str_free(&buf);
} else {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Serialization failed");
smart_str_free(&buf);
}
}
break;
case IS_RESOURCE:
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Type 'IS_RESOURCE' cannot be stored");
break;
default:
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Unsupported valued type to be stored '%d'", flag);
break;
}
return ret;
}
virtual dtStatus compress(const unsigned char* buffer, const int bufferSize,
unsigned char* compressed, const int /*maxCompressedSize*/, int* compressedSize)
{
*compressedSize = fastlz_compress((const void *const)buffer, bufferSize, compressed);
return DT_SUCCESS;
}
static int yac_add_impl(char *prefix, uint prefix_len, char*key, uint len, zval *value, int ttl, int add TSRMLS_DC){
int ret = 0, flag = z_type_p(value);
char *msg, buf[YAC_STORAGE_MAX_KEY_LEN];
if ((len + prefix_len) > YAC_STORAGE_MAX_KEY_LEN){
php_error_docref(NULL TSRMLS_CC, E_WARNING, "key%s can not be longer than %d bytes",
prefix_len?"(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN);
return ret;
}
if (prefix_len) {
len = snprintf(buf, sizeof(buf), "%s%s", prefix, key);
key = (char *)buf;
}
switch(z_type_p(value)) {
case is_null:
ret = yac_storage_update(key, len, (char *)&flag, sizeof(int), flag, add, ttl);
break;
case is_bool:
case is_long:
ret = yac_storage_update(key, len, (char *)&z_lval_p(value), sizeof(long), flag, add, ttl);
break;
case is_double:
ret = yac_storage_update(key, len, (char *)&z_dval_p(value), sizeof(double), flag, add, ttl);
break;
case is_string:
case is_constant:
{
if(z_strlen_p(value) > yac_g(compress_threshold) || z_strlen_p(value) > yac_storage_max_entry_len) {
int compresssed_len;
char *compressed;
/* if longer than this, then we can not stored the length in flag*/
if (z_strlen_p(value) > yac_entry_max_orig_len) {
php_error_docref(null TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", z_strlen_p(value));
return ret;
}
compressed = emalloc(z_strlen_p(value) * 1.05);
compressed_len = fastlz_compress(z_strval_p(value), z_strlen_p(value), compressed);
if (!compressed_len || compressed_len > z_strlen_p(value)) {
php_error_docref(null TSRMLS_CC, E_WARNING, "Compression failed");
efree(compressed);
return ret;
}
if (compressed_len > yac_g(compress_threshold) || compressed_len > yac_storage_max_entry_len) {
php_error_docref(null TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", z_strlen_p(value));
efree(compressed);
return ret;
}
flag |= yac_entry_compressed;
flag |= (z_strlen_p(value) << yac_entry_orgi_len_shit);
ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add);
efree(compressed);
} else {
ret = yac_storage_update(key, len, z_strval_p(value), z_strlen_p(value), flag, ttl, add);
}
}
break;
case is_array:
case is_constant_array:
case is_object:
{
smart_str buf = {0};
if (yac_serializer_php_pack(value, &buf, &msg TSRMLS_CC)) {
if(buf.len > yac_g(compress_threshold) || buf.len > yac_storage_max_entry_len) {
int compressed_len;
char *compressed;
if (buf.len > yac_entry_max_orig_len) {
php_error_docref(null TSRMLS_CC, E_WARNING, "Value is too big to be stored");
return ret;
}
compressed = emalloc(buf.len * 1.05);
compressed_len = fastlz_compress(buf.c, buf.len, compressed);
if (!compressed_len || compressed_len > buf.len) {
php_error_docref(null TSRMLS_CC, E_WARNING, "Compression failed");
efree(compressed);
return re;
}
if (compressed_len > yac_g(compress_threshold) || compressed_len > yac_storage_max_entry_len) {
php_error_docref(null tsrmls_cc, E_WARNING, "Value is too big to be stored");
efree(compressed);
return ret;
}
flag |= yac_entry_compressed;
flag |= (buf.len << yac_entry_orig_len_shit);
ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add);
efree(compressed);
} else {
ret = yac_storage_update(key, len, buf.c, buf.len, flag, ttl, add);
}
smart_str_free(&buf);
} else {
php_error_docref(null tsrmls_cc, E_WARNING, "Serialization failed");
smart_str_free(&buf);
}
}
break;
case is_resource:
php_error_docref(null tsrmls_cc, E_WARNING, "Type 'is_resource' cannot be stored");
break;
default:
php_error_docref(null tsrmls_cc, E_WARNING, "Unsupported valued type to be stored '%d', flag");
break;
}
return ret;
}
bool CompressionSuite::Compress(const char* inputFileName, const Algorithm algorithm, const char* outputFileName, CompressionStats* pStats)
{
if (pStats)
{
pStats->UncompressedDataSize = 0;
pStats->CompressedDataSize = 0;
pStats->TemporaryBufferSize = 0;
pStats->ElapsedTime = 0.0;
}
struct _stat st;
_stat(inputFileName, &st);
int uncompressedDataSize = st.st_size;
if (pStats)
pStats->UncompressedDataSize = uncompressedDataSize;
char* pInputBuffer = new char[uncompressedDataSize];
{
FILE* f = fopen(inputFileName, "rb");
if (!f)
{
delete[] pInputBuffer;
return false;
}
fread(pInputBuffer, uncompressedDataSize, 1, f);
fclose(f);
}
int temporaryBufferSize = 0;
if (algorithm == ALG_DOBOZ)
{
temporaryBufferSize = (int)doboz::Compressor::getMaxCompressedSize(uncompressedDataSize);
}
else if (algorithm == ALG_YAPPY)
{
// A guess... the extra 2b per block is for the 'compressed block length' for decompression.
// The extra 16 is because the sample code allocates 16 more per block, and we get memory
// stomping otherwise. :/
temporaryBufferSize = uncompressedDataSize + (uncompressedDataSize*2/kYappyBlockSize) + (uncompressedDataSize*16/kYappyBlockSize);
// And this is because we still get stomping. :/
temporaryBufferSize *= 2;
}
else if (algorithm == ALG_QUICKLZ)
{
temporaryBufferSize = uncompressedDataSize * 2;
}
else if (algorithm == ALG_FASTLZ)
{
// Must be no smaller than 66b, and at least 5% larger than uncompressed data
temporaryBufferSize = 66 + uncompressedDataSize * 2;
}
else if (algorithm == ALG_LZF)
{
temporaryBufferSize = uncompressedDataSize * 2;
}
else if (algorithm == ALG_SNAPPY)
{
temporaryBufferSize = (int)snappy::MaxCompressedLength(uncompressedDataSize);
}
else if (algorithm == ALG_LZ4)
{
temporaryBufferSize = uncompressedDataSize * 2;
}
if (pStats)
pStats->TemporaryBufferSize = temporaryBufferSize;
char* pOutputBuffer = new char[temporaryBufferSize];
size_t outputSize;
double elapsedTime = 0.0;
bool result = false;
if (algorithm == ALG_DOBOZ)
{
doboz::Compressor compressor;
Timer timer;
timer.delta();
result = (compressor.compress(pInputBuffer, uncompressedDataSize, pOutputBuffer, temporaryBufferSize, outputSize) == doboz::RESULT_OK);
elapsedTime = timer.delta();
}
else if (algorithm == ALG_YAPPY)
{
Yappy_FillTables();
unsigned char* pSrcData = (unsigned char*)pInputBuffer;
unsigned char* pDstData = (unsigned char*)pOutputBuffer;
Timer timer;
timer.delta();
for (int offset=0; offset<uncompressedDataSize; offset+=kYappyBlockSize, pSrcData+=kYappyBlockSize)
{
unsigned int blockSize = ((uncompressedDataSize-offset) >= kYappyBlockSize) ? kYappyBlockSize : uncompressedDataSize-offset;
unsigned short* pCompressedBlockSize = (unsigned short*)pDstData;
pDstData += sizeof(unsigned short);
unsigned char* pBlockEnd = Yappy_Compress(pSrcData, pDstData, blockSize);
unsigned int compressedBlockSize = pBlockEnd - pDstData;
*pCompressedBlockSize = static_cast<unsigned short>(compressedBlockSize);
pDstData = pBlockEnd;
}
elapsedTime = timer.delta();
outputSize = pDstData - (unsigned char*)pOutputBuffer;
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_QUICKLZ)
{
qlz_state_compress qlzState;
memset(&qlzState, 0, sizeof(qlzState));
Timer timer;
timer.delta();
outputSize = qlz_compress(pInputBuffer, pOutputBuffer, uncompressedDataSize, &qlzState);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_FASTLZ)
{
Timer timer;
timer.delta();
outputSize = fastlz_compress(pInputBuffer, uncompressedDataSize, pOutputBuffer);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_LZF)
{
Timer timer;
timer.delta();
outputSize = lzf_compress(pInputBuffer, uncompressedDataSize, pOutputBuffer, temporaryBufferSize);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_SNAPPY)
{
Timer timer;
timer.delta();
snappy::RawCompress(pInputBuffer, uncompressedDataSize, pOutputBuffer, &outputSize);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
else if (algorithm == ALG_LZ4)
{
Timer timer;
timer.delta();
outputSize = LZ4_compress(pInputBuffer, pOutputBuffer, uncompressedDataSize);
elapsedTime = timer.delta();
result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0);
}
if (pStats)
pStats->ElapsedTime = elapsedTime;
if (!result)
{
delete[] pOutputBuffer;
delete[] pInputBuffer;
return false;
}
int compressedDataSize = (int)outputSize;
if (pStats)
pStats->CompressedDataSize = compressedDataSize;
FILE* of = fopen(outputFileName, "wb");
if (!of)
{
delete[] pOutputBuffer;
delete[] pInputBuffer;
return false;
}
FileHeader header;
header.Algorithm = algorithm;
header.UncompressedDataSize = uncompressedDataSize;
fwrite(&header, sizeof(header), 1, of);
fwrite(pOutputBuffer, compressedDataSize, 1, of);
fclose(of);
delete[] pOutputBuffer;
delete[] pInputBuffer;
return true;
}
static int yac_add_impl(zend_string *prefix, zend_string *key, zval *value, int ttl, int add) /* {{{ */ {
int ret = 0, flag = Z_TYPE_P(value);
char *msg;
time_t tv;
zend_string *prefix_key;
if ((ZSTR_LEN(key) + prefix->len) > YAC_STORAGE_MAX_KEY_LEN) {
php_error_docref(NULL, E_WARNING, "Key%s can not be longer than %d bytes",
prefix->len? "(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN);
return ret;
}
if (prefix->len) {
prefix_key = strpprintf(YAC_STORAGE_MAX_KEY_LEN, "%s%s", ZSTR_VAL(prefix), ZSTR_VAL(key));
key = prefix_key;
}
tv = time(NULL);
switch (Z_TYPE_P(value)) {
case IS_NULL:
case IS_TRUE:
case IS_FALSE:
ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), (char *)&flag, sizeof(int), flag, ttl, add, tv);
break;
case IS_LONG:
ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), (char *)&Z_LVAL_P(value), sizeof(long), flag, ttl, add, tv);
break;
case IS_DOUBLE:
ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), (char *)&Z_DVAL_P(value), sizeof(double), flag, ttl, add, tv);
break;
case IS_STRING:
case IS_CONSTANT:
{
if (Z_STRLEN_P(value) > YAC_G(compress_threshold) || Z_STRLEN_P(value) > YAC_STORAGE_MAX_ENTRY_LEN) {
int compressed_len;
char *compressed;
/* if longer than this, then we can not stored the length in flag */
if (Z_STRLEN_P(value) > YAC_ENTRY_MAX_ORIG_LEN) {
php_error_docref(NULL, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value));
if (prefix->len) {
zend_string_release(prefix_key);
}
return ret;
}
compressed = emalloc(Z_STRLEN_P(value) * 1.05);
compressed_len = fastlz_compress(Z_STRVAL_P(value), Z_STRLEN_P(value), compressed);
if (!compressed_len || compressed_len > Z_STRLEN_P(value)) {
php_error_docref(NULL, E_WARNING, "Compression failed");
efree(compressed);
if (prefix->len) {
zend_string_release(prefix_key);
}
return ret;
}
if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) {
php_error_docref(NULL, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value));
efree(compressed);
if (prefix->len) {
zend_string_release(prefix_key);
}
return ret;
}
flag |= YAC_ENTRY_COMPRESSED;
flag |= (Z_STRLEN_P(value) << YAC_ENTRY_ORIG_LEN_SHIT);
ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), compressed, compressed_len, flag, ttl, add, tv);
efree(compressed);
} else {
ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), Z_STRVAL_P(value), Z_STRLEN_P(value), flag, ttl, add, tv);
}
}
break;
case IS_ARRAY:
#ifdef IS_CONSTANT_ARRAY
case IS_CONSTANT_ARRAY:
#endif
case IS_OBJECT:
{
smart_str buf = {0};
#if ENABLE_MSGPACK
if (yac_serializer_msgpack_pack(value, &buf, &msg))
#else
if (yac_serializer_php_pack(value, &buf, &msg))
#endif
{
if (buf.s->len > YAC_G(compress_threshold) || buf.s->len > YAC_STORAGE_MAX_ENTRY_LEN) {
int compressed_len;
char *compressed;
if (buf.s->len > YAC_ENTRY_MAX_ORIG_LEN) {
php_error_docref(NULL, E_WARNING, "Value is too big to be stored");
if (prefix->len) {
zend_string_release(prefix_key);
}
return ret;
}
compressed = emalloc(buf.s->len * 1.05);
compressed_len = fastlz_compress(ZSTR_VAL(buf.s), ZSTR_LEN(buf.s), compressed);
if (!compressed_len || compressed_len > buf.s->len) {
php_error_docref(NULL, E_WARNING, "Compression failed");
efree(compressed);
if (prefix->len) {
zend_string_release(prefix_key);
}
return ret;
}
if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) {
php_error_docref(NULL, E_WARNING, "Value is too big to be stored");
efree(compressed);
if (prefix->len) {
zend_string_release(prefix_key);
}
return ret;
}
flag |= YAC_ENTRY_COMPRESSED;
flag |= (buf.s->len << YAC_ENTRY_ORIG_LEN_SHIT);
ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), compressed, compressed_len, flag, ttl, add, tv);
efree(compressed);
} else {
ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), ZSTR_VAL(buf.s), ZSTR_LEN(buf.s), flag, ttl, add, tv);
}
smart_str_free(&buf);
} else {
php_error_docref(NULL, E_WARNING, "Serialization failed");
smart_str_free(&buf);
}
}
break;
case IS_RESOURCE:
php_error_docref(NULL, E_WARNING, "Type 'IS_RESOURCE' cannot be stored");
break;
default:
php_error_docref(NULL, E_WARNING, "Unsupported valued type to be stored '%d'", flag);
break;
}
if (prefix->len) {
zend_string_release(prefix_key);
}
return ret;
}
static int yac_add_impl(char *prefix, uint prefix_len, char *key, uint len, zval *value, int ttl, int add TSRMLS_DC) /* {{{ */ {
int ret = 0, flag = Z_TYPE_P(value);
char *msg, buf[YAC_STORAGE_MAX_KEY_LEN];
time_t tv;
if ((len + prefix_len) > YAC_STORAGE_MAX_KEY_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Key%s can not be longer than %d bytes",
prefix_len? "(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN);
return ret;
}
if (prefix_len) {
len = snprintf(buf, sizeof(buf), "%s%s", prefix, key);
key = (char *)buf;
}
tv = time(NULL);
switch (Z_TYPE_P(value)) {
case IS_NULL:
ret = yac_storage_update(key, len, (char *)&flag, sizeof(int), flag, ttl, add, tv);
break;
case IS_BOOL:
case IS_LONG:
ret = yac_storage_update(key, len, (char *)&Z_LVAL_P(value), sizeof(long), flag, ttl, add, tv);
break;
case IS_DOUBLE:
ret = yac_storage_update(key, len, (char *)&Z_DVAL_P(value), sizeof(double), flag, ttl, add, tv);
break;
case IS_STRING:
case IS_CONSTANT:
{
if (Z_STRLEN_P(value) > YAC_G(compress_threshold) || Z_STRLEN_P(value) > YAC_STORAGE_MAX_ENTRY_LEN) {
int compressed_len;
char *compressed;
/* if longer than this, then we can not stored the length in flag */
if (Z_STRLEN_P(value) > YAC_ENTRY_MAX_ORIG_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value));
return ret;
}
compressed = emalloc(Z_STRLEN_P(value) * 1.05);
compressed_len = fastlz_compress(Z_STRVAL_P(value), Z_STRLEN_P(value), compressed);
if (!compressed_len || compressed_len > Z_STRLEN_P(value)) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed");
efree(compressed);
return ret;
}
if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value));
efree(compressed);
return ret;
}
flag |= YAC_ENTRY_COMPRESSED;
flag |= (Z_STRLEN_P(value) << YAC_ENTRY_ORIG_LEN_SHIT);
ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add, tv);
efree(compressed);
} else {
ret = yac_storage_update(key, len, Z_STRVAL_P(value), Z_STRLEN_P(value), flag, ttl, add, tv);
}
}
break;
case IS_ARRAY:
case IS_CONSTANT_ARRAY:
case IS_OBJECT:
{
smart_str buf = {0};
#if ENABLE_MSGPACK
if (yac_serializer_msgpack_pack(value, &buf, &msg TSRMLS_CC)) {
#else
if (yac_serializer_php_pack(value, &buf, &msg TSRMLS_CC)) {
#endif
if (buf.len > YAC_G(compress_threshold) || buf.len > YAC_STORAGE_MAX_ENTRY_LEN) {
int compressed_len;
char *compressed;
if (buf.len > YAC_ENTRY_MAX_ORIG_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored");
return ret;
}
compressed = emalloc(buf.len * 1.05);
compressed_len = fastlz_compress(buf.c, buf.len, compressed);
if (!compressed_len || compressed_len > buf.len) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed");
efree(compressed);
return ret;
}
if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored");
efree(compressed);
return ret;
}
flag |= YAC_ENTRY_COMPRESSED;
flag |= (buf.len << YAC_ENTRY_ORIG_LEN_SHIT);
ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add, tv);
efree(compressed);
} else {
ret = yac_storage_update(key, len, buf.c, buf.len, flag, ttl, add, tv);
}
smart_str_free(&buf);
} else {
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Serialization failed");
smart_str_free(&buf);
}
}
break;
case IS_RESOURCE:
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Type 'IS_RESOURCE' cannot be stored");
break;
default:
php_error_docref(NULL TSRMLS_CC, E_WARNING, "Unsupported valued type to be stored '%d'", flag);
break;
}
return ret;
}
/* }}} */
static int yac_add_multi_impl(char *prefix, uint prefix_len, zval *kvs, int ttl, int add TSRMLS_DC) /* {{{ */ {
HashTable *ht = Z_ARRVAL_P(kvs);
for (zend_hash_internal_pointer_reset(ht);
zend_hash_has_more_elements(ht) == SUCCESS;
zend_hash_move_forward(ht)) {
char *key;
ulong idx;
zval **value;
uint len, should_free = 0;
if (zend_hash_get_current_data(ht, (void **)&value) == FAILURE) {
continue;
}
switch (zend_hash_get_current_key_ex(ht, &key, &len, &idx, 0, NULL)) {
case HASH_KEY_IS_LONG:
len = spprintf(&key, 0, "%lu", idx) + 1;
should_free = 1;
case HASH_KEY_IS_STRING:
if (yac_add_impl(prefix, prefix_len, key, len - 1, *value, ttl, add TSRMLS_CC)) {
if (should_free) {
efree(key);
}
continue;
} else {
if (should_free) {
efree(key);
}
return 0;
}
default:
continue;
}
}
return 1;
}
void Message::compress(Message::Compression type, int level) {
if (isCompressed())
return;
switch (type) {
#ifdef BUILD_WITH_COMPRESSION_MINIZ
case COMPRESS_MINIZ: {
mz_ulong compressedSize = mz_compressBound(_size);
int cmp_status;
uint8_t *pCmp;
pCmp = (mz_uint8 *)malloc((size_t)compressedSize);
level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_MINIZ);
// last argument is speed size tradeoff: BEST_SPEED [0-9] BEST_COMPRESSION
cmp_status = mz_compress2(pCmp, &compressedSize, (const unsigned char *)_data.get(), _size, level);
if (cmp_status != Z_OK) {
// error
free(pCmp);
}
_data = SharedPtr<char>((char*)pCmp);
_meta["um.compressed"] = toStr(_size) + ":miniz";
_size = compressedSize;
}
return;
#endif
#ifdef BUILD_WITH_COMPRESSION_FASTLZ
case COMPRESS_FASTLZ: {
// The minimum input buffer size is 16.
if (_size < 16)
return;
// The output buffer must be at least 5% larger than the input buffer and can not be smaller than 66 bytes.
int compressedSize = _size + (double)_size * 0.06;
if (compressedSize < 66)
compressedSize = 66;
char* compressedData = (char*)malloc(compressedSize);
compressedSize = fastlz_compress(_data.get(), _size, compressedData);
// If the input is not compressible, the return value might be larger than length
if (compressedSize > _size) {
free(compressedData);
return;
}
// std::cout << _size << " -> " << compressedSize << " = " << ((float)compressedSize / (float)_size) << std::endl;
_data = SharedPtr<char>((char*)compressedData);
_meta["um.compressed"] = toStr(_size) + ":fastlz";
_size = compressedSize;
}
return;
#endif
#ifdef BUILD_WITH_COMPRESSION_LZ4
case COMPRESS_LZ4: {
level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_LZ4);
#ifdef LZ4_FRAME
LZ4F_preferences_t lz4_preferences = {
{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } },
level, /* compression level */
0, /* autoflush */
{ 0, 0, 0, 0 }, /* reserved, must be set to 0 */
};
LZ4F_errorCode_t r;
LZ4F_compressionContext_t ctx;
r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(r)) {
// printf("Failed to create context: error %zu", r);
return;
}
#define LZ4_HEADER_SIZE 19
#define LZ4_FOOTER_SIZE 4
size_t n, offset = 0;
size_t frameSize = LZ4F_compressBound(_size, &lz4_preferences);
size_t compressedSize = frameSize + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
char* compressedData = (char*)malloc(compressedSize);
n = LZ4F_compressBegin(ctx, compressedData, compressedSize, &lz4_preferences);
if (LZ4F_isError(n)) {
// printf("Failed to start compression: error %zu", n);
LZ4F_freeCompressionContext(ctx);
free(compressedData);
return;
}
offset += n;
n = LZ4F_compressUpdate(ctx, compressedData + offset, compressedSize - offset, _data.get(), _size, NULL);
if (LZ4F_isError(n)) {
// printf("Compression failed: error %zu", n);
LZ4F_freeCompressionContext(ctx);
free(compressedData);
return;
}
offset += n;
n = LZ4F_compressEnd(ctx, compressedData + offset, compressedSize - offset, NULL);
if (LZ4F_isError(n)) {
// printf("Failed to end compression: error %zu", n);
LZ4F_freeCompressionContext(ctx);
free(compressedData);
return;
}
offset += n;
_data = SharedPtr<char>((char*)compressedData);
_meta["um.compressed"] = toStr(_size) + ":lz4";
_size = offset;
LZ4F_freeCompressionContext(ctx);
#else
size_t compressedSize = LZ4_compressBound(_size);
char* compressedData = (char*)malloc(compressedSize);
int actualSize = 0;
actualSize = LZ4_compress_fast(_data.get(), compressedData, _size, compressedSize, level);
if (actualSize == 0) {
free(compressedData);
return;
}
_data = SharedPtr<char>((char*)compressedData);
_meta["um.compressed"] = toStr(_size) + ":lz4";
_size = actualSize;
#endif
}
return;
#endif
default:
break;
}
}