int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,Mode p_mode) { switch(p_mode) { case MODE_FASTLZ: { if (p_src_size<16) { uint8_t src[16]; zeromem(&src[p_src_size],16-p_src_size); copymem(src,p_src,p_src_size); return fastlz_compress(src,16,p_dst); } else { return fastlz_compress(p_src,p_src_size,p_dst); } } break; case MODE_DEFLATE: { z_stream strm; strm.zalloc = zipio_alloc; strm.zfree = zipio_free; strm.opaque = Z_NULL; int err = deflateInit(&strm,Z_DEFAULT_COMPRESSION); if (err!=Z_OK) return -1; strm.avail_in=p_src_size; int aout = deflateBound(&strm,p_src_size);; /*if (aout>p_src_size) { deflateEnd(&strm); return -1; }*/ strm.avail_out=aout; strm.next_in=(Bytef*)p_src; strm.next_out=p_dst; deflate(&strm,Z_FINISH); aout = aout - strm.avail_out; deflateEnd(&strm); return aout; } break; } ERR_FAIL_V(-1); }
PHP_COUCHBASE_LOCAL int php_couchbase_compress_fastlz(const smart_str *input, php_couchbase_comp *output) { cbcomp_new(output, (size_t)((input->len * 1.05) + 1), input->len); return (output->compressed_len = (fastlz_compress(input->c, input->len, output->data))); }
void Message::compress() { if (isCompressed()) return; #ifdef BUILD_WITH_COMPRESSION_MINIZ mz_ulong compressedSize = mz_compressBound(_size); int cmp_status; uint8_t *pCmp; pCmp = (mz_uint8 *)malloc((size_t)compressedSize); // last argument is speed size tradeoff: BEST_SPEED [0-9] BEST_COMPRESSION cmp_status = mz_compress2(pCmp, &compressedSize, (const unsigned char *)_data.get(), _size, 5); if (cmp_status != Z_OK) { // error free(pCmp); } _data = SharedPtr<char>((char*)pCmp); _meta["um.compressed"] = toStr(_size); _size = compressedSize; #elif defined(BUILD_WITH_COMPRESSION_FASTLZ) // The minimum input buffer size is 16. if (_size < 16) return; // The output buffer must be at least 5% larger than the input buffer and can not be smaller than 66 bytes. int compressedSize = _size + (double)_size * 0.06; if (compressedSize < 66) compressedSize = 66; char* compressedData = (char*)malloc(compressedSize); compressedSize = fastlz_compress(_data.get(), _size, compressedData); // If the input is not compressible, the return value might be larger than length if (compressedSize > _size) { free(compressedData); return; } // std::cout << _size << " -> " << compressedSize << " = " << ((float)compressedSize / (float)_size) << std::endl; _data = SharedPtr<char>((char*)compressedData); _meta["um.compressed"] = toStr(_size); _size = compressedSize; #endif }
int MPK_Compress(byte * data, int len) { int elen = len > 16 ? len / 8 : 2; static_memory buff(M_STATIC_MEMORY_I_CHANEL, len + elen); int encodes = fastlz_compress(data, len, buff.data()); if (encodes >= 0 && encodes < len) { memcpy(data, buff.data(), encodes); return encodes; } return -1; }
static int lua_f_fastlz_compress ( lua_State *L ) { if ( !lua_isstring ( L, 1 ) ) { lua_pushnil ( L ); return 1; } size_t vlen = 0; const char *value = lua_tolstring ( L, 1, &vlen ); if ( vlen < 1 || vlen > 1 * 1024 * 1024 ) { /// Max 1Mb !!! lua_pushnil ( L ); return 1; } char *dst = ( char * ) &temp_buf; if ( vlen + 20 > 4096 ) { dst = large_malloc ( vlen + 20 ); } if ( dst == NULL ) { lua_pushnil ( L ); lua_pushstring ( L, "not enough memory!" ); return 2; } const unsigned int size_header = ( vlen ); // const unsigned int size_header = htonl ( vlen ); memcpy ( dst, &size_header, sizeof ( unsigned int ) ); int dlen = fastlz_compress ( value, vlen, dst + sizeof ( unsigned int ) ); if ( dst ) { lua_pushlstring ( L, dst, dlen + sizeof ( unsigned int ) ); if ( dst != ( char * ) &temp_buf ) { free ( dst ); } return 1; } else { lua_pushnil ( L ); return 1; } }
static int yac_add_impl(char *prefix, uint prefix_len, char *key, uint len, zval *value, int ttl, int add TSRMLS_DC) /* {{{ */ { int ret = 0, flag = Z_TYPE_P(value); char *msg, buf[YAC_STORAGE_MAX_KEY_LEN]; if ((len + prefix_len) > YAC_STORAGE_MAX_KEY_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Key%s can not be longer than %d bytes", prefix_len? "(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN); return ret; } if (prefix_len) { len = snprintf(buf, sizeof(buf), "%s%s", prefix, key); key = (char *)buf; } switch (Z_TYPE_P(value)) { case IS_NULL: ret = yac_storage_update(key, len, (char *)&flag, sizeof(int), flag, add, ttl); break; case IS_BOOL: case IS_LONG: ret = yac_storage_update(key, len, (char *)&Z_LVAL_P(value), sizeof(long), flag, add, ttl); break; case IS_DOUBLE: ret = yac_storage_update(key, len, (char *)&Z_DVAL_P(value), sizeof(double), flag, add, ttl); break; case IS_STRING: case IS_CONSTANT: { if (Z_STRLEN_P(value) > YAC_G(compress_threshold) || Z_STRLEN_P(value) > YAC_STORAGE_MAX_ENTRY_LEN) { int compressed_len; char *compressed; /* if longer than this, then we can not stored the length in flag */ if (Z_STRLEN_P(value) > YAC_ENTRY_MAX_ORIG_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value)); return ret; } compressed = emalloc(Z_STRLEN_P(value) * 1.05); compressed_len = fastlz_compress(Z_STRVAL_P(value), Z_STRLEN_P(value), compressed); if (!compressed_len || compressed_len > Z_STRLEN_P(value)) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed"); efree(compressed); return ret; } if (compressed_len > YAC_G(compress_threshold) || compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value)); efree(compressed); return ret; } flag |= YAC_ENTRY_COMPRESSED; flag |= (Z_STRLEN_P(value) << YAC_ENTRY_ORIG_LEN_SHIT); ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add); efree(compressed); } else { ret = yac_storage_update(key, len, Z_STRVAL_P(value), Z_STRLEN_P(value), flag, ttl, add); } } break; case IS_ARRAY: case IS_CONSTANT_ARRAY: case IS_OBJECT: { smart_str buf = {0}; if (yac_serializer_php_pack(value, &buf, &msg TSRMLS_CC)) { if (buf.len > YAC_G(compress_threshold) || buf.len > YAC_STORAGE_MAX_ENTRY_LEN) { int compressed_len; char *compressed; if (buf.len > YAC_ENTRY_MAX_ORIG_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored"); return ret; } compressed = emalloc(buf.len * 1.05); compressed_len = fastlz_compress(buf.c, buf.len, compressed); if (!compressed_len || compressed_len > buf.len) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed"); efree(compressed); return ret; } if (compressed_len > YAC_G(compress_threshold) || compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored"); efree(compressed); return ret; } flag |= YAC_ENTRY_COMPRESSED; flag |= (buf.len << YAC_ENTRY_ORIG_LEN_SHIT); ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add); efree(compressed); } else { ret = yac_storage_update(key, len, buf.c, buf.len, flag, ttl, add); } smart_str_free(&buf); } else { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Serialization failed"); smart_str_free(&buf); } } break; case IS_RESOURCE: php_error_docref(NULL TSRMLS_CC, E_WARNING, "Type 'IS_RESOURCE' cannot be stored"); break; default: php_error_docref(NULL TSRMLS_CC, E_WARNING, "Unsupported valued type to be stored '%d'", flag); break; } return ret; }
virtual dtStatus compress(const unsigned char* buffer, const int bufferSize, unsigned char* compressed, const int /*maxCompressedSize*/, int* compressedSize) { *compressedSize = fastlz_compress((const void *const)buffer, bufferSize, compressed); return DT_SUCCESS; }
static int yac_add_impl(char *prefix, uint prefix_len, char*key, uint len, zval *value, int ttl, int add TSRMLS_DC){ int ret = 0, flag = z_type_p(value); char *msg, buf[YAC_STORAGE_MAX_KEY_LEN]; if ((len + prefix_len) > YAC_STORAGE_MAX_KEY_LEN){ php_error_docref(NULL TSRMLS_CC, E_WARNING, "key%s can not be longer than %d bytes", prefix_len?"(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN); return ret; } if (prefix_len) { len = snprintf(buf, sizeof(buf), "%s%s", prefix, key); key = (char *)buf; } switch(z_type_p(value)) { case is_null: ret = yac_storage_update(key, len, (char *)&flag, sizeof(int), flag, add, ttl); break; case is_bool: case is_long: ret = yac_storage_update(key, len, (char *)&z_lval_p(value), sizeof(long), flag, add, ttl); break; case is_double: ret = yac_storage_update(key, len, (char *)&z_dval_p(value), sizeof(double), flag, add, ttl); break; case is_string: case is_constant: { if(z_strlen_p(value) > yac_g(compress_threshold) || z_strlen_p(value) > yac_storage_max_entry_len) { int compresssed_len; char *compressed; /* if longer than this, then we can not stored the length in flag*/ if (z_strlen_p(value) > yac_entry_max_orig_len) { php_error_docref(null TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", z_strlen_p(value)); return ret; } compressed = emalloc(z_strlen_p(value) * 1.05); compressed_len = fastlz_compress(z_strval_p(value), z_strlen_p(value), compressed); if (!compressed_len || compressed_len > z_strlen_p(value)) { php_error_docref(null TSRMLS_CC, E_WARNING, "Compression failed"); efree(compressed); return ret; } if (compressed_len > yac_g(compress_threshold) || compressed_len > yac_storage_max_entry_len) { php_error_docref(null TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", z_strlen_p(value)); efree(compressed); return ret; } flag |= yac_entry_compressed; flag |= (z_strlen_p(value) << yac_entry_orgi_len_shit); ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add); efree(compressed); } else { ret = yac_storage_update(key, len, z_strval_p(value), z_strlen_p(value), flag, ttl, add); } } break; case is_array: case is_constant_array: case is_object: { smart_str buf = {0}; if (yac_serializer_php_pack(value, &buf, &msg TSRMLS_CC)) { if(buf.len > yac_g(compress_threshold) || buf.len > yac_storage_max_entry_len) { int compressed_len; char *compressed; if (buf.len > yac_entry_max_orig_len) { php_error_docref(null TSRMLS_CC, E_WARNING, "Value is too big to be stored"); return ret; } compressed = emalloc(buf.len * 1.05); compressed_len = fastlz_compress(buf.c, buf.len, compressed); if (!compressed_len || compressed_len > buf.len) { php_error_docref(null TSRMLS_CC, E_WARNING, "Compression failed"); efree(compressed); return re; } if (compressed_len > yac_g(compress_threshold) || compressed_len > yac_storage_max_entry_len) { php_error_docref(null tsrmls_cc, E_WARNING, "Value is too big to be stored"); efree(compressed); return ret; } flag |= yac_entry_compressed; flag |= (buf.len << yac_entry_orig_len_shit); ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add); efree(compressed); } else { ret = yac_storage_update(key, len, buf.c, buf.len, flag, ttl, add); } smart_str_free(&buf); } else { php_error_docref(null tsrmls_cc, E_WARNING, "Serialization failed"); smart_str_free(&buf); } } break; case is_resource: php_error_docref(null tsrmls_cc, E_WARNING, "Type 'is_resource' cannot be stored"); break; default: php_error_docref(null tsrmls_cc, E_WARNING, "Unsupported valued type to be stored '%d', flag"); break; } return ret; }
bool CompressionSuite::Compress(const char* inputFileName, const Algorithm algorithm, const char* outputFileName, CompressionStats* pStats) { if (pStats) { pStats->UncompressedDataSize = 0; pStats->CompressedDataSize = 0; pStats->TemporaryBufferSize = 0; pStats->ElapsedTime = 0.0; } struct _stat st; _stat(inputFileName, &st); int uncompressedDataSize = st.st_size; if (pStats) pStats->UncompressedDataSize = uncompressedDataSize; char* pInputBuffer = new char[uncompressedDataSize]; { FILE* f = fopen(inputFileName, "rb"); if (!f) { delete[] pInputBuffer; return false; } fread(pInputBuffer, uncompressedDataSize, 1, f); fclose(f); } int temporaryBufferSize = 0; if (algorithm == ALG_DOBOZ) { temporaryBufferSize = (int)doboz::Compressor::getMaxCompressedSize(uncompressedDataSize); } else if (algorithm == ALG_YAPPY) { // A guess... the extra 2b per block is for the 'compressed block length' for decompression. // The extra 16 is because the sample code allocates 16 more per block, and we get memory // stomping otherwise. :/ temporaryBufferSize = uncompressedDataSize + (uncompressedDataSize*2/kYappyBlockSize) + (uncompressedDataSize*16/kYappyBlockSize); // And this is because we still get stomping. :/ temporaryBufferSize *= 2; } else if (algorithm == ALG_QUICKLZ) { temporaryBufferSize = uncompressedDataSize * 2; } else if (algorithm == ALG_FASTLZ) { // Must be no smaller than 66b, and at least 5% larger than uncompressed data temporaryBufferSize = 66 + uncompressedDataSize * 2; } else if (algorithm == ALG_LZF) { temporaryBufferSize = uncompressedDataSize * 2; } else if (algorithm == ALG_SNAPPY) { temporaryBufferSize = (int)snappy::MaxCompressedLength(uncompressedDataSize); } else if (algorithm == ALG_LZ4) { temporaryBufferSize = uncompressedDataSize * 2; } if (pStats) pStats->TemporaryBufferSize = temporaryBufferSize; char* pOutputBuffer = new char[temporaryBufferSize]; size_t outputSize; double elapsedTime = 0.0; bool result = false; if (algorithm == ALG_DOBOZ) { doboz::Compressor compressor; Timer timer; timer.delta(); result = (compressor.compress(pInputBuffer, uncompressedDataSize, pOutputBuffer, temporaryBufferSize, outputSize) == doboz::RESULT_OK); elapsedTime = timer.delta(); } else if (algorithm == ALG_YAPPY) { Yappy_FillTables(); unsigned char* pSrcData = (unsigned char*)pInputBuffer; unsigned char* pDstData = (unsigned char*)pOutputBuffer; Timer timer; timer.delta(); for (int offset=0; offset<uncompressedDataSize; offset+=kYappyBlockSize, pSrcData+=kYappyBlockSize) { unsigned int blockSize = ((uncompressedDataSize-offset) >= kYappyBlockSize) ? kYappyBlockSize : uncompressedDataSize-offset; unsigned short* pCompressedBlockSize = (unsigned short*)pDstData; pDstData += sizeof(unsigned short); unsigned char* pBlockEnd = Yappy_Compress(pSrcData, pDstData, blockSize); unsigned int compressedBlockSize = pBlockEnd - pDstData; *pCompressedBlockSize = static_cast<unsigned short>(compressedBlockSize); pDstData = pBlockEnd; } elapsedTime = timer.delta(); outputSize = pDstData - (unsigned char*)pOutputBuffer; result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0); } else if (algorithm == ALG_QUICKLZ) { qlz_state_compress qlzState; memset(&qlzState, 0, sizeof(qlzState)); Timer timer; timer.delta(); outputSize = qlz_compress(pInputBuffer, pOutputBuffer, uncompressedDataSize, &qlzState); elapsedTime = timer.delta(); result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0); } else if (algorithm == ALG_FASTLZ) { Timer timer; timer.delta(); outputSize = fastlz_compress(pInputBuffer, uncompressedDataSize, pOutputBuffer); elapsedTime = timer.delta(); result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0); } else if (algorithm == ALG_LZF) { Timer timer; timer.delta(); outputSize = lzf_compress(pInputBuffer, uncompressedDataSize, pOutputBuffer, temporaryBufferSize); elapsedTime = timer.delta(); result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0); } else if (algorithm == ALG_SNAPPY) { Timer timer; timer.delta(); snappy::RawCompress(pInputBuffer, uncompressedDataSize, pOutputBuffer, &outputSize); elapsedTime = timer.delta(); result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0); } else if (algorithm == ALG_LZ4) { Timer timer; timer.delta(); outputSize = LZ4_compress(pInputBuffer, pOutputBuffer, uncompressedDataSize); elapsedTime = timer.delta(); result = (outputSize <= (unsigned int)temporaryBufferSize) && (outputSize != 0); } if (pStats) pStats->ElapsedTime = elapsedTime; if (!result) { delete[] pOutputBuffer; delete[] pInputBuffer; return false; } int compressedDataSize = (int)outputSize; if (pStats) pStats->CompressedDataSize = compressedDataSize; FILE* of = fopen(outputFileName, "wb"); if (!of) { delete[] pOutputBuffer; delete[] pInputBuffer; return false; } FileHeader header; header.Algorithm = algorithm; header.UncompressedDataSize = uncompressedDataSize; fwrite(&header, sizeof(header), 1, of); fwrite(pOutputBuffer, compressedDataSize, 1, of); fclose(of); delete[] pOutputBuffer; delete[] pInputBuffer; return true; }
static int yac_add_impl(zend_string *prefix, zend_string *key, zval *value, int ttl, int add) /* {{{ */ { int ret = 0, flag = Z_TYPE_P(value); char *msg; time_t tv; zend_string *prefix_key; if ((ZSTR_LEN(key) + prefix->len) > YAC_STORAGE_MAX_KEY_LEN) { php_error_docref(NULL, E_WARNING, "Key%s can not be longer than %d bytes", prefix->len? "(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN); return ret; } if (prefix->len) { prefix_key = strpprintf(YAC_STORAGE_MAX_KEY_LEN, "%s%s", ZSTR_VAL(prefix), ZSTR_VAL(key)); key = prefix_key; } tv = time(NULL); switch (Z_TYPE_P(value)) { case IS_NULL: case IS_TRUE: case IS_FALSE: ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), (char *)&flag, sizeof(int), flag, ttl, add, tv); break; case IS_LONG: ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), (char *)&Z_LVAL_P(value), sizeof(long), flag, ttl, add, tv); break; case IS_DOUBLE: ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), (char *)&Z_DVAL_P(value), sizeof(double), flag, ttl, add, tv); break; case IS_STRING: case IS_CONSTANT: { if (Z_STRLEN_P(value) > YAC_G(compress_threshold) || Z_STRLEN_P(value) > YAC_STORAGE_MAX_ENTRY_LEN) { int compressed_len; char *compressed; /* if longer than this, then we can not stored the length in flag */ if (Z_STRLEN_P(value) > YAC_ENTRY_MAX_ORIG_LEN) { php_error_docref(NULL, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value)); if (prefix->len) { zend_string_release(prefix_key); } return ret; } compressed = emalloc(Z_STRLEN_P(value) * 1.05); compressed_len = fastlz_compress(Z_STRVAL_P(value), Z_STRLEN_P(value), compressed); if (!compressed_len || compressed_len > Z_STRLEN_P(value)) { php_error_docref(NULL, E_WARNING, "Compression failed"); efree(compressed); if (prefix->len) { zend_string_release(prefix_key); } return ret; } if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) { php_error_docref(NULL, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value)); efree(compressed); if (prefix->len) { zend_string_release(prefix_key); } return ret; } flag |= YAC_ENTRY_COMPRESSED; flag |= (Z_STRLEN_P(value) << YAC_ENTRY_ORIG_LEN_SHIT); ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), compressed, compressed_len, flag, ttl, add, tv); efree(compressed); } else { ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), Z_STRVAL_P(value), Z_STRLEN_P(value), flag, ttl, add, tv); } } break; case IS_ARRAY: #ifdef IS_CONSTANT_ARRAY case IS_CONSTANT_ARRAY: #endif case IS_OBJECT: { smart_str buf = {0}; #if ENABLE_MSGPACK if (yac_serializer_msgpack_pack(value, &buf, &msg)) #else if (yac_serializer_php_pack(value, &buf, &msg)) #endif { if (buf.s->len > YAC_G(compress_threshold) || buf.s->len > YAC_STORAGE_MAX_ENTRY_LEN) { int compressed_len; char *compressed; if (buf.s->len > YAC_ENTRY_MAX_ORIG_LEN) { php_error_docref(NULL, E_WARNING, "Value is too big to be stored"); if (prefix->len) { zend_string_release(prefix_key); } return ret; } compressed = emalloc(buf.s->len * 1.05); compressed_len = fastlz_compress(ZSTR_VAL(buf.s), ZSTR_LEN(buf.s), compressed); if (!compressed_len || compressed_len > buf.s->len) { php_error_docref(NULL, E_WARNING, "Compression failed"); efree(compressed); if (prefix->len) { zend_string_release(prefix_key); } return ret; } if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) { php_error_docref(NULL, E_WARNING, "Value is too big to be stored"); efree(compressed); if (prefix->len) { zend_string_release(prefix_key); } return ret; } flag |= YAC_ENTRY_COMPRESSED; flag |= (buf.s->len << YAC_ENTRY_ORIG_LEN_SHIT); ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), compressed, compressed_len, flag, ttl, add, tv); efree(compressed); } else { ret = yac_storage_update(ZSTR_VAL(key), ZSTR_LEN(key), ZSTR_VAL(buf.s), ZSTR_LEN(buf.s), flag, ttl, add, tv); } smart_str_free(&buf); } else { php_error_docref(NULL, E_WARNING, "Serialization failed"); smart_str_free(&buf); } } break; case IS_RESOURCE: php_error_docref(NULL, E_WARNING, "Type 'IS_RESOURCE' cannot be stored"); break; default: php_error_docref(NULL, E_WARNING, "Unsupported valued type to be stored '%d'", flag); break; } if (prefix->len) { zend_string_release(prefix_key); } return ret; }
static int yac_add_impl(char *prefix, uint prefix_len, char *key, uint len, zval *value, int ttl, int add TSRMLS_DC) /* {{{ */ { int ret = 0, flag = Z_TYPE_P(value); char *msg, buf[YAC_STORAGE_MAX_KEY_LEN]; time_t tv; if ((len + prefix_len) > YAC_STORAGE_MAX_KEY_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Key%s can not be longer than %d bytes", prefix_len? "(include prefix)" : "", YAC_STORAGE_MAX_KEY_LEN); return ret; } if (prefix_len) { len = snprintf(buf, sizeof(buf), "%s%s", prefix, key); key = (char *)buf; } tv = time(NULL); switch (Z_TYPE_P(value)) { case IS_NULL: ret = yac_storage_update(key, len, (char *)&flag, sizeof(int), flag, ttl, add, tv); break; case IS_BOOL: case IS_LONG: ret = yac_storage_update(key, len, (char *)&Z_LVAL_P(value), sizeof(long), flag, ttl, add, tv); break; case IS_DOUBLE: ret = yac_storage_update(key, len, (char *)&Z_DVAL_P(value), sizeof(double), flag, ttl, add, tv); break; case IS_STRING: case IS_CONSTANT: { if (Z_STRLEN_P(value) > YAC_G(compress_threshold) || Z_STRLEN_P(value) > YAC_STORAGE_MAX_ENTRY_LEN) { int compressed_len; char *compressed; /* if longer than this, then we can not stored the length in flag */ if (Z_STRLEN_P(value) > YAC_ENTRY_MAX_ORIG_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value)); return ret; } compressed = emalloc(Z_STRLEN_P(value) * 1.05); compressed_len = fastlz_compress(Z_STRVAL_P(value), Z_STRLEN_P(value), compressed); if (!compressed_len || compressed_len > Z_STRLEN_P(value)) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed"); efree(compressed); return ret; } if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too long(%d bytes) to be stored", Z_STRLEN_P(value)); efree(compressed); return ret; } flag |= YAC_ENTRY_COMPRESSED; flag |= (Z_STRLEN_P(value) << YAC_ENTRY_ORIG_LEN_SHIT); ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add, tv); efree(compressed); } else { ret = yac_storage_update(key, len, Z_STRVAL_P(value), Z_STRLEN_P(value), flag, ttl, add, tv); } } break; case IS_ARRAY: case IS_CONSTANT_ARRAY: case IS_OBJECT: { smart_str buf = {0}; #if ENABLE_MSGPACK if (yac_serializer_msgpack_pack(value, &buf, &msg TSRMLS_CC)) { #else if (yac_serializer_php_pack(value, &buf, &msg TSRMLS_CC)) { #endif if (buf.len > YAC_G(compress_threshold) || buf.len > YAC_STORAGE_MAX_ENTRY_LEN) { int compressed_len; char *compressed; if (buf.len > YAC_ENTRY_MAX_ORIG_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored"); return ret; } compressed = emalloc(buf.len * 1.05); compressed_len = fastlz_compress(buf.c, buf.len, compressed); if (!compressed_len || compressed_len > buf.len) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Compression failed"); efree(compressed); return ret; } if (compressed_len > YAC_STORAGE_MAX_ENTRY_LEN) { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Value is too big to be stored"); efree(compressed); return ret; } flag |= YAC_ENTRY_COMPRESSED; flag |= (buf.len << YAC_ENTRY_ORIG_LEN_SHIT); ret = yac_storage_update(key, len, compressed, compressed_len, flag, ttl, add, tv); efree(compressed); } else { ret = yac_storage_update(key, len, buf.c, buf.len, flag, ttl, add, tv); } smart_str_free(&buf); } else { php_error_docref(NULL TSRMLS_CC, E_WARNING, "Serialization failed"); smart_str_free(&buf); } } break; case IS_RESOURCE: php_error_docref(NULL TSRMLS_CC, E_WARNING, "Type 'IS_RESOURCE' cannot be stored"); break; default: php_error_docref(NULL TSRMLS_CC, E_WARNING, "Unsupported valued type to be stored '%d'", flag); break; } return ret; } /* }}} */ static int yac_add_multi_impl(char *prefix, uint prefix_len, zval *kvs, int ttl, int add TSRMLS_DC) /* {{{ */ { HashTable *ht = Z_ARRVAL_P(kvs); for (zend_hash_internal_pointer_reset(ht); zend_hash_has_more_elements(ht) == SUCCESS; zend_hash_move_forward(ht)) { char *key; ulong idx; zval **value; uint len, should_free = 0; if (zend_hash_get_current_data(ht, (void **)&value) == FAILURE) { continue; } switch (zend_hash_get_current_key_ex(ht, &key, &len, &idx, 0, NULL)) { case HASH_KEY_IS_LONG: len = spprintf(&key, 0, "%lu", idx) + 1; should_free = 1; case HASH_KEY_IS_STRING: if (yac_add_impl(prefix, prefix_len, key, len - 1, *value, ttl, add TSRMLS_CC)) { if (should_free) { efree(key); } continue; } else { if (should_free) { efree(key); } return 0; } default: continue; } } return 1; }
void Message::compress(Message::Compression type, int level) { if (isCompressed()) return; switch (type) { #ifdef BUILD_WITH_COMPRESSION_MINIZ case COMPRESS_MINIZ: { mz_ulong compressedSize = mz_compressBound(_size); int cmp_status; uint8_t *pCmp; pCmp = (mz_uint8 *)malloc((size_t)compressedSize); level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_MINIZ); // last argument is speed size tradeoff: BEST_SPEED [0-9] BEST_COMPRESSION cmp_status = mz_compress2(pCmp, &compressedSize, (const unsigned char *)_data.get(), _size, level); if (cmp_status != Z_OK) { // error free(pCmp); } _data = SharedPtr<char>((char*)pCmp); _meta["um.compressed"] = toStr(_size) + ":miniz"; _size = compressedSize; } return; #endif #ifdef BUILD_WITH_COMPRESSION_FASTLZ case COMPRESS_FASTLZ: { // The minimum input buffer size is 16. if (_size < 16) return; // The output buffer must be at least 5% larger than the input buffer and can not be smaller than 66 bytes. int compressedSize = _size + (double)_size * 0.06; if (compressedSize < 66) compressedSize = 66; char* compressedData = (char*)malloc(compressedSize); compressedSize = fastlz_compress(_data.get(), _size, compressedData); // If the input is not compressible, the return value might be larger than length if (compressedSize > _size) { free(compressedData); return; } // std::cout << _size << " -> " << compressedSize << " = " << ((float)compressedSize / (float)_size) << std::endl; _data = SharedPtr<char>((char*)compressedData); _meta["um.compressed"] = toStr(_size) + ":fastlz"; _size = compressedSize; } return; #endif #ifdef BUILD_WITH_COMPRESSION_LZ4 case COMPRESS_LZ4: { level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_LZ4); #ifdef LZ4_FRAME LZ4F_preferences_t lz4_preferences = { { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } }, level, /* compression level */ 0, /* autoflush */ { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ }; LZ4F_errorCode_t r; LZ4F_compressionContext_t ctx; r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(r)) { // printf("Failed to create context: error %zu", r); return; } #define LZ4_HEADER_SIZE 19 #define LZ4_FOOTER_SIZE 4 size_t n, offset = 0; size_t frameSize = LZ4F_compressBound(_size, &lz4_preferences); size_t compressedSize = frameSize + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; char* compressedData = (char*)malloc(compressedSize); n = LZ4F_compressBegin(ctx, compressedData, compressedSize, &lz4_preferences); if (LZ4F_isError(n)) { // printf("Failed to start compression: error %zu", n); LZ4F_freeCompressionContext(ctx); free(compressedData); return; } offset += n; n = LZ4F_compressUpdate(ctx, compressedData + offset, compressedSize - offset, _data.get(), _size, NULL); if (LZ4F_isError(n)) { // printf("Compression failed: error %zu", n); LZ4F_freeCompressionContext(ctx); free(compressedData); return; } offset += n; n = LZ4F_compressEnd(ctx, compressedData + offset, compressedSize - offset, NULL); if (LZ4F_isError(n)) { // printf("Failed to end compression: error %zu", n); LZ4F_freeCompressionContext(ctx); free(compressedData); return; } offset += n; _data = SharedPtr<char>((char*)compressedData); _meta["um.compressed"] = toStr(_size) + ":lz4"; _size = offset; LZ4F_freeCompressionContext(ctx); #else size_t compressedSize = LZ4_compressBound(_size); char* compressedData = (char*)malloc(compressedSize); int actualSize = 0; actualSize = LZ4_compress_fast(_data.get(), compressedData, _size, compressedSize, level); if (actualSize == 0) { free(compressedData); return; } _data = SharedPtr<char>((char*)compressedData); _meta["um.compressed"] = toStr(_size) + ":lz4"; _size = actualSize; #endif } return; #endif default: break; } }