int lz4_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen, int level, uchar_t chdr, void *data) { int rv; struct lz4_params *lzdat = (struct lz4_params *)data; int _srclen = srclen; uchar_t *dst2; if (lzdat->level == 1) { rv = LZ4_compress((const char *)src, (char *)dst, _srclen); } else if (lzdat->level == 2) { rv = LZ4_compress((const char *)src, (char *)dst, _srclen); if (rv == 0 || rv > *dstlen) { return (-1); } dst2 = (uchar_t *)slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv)); *((int *)dst2) = htonl(rv); rv = LZ4_compressHC((const char *)dst, (char *)(dst2 + sizeof (int)), rv); if (rv != 0) { rv += sizeof (int); memcpy(dst, dst2, rv); } slab_free(NULL, dst2); } else { rv = LZ4_compressHC((const char *)src, (char *)dst, _srclen); } if (rv == 0) { return (-1); } *dstlen = rv; return (0); }
static bool compress( T2 &buffer_out, const T1 &buffer_in, bool highest_compression = true ) { static const bool verbose = false; bool ret = false; if( 1 ) { // resize to worst case buffer_out.resize( LZ4_compressBound((int)(buffer_in.size())) ); // compress size_t compressed_size = highest_compression ? LZ4_compressHC( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() ) : LZ4_compress( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() ); ret = ( compressed_size > 0 && compressed_size < buffer_in.size() ); if( ret ) { // if ok, resize properly to unused space buffer_out.resize( compressed_size ); } if( verbose ) { // std::cout << moon9::echo( ret, compressed_size, buffer_in.size() ); } } return ret; }
static int lsm_lz4_xCompress(void *pCtx, void *pOut, int *pnOut, const void *pIn, int nIn) { int rc = LZ4_compress((const char*)pIn, (char*)pOut, nIn); *pnOut = rc; return (rc != 0 ? LSM_OK : LSM_ERROR); }
void compress_data(Data *d) { int slen = d->datalen; int olen = LZ4_COMPRESSBOUND(slen); char cbuf[olen]; #if LZ4_VERSION_MINOR >= 7 int ocnt = LZ4_compress_default((const char*)d->data, cbuf, slen, olen); #else /* In this API version, $cbuf is required to be large enough to handle * up to LZ4_COMPRESSBOUND($slen) bytes. Since we already allocate * this large of a buffer anyway, we just drop $olen. * * This is required for building against the liblz4 version in Debian * Jessie. */ int ocnt = LZ4_compress((const char*)d->data, cbuf, slen); #endif if(ocnt == 0) /* should never happen */ LERROR(EXIT_FAILURE, 0, "LZ4_compress_default() failed"); free(d->data); d->data = malloc(ocnt); GUARD_MALLOC(d->data); d->datalen = (size_t)ocnt; memcpy(d->data, (const void*)cbuf, d->datalen); }
size_t lz4Compress(std::shared_ptr<uint8_t> src, size_t uncompressedBytes, std::shared_ptr<uint8_t>& dst, bool /*highCompression*/) { if (uncompressedBytes > size_t(std::numeric_limits<int>::max())) throw std::runtime_error("Input data too big for LZ4 (max ~1.9GB)"); int const inputSize = static_cast<int>(uncompressedBytes); int const upperBound = LZ4_compressBound(inputSize); if (upperBound < 0) throw std::runtime_error("Input data too big for LZ4 (max ~1.9GB)"); dst.reset(new uint8_t[size_t(upperBound)], nonstd::DeleteArray<uint8_t>()); // NOTE: LZ4_compressHC compresses stronger and slower but decompresses // faster. It causes some bad memory accesses that's why we disable it here // for now. (e.g. processing the HeadAneurysm dataset with brick size 256 at // brick index 8) int compressedBytes = 0; //if (!highCompression) compressedBytes = LZ4_compress((const char*)src.get(), (char*)dst.get(), inputSize); /* else compressedBytes = LZ4_compressHC((const char*)src.get(), (char*)dst.get(), inputSize); */ assert(compressedBytes >= 0); return compressedBytes; }
void store_table(ubyte *table, char color) { FILE *F; char name[64]; if (!lz4_buf) { lz4_buf = malloc(4 + LZ4_compressBound(COPYSIZE)); if (!lz4_buf) { printf("Out of memory.\n"); exit(0); } } sprintf(name, "%s.%c", tablename, color); if (!(F = fopen(name, "wb"))) { printf("Could not open %s for writing.\n", name); exit(-1); } ubyte *ptr = table; long64 total = size; while (total > 0) { int chunk = COPYSIZE; if (total < chunk) chunk = total; total -= chunk; uint32 lz4_size = LZ4_compress((char *)ptr, lz4_buf + 4, chunk); ptr += chunk; *(uint32 *)lz4_buf = lz4_size; fwrite(lz4_buf, 1, lz4_size + 4, F); } fclose(F); }
void dnCalcDelta( const snapshot_t *olds, const snapshot_t *news, snapshotDelta_t *delta ) { unsigned char bitmask[ sizeof( snapshot_t ) / 8 ] = { 0 }; std::vector<char> stream; char *oldp = (char*)olds; char *newp = (char*)news; stream.reserve( 256 * 1024 ); for ( int i = 0; i < sizeof( snapshot_t ); i++, oldp++, newp++ ) { if ( *newp != *oldp ) { bitmask[ i/8 ] |= ( 1 << i%8 ); /* mark changed byte */ stream.push_back( *newp ); /* put new value to the stream */ } } std::vector<char> buffer; buffer.resize( LZ4_compressBound( sizeof( bitmask ) ) ); delta->bitmaskSize = LZ4_compress( (char*)&bitmask[0], &buffer[0], sizeof( bitmask )); delta->bitmask = (char*)malloc( delta->bitmaskSize ); memcpy( delta->bitmask, &buffer[0], delta->bitmaskSize ); delta->streamSize = stream.size(); delta->stream = (char*)malloc( delta->streamSize ); memcpy( delta->stream, &stream[0], delta->streamSize ); }
std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) { std::unique_ptr<IOBuf> clone; if (data->isChained()) { // LZ4 doesn't support streaming, so we have to coalesce clone = data->clone(); clone->coalesce(); data = clone.get(); } uint32_t extraSize = encodeSize() ? kMaxVarintLength64 : 0; auto out = IOBuf::create(extraSize + LZ4_compressBound(data->length())); if (encodeSize()) { encodeVarintToIOBuf(data->length(), out.get()); } int n; if (highCompression_) { n = LZ4_compressHC(reinterpret_cast<const char*>(data->data()), reinterpret_cast<char*>(out->writableTail()), data->length()); } else { n = LZ4_compress(reinterpret_cast<const char*>(data->data()), reinterpret_cast<char*>(out->writableTail()), data->length()); } CHECK_GE(n, 0); CHECK_LE(n, out->capacity()); out->append(n); return out; }
QByteArray Lz4CompressionTcpSocket::compressDataWithoutHeader(const QByteArray rawData) { QByteArray compressedData; compressedData.resize(maxCompressedSize(rawData.size())); int returnSize=LZ4_compress(rawData.constData(),compressedData.data(),rawData.size()); compressedData.resize(returnSize); return compressedData; }
static ZEND_FUNCTION(lz4_compress) { zval *data; char *output; int output_len, data_len; zend_bool high = 0; char *extra = NULL; int extra_len = -1; int offset = 0; if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z|bs", &data, &high, &extra, &extra_len) == FAILURE) { RETURN_FALSE; } if (Z_TYPE_P(data) != IS_STRING) { zend_error(E_WARNING, "lz4_compress : expects parameter to be string."); RETURN_FALSE; } if (extra && extra_len > 0) { offset = extra_len; } else { offset = sizeof(int); } data_len = Z_STRLEN_P(data); output = (char *)emalloc(LZ4_compressBound(data_len) + offset); if (!output) { zend_error(E_WARNING, "lz4_compress : memory error"); RETURN_FALSE; } if (extra && extra_len > 0) { memcpy(output, extra, offset); } else { /* Set the data length */ memcpy(output, &data_len, offset); } if (high) { output_len = LZ4_compressHC(Z_STRVAL_P(data), output + offset, data_len); } else { output_len = LZ4_compress(Z_STRVAL_P(data), output + offset, data_len); } if (output_len <= 0) { RETVAL_FALSE; } else { RETVAL_STRINGL(output, output_len + offset, 1); } efree(output); }
/* compression backend for LZ4 */ static int lz4_backend_compress(int level, const void* input, int length, void* output) { if (level == Z_BEST_COMPRESSION) { return LZ4_compressHC(input, output, length); } else { return LZ4_compress(input, output, length); } }
compressedSnapshot_t* dnCompressSnapshot( const snapshot_t *snapshot ) { compressedSnapshot_t *result; char *buffer = (char*)malloc( LZ4_compressBound( sizeof( snapshot_t ) ) ); int size = LZ4_compress( (char*)snapshot, buffer, sizeof( snapshot_t ) ); result = (compressedSnapshot_t*)malloc( sizeof( result->size ) + size ); result->size = size; memcpy( &result->data[0], buffer, size ); free( buffer ); return result; }
//assume dest is big enough to hold the compressed data int RemoteDesktop::Compression_Handler::Compress(const char* source, char* dest, int inputSize, int dest_size){ if (inputSize < 1024){ assert(inputSize <= dest_size); memcpy(dest, source, inputSize); return -1;//no compression occurred too small to waste time trying } auto dstsize = (int*)dest; auto compressedsize = LZ4_compress(source, dest + sizeof(int), inputSize); *dstsize = inputSize; assert(dest_size + sizeof(int) >= compressedsize); return compressedsize + sizeof(int);//return new size of compressed data }
size_t block_writer::write_block(size_t segment_id, size_t column_id, char* data, block_info block) { DASSERT_LT(segment_id, m_index_info.nsegments); DASSERT_LT(column_id, m_index_info.columns.size()); DASSERT_TRUE(m_output_files[segment_id] != nullptr); // try to compress the data size_t compress_bound = LZ4_compressBound(block.block_size); auto compression_buffer = m_buffer_pool.get_new_buffer(); compression_buffer->resize(compress_bound); char* cbuffer = compression_buffer->data(); size_t clen = compress_bound; clen = LZ4_compress(data, cbuffer, block.block_size); char* buffer_to_write = NULL; size_t buffer_to_write_len = 0; if (clen < COMPRESSION_DISABLE_THRESHOLD * block.block_size) { // compression has a benefit! block.flags |= LZ4_COMPRESSION; block.length = clen; buffer_to_write = cbuffer; buffer_to_write_len = clen; } else { // compression has no benefit! do not compress! // unset LZ4 block.flags &= (~(size_t)LZ4_COMPRESSION); block.length = block.block_size; buffer_to_write = data; buffer_to_write_len = block.block_size; } size_t padding = ((buffer_to_write_len + 4095) / 4096) * 4096 - buffer_to_write_len; ASSERT_LT(padding, 4096); // write! m_output_file_locks[segment_id].lock(); block.offset = m_output_bytes_written[segment_id]; m_output_bytes_written[segment_id] += buffer_to_write_len + padding; m_index_info.columns[column_id].segment_sizes[segment_id] += block.num_elem; m_output_files[segment_id]->write(buffer_to_write, buffer_to_write_len); m_output_files[segment_id]->write(padding_bytes, padding); m_blocks[segment_id][column_id].push_back(block); m_output_file_locks[segment_id].unlock(); m_buffer_pool.release_buffer(std::move(compression_buffer)); if (!m_output_files[segment_id]->good()) { log_and_throw_io_failure("Fail to write. Disk may be full."); } return buffer_to_write_len; }
bool Packet::compress() { unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) { int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); int cl = LZ4_compress((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl); if ((cl > 0)&&(cl < pl)) { (*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)cl),buf,cl); return true; } } (*this)[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED); return false; }
void Compress(const std::vector<T>& data, BufferT& out) { ssize_t countBS = sizeof(T)*data.size(); // allocate as much memory as we need int maxOut = LZ4_compressBound(countBS); out.resize(maxOut); int r = LZ4_compress((const char*)&data.front(), (char*)&out.front(), countBS); if(r == 0) { Util::fire_exception("lz4 compression failed"); } // trim to really used size out.resize(r); }
static SquashStatus squash_lz4_compress_buffer_unsafe (SquashCodec* codec, size_t* compressed_size, uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_size)], size_t uncompressed_size, const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_size)], SquashOptions* options) { int level = squash_codec_get_option_int_index (codec, options, SQUASH_LZ4_OPT_LEVEL); #if INT_MAX < SIZE_MAX if (SQUASH_UNLIKELY(INT_MAX < uncompressed_size) || SQUASH_UNLIKELY(INT_MAX < *compressed_size)) return squash_error (SQUASH_RANGE); #endif assert (*compressed_size >= LZ4_COMPRESSBOUND(uncompressed_size)); int lz4_r; if (level == 7) { lz4_r = LZ4_compress ((char*) uncompressed, (char*) compressed, (int) uncompressed_size); } else if (level < 7) { lz4_r = LZ4_compress_fast ((const char*) uncompressed, (char*) compressed, (int) uncompressed_size, (int) *compressed_size, squash_lz4_level_to_fast_mode (level)); } else { lz4_r = LZ4_compressHC2 ((char*) uncompressed, (char*) compressed, (int) uncompressed_size, squash_lz4_level_to_hc_level (level)); } #if SIZE_MAX < INT_MAX if (SQUASH_UNLIKELY(SIZE_MAX < lz4_r)) return squash_error (SQUASH_RANGE); #endif *compressed_size = lz4_r; return (lz4_r == 0) ? SQUASH_BUFFER_FULL : SQUASH_OK; }
void LZ4CompressedWriterAdapter::flush_buffer() { const size_t max_compressed_buffer_size = static_cast<size_t>(LZ4_compressBound(static_cast<int>(m_buffer_index))); ensure_minimum_size(m_compressed_buffer, max_compressed_buffer_size); const int compressed_buffer_size = LZ4_compress( reinterpret_cast<const char*>(&m_buffer[0]), reinterpret_cast<char*>(&m_compressed_buffer[0]), static_cast<int>(m_buffer_index)); m_file.write(static_cast<uint64>(m_buffer_index)); m_file.write(static_cast<uint64>(compressed_buffer_size)); m_file.write(&m_compressed_buffer[0], compressed_buffer_size); m_buffer_index = 0; }
static ZEND_FUNCTION(horde_lz4_compress) { zval *data; char *output; int header_offset = (sizeof(headerid) + sizeof(int)); int output_len, data_len; zend_bool high = 0; if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z|b", &data, &high) == FAILURE) { RETURN_FALSE; } if (Z_TYPE_P(data) != IS_STRING) { zend_error(E_WARNING, "horde_lz4_compress: uncompressed data must be a string."); RETURN_FALSE; } data_len = Z_STRLEN_P(data); output = (char *)emalloc(LZ4_compressBound(data_len) + header_offset); if (!output) { zend_error(E_WARNING, "horde_lz4_compress: memory error"); RETURN_FALSE; } *output = headerid; memcpy(output + sizeof(headerid), &data_len, sizeof(int)); if (high) { output_len = LZ4_compressHC(Z_STRVAL_P(data), output + header_offset, data_len); } else { output_len = LZ4_compress(Z_STRVAL_P(data), output + header_offset, data_len); } if (output_len <= 0) { RETVAL_FALSE; } else { RETVAL_STRINGL(output, output_len + header_offset, 1); } efree(output); }
static yrmcds_error send_data( yrmcds* c, yrmcds_command cmd, const char* key, size_t key_len, const char* data, size_t data_len, uint32_t flags, uint32_t expire, uint64_t cas, uint32_t* serial) { if( c == NULL || key == NULL || key_len == 0 || data == NULL || data_len == 0 ) return YRMCDS_BAD_ARGUMENT; int compressed = 0; #ifdef LIBYRMCDS_USE_LZ4 if( (c->compress_size > 0) && (data_len > c->compress_size) ) { if( flags & YRMCDS_FLAG_COMPRESS ) return YRMCDS_BAD_ARGUMENT; size_t bound = (size_t)LZ4_compressBound((int)data_len); char* new_data = (char*)malloc(bound + sizeof(uint32_t)); if( new_data == NULL ) return YRMCDS_OUT_OF_MEMORY; uint32_t new_size = (uint32_t)LZ4_compress(data, new_data + sizeof(uint32_t), (int)data_len); if( new_size == 0 ) { free(new_data); return YRMCDS_COMPRESS_FAILED; } hton32((uint32_t)data_len, new_data); flags |= YRMCDS_FLAG_COMPRESS; data_len = sizeof(uint32_t) + new_size; data = new_data; compressed = 1; } #endif // LIBYRMCDS_USE_LZ4 char extras[8]; hton32(flags, extras); hton32(expire, &extras[4]); yrmcds_error e = send_command(c, cmd, cas, serial, key_len, key, sizeof(extras), extras, data_len, data); if( compressed ) free((void*)data); return e; }
/* * lz4_compress -- * WiredTiger LZ4 compression. */ static int lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len, size_t *result_lenp, int *compression_failed) { char *lz4buf; size_t lz4_len; /* * The buffer should always be large enough due to the lz4_pre_size * call, but be paranoid and error if it isn't. */ if (dst_len < src_len + sizeof(size_t)) return (lz4_error(compressor, session, "LZ4 compress buffer too small", 0)); /* Store the length of the compressed block in the first 8 bytes. */ lz4buf = (char *)dst + sizeof(size_t); lz4_len = (size_t)LZ4_compress((const char *)src, lz4buf, (int)src_len); /* * Flag no-compression if the result was larger than the original * size or compression failed. */ if (lz4_len == 0 || lz4_len + sizeof(size_t) >= src_len) *compression_failed = 1; else { /* * On decompression, lz4 requires the exact compressed byte * count (the current value of lz4_len). WiredTiger does not * preserve that value, so save lz4_len at the beginning of the * destination buffer. */ *(size_t *)dst = lz4_len; *result_lenp = lz4_len + sizeof(size_t); *compression_failed = 0; } return (0); }
size_t lz4Compress(std::shared_ptr<uint8_t> src, size_t uncompressedBytes, std::shared_ptr<uint8_t>& dst, uint32_t compressionLevel) { if (uncompressedBytes > size_t(LZ4_MAX_INPUT_SIZE)) throw std::runtime_error("Input data too big for LZ4 (max LZ4_MAX_INPUT_SIZE)"); int const inputSize = static_cast<int>(uncompressedBytes); int const upperBound = LZ4_compressBound(inputSize); if (upperBound < 0) throw std::runtime_error("Input data too big for LZ4 (max LZ4_MAX_INPUT_SIZE)"); dst.reset(new uint8_t[size_t(upperBound)], nonstd::DeleteArray<uint8_t>()); if (compressionLevel > 17) compressionLevel = 17; else if (compressionLevel < 1) compressionLevel = 1; int compressedBytes = 0; switch (compressionLevel) { case 1: compressedBytes = LZ4_compress((const char*)src.get(), (char*)dst.get(), inputSize); break; default: // compression level 0 is default mode which seems to equal level 9 and HC compressedBytes = LZ4_compressHC2((const char*)src.get(), (char*)dst.get(), inputSize, compressionLevel - 1); break; } assert(compressedBytes >= 0); if (compressedBytes <= 0) throw std::runtime_error(std::string("LZ4_compress[HC] failed, returned value: ") + SysTools::ToString(compressedBytes)); return compressedBytes; }
JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_compressBytesDirect (JNIEnv *env, jobject thisj){ const char* uncompressed_bytes; char *compressed_bytes; // Get members of Lz4Compressor jobject clazz = (*env)->GetStaticObjectField(env, thisj, Lz4Compressor_clazz); jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_uncompressedDirectBuf); jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_uncompressedDirectBufLen); jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_compressedDirectBuf); jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_directBufferSize); // Get the input direct buffer LOCK_CLASS(env, clazz, "Lz4Compressor"); uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Compressor"); if (uncompressed_bytes == 0) { return (jint)0; } // Get the output direct buffer LOCK_CLASS(env, clazz, "Lz4Compressor"); compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Compressor"); if (compressed_bytes == 0) { return (jint)0; } compressed_direct_buf_len = LZ4_compress(uncompressed_bytes, compressed_bytes, uncompressed_direct_buf_len); if (compressed_direct_buf_len < 0){ THROW(env, "java/lang/InternalError", "LZ4_compress failed"); } (*env)->SetIntField(env, thisj, Lz4Compressor_uncompressedDirectBufLen, 0); return (jint)compressed_direct_buf_len; }
void LZ4Writer::compressAndWriteBuffer() { //First 8 bytes is LZOBlock. Skip them //Then there is a token which has encoded in the 0xF0 bits //the type of compression. compressedBuffer[8] = 32; //Then there is the compressed size but I will write it later... //... and finally the uncompressed size Utils::encode_intLE(compressedBuffer, 13, uncompressedBufferLen); int compressedSize = LZ4_compress(uncompressedBuffer, compressedBuffer + 21, uncompressedBufferLen); Utils::encode_intLE(compressedBuffer, 9, compressedSize); os.write(compressedBuffer, compressedSize + 21); if (!os.good()) { BOOST_LOG_TRIVIAL(error)<< "Problems with writing the last file"; } uncompressedBufferLen = 0; }
/* Bitshuffle and compress a single block. */ int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, const size_t size, const size_t elem_size) { int64_t nbytes, count; void* tmp_buf_bshuf = malloc(size * elem_size); if (tmp_buf_bshuf == NULL) return -1; void* tmp_buf_lz4 = malloc(LZ4_compressBound(size * elem_size)); if (tmp_buf_lz4 == NULL){ free(tmp_buf_bshuf); return -1; } size_t this_iter; void *in = ioc_get_in(C_ptr, &this_iter); ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size)); count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size); if (count < 0) { free(tmp_buf_lz4); free(tmp_buf_bshuf); return count; } nbytes = LZ4_compress((const char*) tmp_buf_bshuf, (char*) tmp_buf_lz4, size * elem_size); free(tmp_buf_bshuf); CHECK_ERR_FREE_LZ(nbytes, tmp_buf_lz4); void *out = ioc_get_out(C_ptr, &this_iter); ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4)); bshuf_write_uint32_BE(out, nbytes); memcpy((char *) out + 4, tmp_buf_lz4, nbytes); free(tmp_buf_lz4); return nbytes + 4; }
compressed_data compress(const T& in) { size_t byte_size = in.size() * sizeof(typename T::value_type); if (byte_size > 0xffff) throw std::runtime_error("too much data for compression"); compressed_data out; if (byte_size > 0) { auto in_ptr = reinterpret_cast<const char*>(&*in.begin()); out.resize(byte_size + 16); std::fill(out.begin(), out.end(), 0); out.unpacked_len = byte_size; int compressed_length = LZ4_compress(in_ptr, out.ptr(), byte_size); if (compressed_length <= 0) throw std::runtime_error("lz4 compression failed"); out.resize(compressed_length); } return out; }
aos_status_t *log_post_logs_with_sts_token(aos_pool_t *p, const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken, const char *project, const char *logstore, cJSON *root) { aos_string_t project_name, logstore_name; aos_table_t *headers = NULL; aos_table_t *resp_headers = NULL; log_request_options_t *options = NULL; aos_list_t buffer; unsigned char *md5 = NULL; char *buf = NULL; int64_t buf_len; char *b64_value = NULL; aos_buf_t *content = NULL; aos_status_t *s = NULL; options = log_request_options_create(p); options->config = log_config_create(options->pool); aos_str_set(&(options->config->endpoint), endpoint); aos_str_set(&(options->config->access_key_id), accesskeyId); aos_str_set(&(options->config->access_key_secret), accessKey); if(stsToken != NULL) { aos_str_set(&(options->config->sts_token), stsToken); } options->ctl = aos_http_controller_create(options->pool, 0); headers = aos_table_make(p, 5); apr_table_set(headers, LOG_API_VERSION, "0.6.0"); apr_table_set(headers, LOG_COMPRESS_TYPE, "lz4"); apr_table_set(headers, LOG_SIGNATURE_METHOD, "hmac-sha1"); apr_table_set(headers, LOG_CONTENT_TYPE, "application/json"); aos_str_set(&project_name, project); aos_str_set(&logstore_name, logstore); aos_list_init(&buffer); char *body = cJSON_PrintUnformatted(root); if(body == NULL) { s = aos_status_create(options->pool); aos_status_set(s, 400, AOS_CLIENT_ERROR_CODE, "fail to format cJSON data"); return s; } int org_body_size = strlen(body); apr_table_set(headers, LOG_BODY_RAW_SIZE, apr_itoa(options->pool, org_body_size)); int compress_bound = LZ4_compressBound(org_body_size); char *compress_data = aos_pcalloc(options->pool, compress_bound); int compressed_size = LZ4_compress(body, compress_data, org_body_size); if(compressed_size <= 0) { s = aos_status_create(options->pool); aos_status_set(s, 400, AOS_CLIENT_ERROR_CODE, "fail to compress json data"); return s; } content = aos_buf_pack(options->pool, compress_data, compressed_size); aos_list_add_tail(&content->node, &buffer); //add Content-MD5 buf_len = aos_buf_list_len(&buffer); buf = aos_buf_list_content(options->pool, &buffer); md5 = aos_md5(options->pool, buf, (apr_size_t)buf_len); b64_value = aos_pcalloc(options->pool, 50); int loop = 0; for(; loop < 16; ++loop) { unsigned char a = ((*md5)>>4) & 0xF, b = (*md5) & 0xF; b64_value[loop<<1] = a > 9 ? (a - 10 + 'A') : (a + '0'); b64_value[(loop<<1)|1] = b > 9 ? (b - 10 + 'A') : (b + '0'); ++md5; } b64_value[loop<<1] = '\0'; apr_table_set(headers, LOG_CONTENT_MD5, b64_value); s = log_post_logs_from_buffer(options, &project_name, &logstore_name, &buffer, headers, &resp_headers); free(body); return s; }
void save_table(ubyte *table, char color) { int i; FILE *F; char name[64]; ubyte v[256]; if (!lz4_buf) { lz4_buf = malloc(4 + LZ4_compressBound(COPYSIZE)); if (!lz4_buf) { printf("Out of memory.\n"); exit(0); } } sprintf(name, "%s.%c.%d", tablename, color, num_saves); if (!(F = fopen(name, "wb"))) { printf("Could not open %s for writing.\n", name); exit(-1); } for (i = 0; i < 256; i++) v[i] = 0; #ifndef SUICIDE if (num_saves == 0) { v[MATE] = 255; for (i = 0; i < DRAW_RULE; i++) { v[WIN_IN_ONE + i] = 1 + i; v[LOSS_IN_ONE - i] = 254 - i; } for (; i < REDUCE_PLY - 2; i++) { v[WIN_IN_ONE + i + 1] = 1 + DRAW_RULE + (i - DRAW_RULE) / 2; v[LOSS_IN_ONE - i] = 254 - DRAW_RULE - (i - DRAW_RULE) / 2; } v[LOSS_IN_ONE - i] = 254 - DRAW_RULE - (i - DRAW_RULE) / 2; } else { for (i = 0; i < REDUCE_PLY_RED; i++) { v[CAPT_CWIN_RED + i + 2] = 1 + ((reduce_cnt & 1) + i) / 2; v[LOSS_IN_ONE - i - 1] = 255 - ((reduce_cnt & 1) + i + 1) / 2; } } #else if (num_saves == 0) { for (i = 3; i <= DRAW_RULE; i++) v[BASE_WIN + i] = 1 + (i - 3); v[BASE_WIN + DRAW_RULE + 1] = DRAW_RULE - 1; // should we save THREAT_CWIN1/2 (separately) ? for (i = DRAW_RULE + 2; i < REDUCE_PLY; i++) v[BASE_WIN + i + 2] = DRAW_RULE - 1 + (i - DRAW_RULE - 1) / 2; for (i = 2; i <= DRAW_RULE; i++) v[BASE_LOSS - i] = 255 - (i - 2); for (; i < REDUCE_PLY; i++) v[BASE_LOSS - i] = 255 - (DRAW_RULE - 2) - 1 - (i - DRAW_RULE - 1) / 2; } else { for (i = 0; i < REDUCE_PLY_RED; i++) { v[BASE_WIN + i + 6] = 1 + ((reduce_cnt & 1) + i) / 2; v[BASE_LOSS - i - 4] = 255 - ((reduce_cnt & 1) + i) / 2; } } #endif ubyte *ptr = table; long64 total = size; while (total > 0) { int chunk = COPYSIZE; if (total < chunk) chunk = total; total -= chunk; for (i = 0; i < chunk; i++) copybuf[i] = v[ptr[i]]; ptr += chunk; uint32 lz4_size = LZ4_compress((char *)copybuf, lz4_buf + 4, chunk); *(uint32 *)lz4_buf = lz4_size; fwrite(lz4_buf, 1, lz4_size + 4, F); } fclose(F); }
int compress_file(char* input_filename, char* output_filename) { U64 filesize = 0; U64 compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE; char* in_buff; char* out_buff; char stdinout[] = "std"; FILE* finput; if (!strcmp (input_filename, stdinout)) { fprintf(stderr, "Using stdin for input\n"); finput = stdin; } else { finput = fopen( input_filename, "rb" ); } FILE* foutput; if (!strcmp (output_filename, stdinout)) { fprintf(stderr, "Using stdout for output\n"); foutput = stdout; } else { foutput = fopen( output_filename, "wb" ); } if ( finput==0 ) { printf("Pb opening %s\n", input_filename); return 2; } if ( foutput==0) { printf("Pb opening %s\n", output_filename); return 3; } // Allocate Memory in_buff = malloc(CHUNKSIZE); out_buff = malloc(OUT_CHUNKSIZE); // Write Archive Header *(U32*)out_buff = ARCHIVE_MAGICNUMBER; fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); // Main Loop while (1) { int outSize; // Read Block int inSize = fread(in_buff, 1, CHUNKSIZE, finput); if( inSize<=0 ) break; filesize += inSize; // Compress Block outSize = LZ4_compress(in_buff, out_buff+4, inSize); * (U32*) out_buff = outSize; compressedfilesize += outSize+4; // Write Block fwrite(out_buff, 1, outSize+4, foutput); } // Status fprintf(stderr, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); fclose(finput); fclose(foutput); return 0; }
int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { unsigned long long bytes = 0; unsigned long long cbytes = 0; unsigned long long hcbytes = 0; unsigned long long ccbytes = 0; void* CNBuffer; char* compressedBuffer; char* decodedBuffer; # define FUZ_max LZ4_COMPRESSBOUND(LEN) unsigned int randState=seed; int ret, cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \ printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } # define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); } void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); void* LZ4continue; LZ4_stream_t LZ4dict; U32 crcOrig, crcCheck; int displayRefresh; // init memset(&LZ4dict, 0, sizeof(LZ4dict)); // Create compressible test buffer CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); // display refresh rate switch(displayLevel) { case 0: displayRefresh = nbCycles+1; break; case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break; case 2: displayRefresh=89; break; default : displayRefresh=1; } // move to startCycle for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { // synd rand & dict int dictSize, blockSize, blockStart; char* dict; char* block; blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; if (dictSize > blockStart) dictSize = blockStart; block = ((char*)CNBuffer) + blockStart; dict = block - dictSize; LZ4_loadDict(&LZ4dict, dict, dictSize); LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); LZ4_loadDict(&LZ4dict, dict, dictSize); LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); LZ4_loadDict(&LZ4dict, dict, dictSize); LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); } // Test loop for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++) { int testNb = 0; char* dict; char* block; int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; int blockContinueCompressedSize; if ((cycleNb%displayRefresh) == 0) { printf("\r%7i /%7i - ", cycleNb, nbCycles); fflush(stdout); } // Select block to test blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; if (dictSize > blockStart) dictSize = blockStart; block = ((char*)CNBuffer) + blockStart; dict = block - dictSize; /* Compression tests */ // Test compression HC FUZ_DISPLAYTEST; ret = LZ4_compressHC(block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; // Test compression HC using external state FUZ_DISPLAYTEST; ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); // Test compression using external state FUZ_DISPLAYTEST; ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); // Test compression FUZ_DISPLAYTEST; ret = LZ4_compress(block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); compressedSize = ret; /* Decompression tests */ crcOrig = XXH32(block, blockSize, 0); // Test decoding with output size being exactly what's necessary => must work FUZ_DISPLAYTEST; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); // Test decoding with one byte missing => must fail FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); // Test decoding with one byte too much => must fail FUZ_DISPLAYTEST; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); // Test decoding with output size exactly what's necessary => must work FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); // Test decoding with more than enough output size => must work FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; decodedBuffer[blockSize+1] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space"); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); // Test decoding with output size being one byte too short => must fail FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size"); // Test decoding with output size being 10 bytes too short => must fail FUZ_DISPLAYTEST; if (blockSize>10) { decodedBuffer[blockSize-10] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short"); FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size"); } // Test decoding with input size being one byte too short => must fail FUZ_DISPLAYTEST; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); // Test decoding with input size being one byte too large => must fail FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); // Test partial decoding with target output size being max/2 => must work FUZ_DISPLAYTEST; ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); // Test partial decoding with target output size being just below max => must work FUZ_DISPLAYTEST; ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); /* Test Compression with limited output size */ // Test compression with output size being exactly what's necessary (should work) FUZ_DISPLAYTEST; ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); // Test compression with output size being exactly what's necessary and external state (should work) FUZ_DISPLAYTEST; ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); // Test HC compression with output size being exactly what's necessary (should work) FUZ_DISPLAYTEST; ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); // Test HC compression with output size being exactly what's necessary (should work) FUZ_DISPLAYTEST; ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); // Test compression with just one missing byte into output buffer => must fail FUZ_DISPLAYTEST; compressedBuffer[compressedSize-1] = 0; ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1); FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)"); FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer") // Test HC compression with just one missing byte into output buffer => must fail FUZ_DISPLAYTEST; compressedBuffer[compressedSize-1] = 0; ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)"); FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer") /* Dictionary tests */ // Compress using dictionary FUZ_DISPLAYTEST; LZ4continue = LZ4_create (dict); LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); LZ4_free (LZ4continue); // Decompress with dictionary as prefix FUZ_DISPLAYTEST; memcpy(decodedBuffer, dict, dictSize); ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); if (crcCheck!=crcOrig) { int i=0; while (block[i]==decodedBuffer[i]) i++; printf("Wrong Byte at position %i/%i\n", i, blockSize); } FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data"); // Compress using External dictionary FUZ_DISPLAYTEST; dict -= 9; // Separation, so it is an ExtDict if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); // Decompress with dictionary as external FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) { int i=0; while (block[i]==decodedBuffer[i]) i++; printf("Wrong Byte at position %i/%i\n", i, blockSize); } FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); FUZ_DISPLAYTEST; if (blockSize > 10) { decodedBuffer[blockSize-10] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize); } // Fill stats bytes += blockSize; cbytes += compressedSize; hcbytes += HCcompressedSize; ccbytes += blockContinueCompressedSize; } printf("\r%7i /%7i - ", cycleNb, nbCycles); printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); // unalloc if(!no_prompt) getchar(); free(CNBuffer); free(compressedBuffer); free(decodedBuffer); free(stateLZ4); free(stateLZ4HC); return 0; _output_error: if(!no_prompt) getchar(); free(CNBuffer); free(compressedBuffer); free(decodedBuffer); free(stateLZ4); free(stateLZ4HC); return 1; }