int
lz4_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
int level, uchar_t chdr, void *data)
{
int rv;
struct lz4_params *lzdat = (struct lz4_params *)data;
int _srclen = srclen;
uchar_t *dst2;
if (lzdat->level == 1) {
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
} else if (lzdat->level == 2) {
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
if (rv == 0 || rv > *dstlen) {
return (-1);
}
dst2 = (uchar_t *)slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv));
*((int *)dst2) = htonl(rv);
rv = LZ4_compressHC((const char *)dst, (char *)(dst2 + sizeof (int)), rv);
if (rv != 0) {
rv += sizeof (int);
memcpy(dst, dst2, rv);
}
slab_free(NULL, dst2);
} else {
rv = LZ4_compressHC((const char *)src, (char *)dst, _srclen);
}
if (rv == 0) {
return (-1);
}
*dstlen = rv;
return (0);
}
static bool compress( T2 &buffer_out, const T1 &buffer_in, bool highest_compression = true )
{
static const bool verbose = false;
bool ret = false;
if( 1 )
{
// resize to worst case
buffer_out.resize( LZ4_compressBound((int)(buffer_in.size())) );
// compress
size_t compressed_size = highest_compression ?
LZ4_compressHC( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() )
:
LZ4_compress( &buffer_in.at(0), &buffer_out.at(0), buffer_in.size() );
ret = ( compressed_size > 0 && compressed_size < buffer_in.size() );
if( ret )
{
// if ok, resize properly to unused space
buffer_out.resize( compressed_size );
}
if( verbose )
{
// std::cout << moon9::echo( ret, compressed_size, buffer_in.size() );
}
}
return ret;
}
static int
lsm_lz4_xCompress(void *pCtx, void *pOut, int *pnOut, const void *pIn, int nIn)
{
int rc = LZ4_compress((const char*)pIn, (char*)pOut, nIn);
*pnOut = rc;
return (rc != 0 ? LSM_OK : LSM_ERROR);
}
void compress_data(Data *d) {
int slen = d->datalen;
int olen = LZ4_COMPRESSBOUND(slen);
char cbuf[olen];
#if LZ4_VERSION_MINOR >= 7
int ocnt = LZ4_compress_default((const char*)d->data, cbuf, slen, olen);
#else
/* In this API version, $cbuf is required to be large enough to handle
* up to LZ4_COMPRESSBOUND($slen) bytes. Since we already allocate
* this large of a buffer anyway, we just drop $olen.
*
* This is required for building against the liblz4 version in Debian
* Jessie.
*/
int ocnt = LZ4_compress((const char*)d->data, cbuf, slen);
#endif
if(ocnt == 0) /* should never happen */
LERROR(EXIT_FAILURE, 0, "LZ4_compress_default() failed");
free(d->data);
d->data = malloc(ocnt);
GUARD_MALLOC(d->data);
d->datalen = (size_t)ocnt;
memcpy(d->data, (const void*)cbuf, d->datalen);
}
size_t lz4Compress(std::shared_ptr<uint8_t> src, size_t uncompressedBytes,
std::shared_ptr<uint8_t>& dst,
bool /*highCompression*/)
{
if (uncompressedBytes > size_t(std::numeric_limits<int>::max()))
throw std::runtime_error("Input data too big for LZ4 (max ~1.9GB)");
int const inputSize = static_cast<int>(uncompressedBytes);
int const upperBound = LZ4_compressBound(inputSize);
if (upperBound < 0)
throw std::runtime_error("Input data too big for LZ4 (max ~1.9GB)");
dst.reset(new uint8_t[size_t(upperBound)], nonstd::DeleteArray<uint8_t>());
// NOTE: LZ4_compressHC compresses stronger and slower but decompresses
// faster. It causes some bad memory accesses that's why we disable it here
// for now. (e.g. processing the HeadAneurysm dataset with brick size 256 at
// brick index 8)
int compressedBytes = 0;
//if (!highCompression)
compressedBytes = LZ4_compress((const char*)src.get(),
(char*)dst.get(),
inputSize);
/*
else
compressedBytes = LZ4_compressHC((const char*)src.get(),
(char*)dst.get(),
inputSize);
*/
assert(compressedBytes >= 0);
return compressedBytes;
}
void store_table(ubyte *table, char color)
{
FILE *F;
char name[64];
if (!lz4_buf) {
lz4_buf = malloc(4 + LZ4_compressBound(COPYSIZE));
if (!lz4_buf) {
printf("Out of memory.\n");
exit(0);
}
}
sprintf(name, "%s.%c", tablename, color);
if (!(F = fopen(name, "wb"))) {
printf("Could not open %s for writing.\n", name);
exit(-1);
}
ubyte *ptr = table;
long64 total = size;
while (total > 0) {
int chunk = COPYSIZE;
if (total < chunk) chunk = total;
total -= chunk;
uint32 lz4_size = LZ4_compress((char *)ptr, lz4_buf + 4, chunk);
ptr += chunk;
*(uint32 *)lz4_buf = lz4_size;
fwrite(lz4_buf, 1, lz4_size + 4, F);
}
fclose(F);
}
void dnCalcDelta( const snapshot_t *olds, const snapshot_t *news, snapshotDelta_t *delta ) {
unsigned char bitmask[ sizeof( snapshot_t ) / 8 ] = { 0 };
std::vector<char> stream;
char *oldp = (char*)olds;
char *newp = (char*)news;
stream.reserve( 256 * 1024 );
for ( int i = 0; i < sizeof( snapshot_t ); i++, oldp++, newp++ ) {
if ( *newp != *oldp ) {
bitmask[ i/8 ] |= ( 1 << i%8 ); /* mark changed byte */
stream.push_back( *newp ); /* put new value to the stream */
}
}
std::vector<char> buffer;
buffer.resize( LZ4_compressBound( sizeof( bitmask ) ) );
delta->bitmaskSize = LZ4_compress( (char*)&bitmask[0], &buffer[0], sizeof( bitmask ));
delta->bitmask = (char*)malloc( delta->bitmaskSize );
memcpy( delta->bitmask, &buffer[0], delta->bitmaskSize );
delta->streamSize = stream.size();
delta->stream = (char*)malloc( delta->streamSize );
memcpy( delta->stream, &stream[0], delta->streamSize );
}
std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {
std::unique_ptr<IOBuf> clone;
if (data->isChained()) {
// LZ4 doesn't support streaming, so we have to coalesce
clone = data->clone();
clone->coalesce();
data = clone.get();
}
uint32_t extraSize = encodeSize() ? kMaxVarintLength64 : 0;
auto out = IOBuf::create(extraSize + LZ4_compressBound(data->length()));
if (encodeSize()) {
encodeVarintToIOBuf(data->length(), out.get());
}
int n;
if (highCompression_) {
n = LZ4_compressHC(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
} else {
n = LZ4_compress(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
}
CHECK_GE(n, 0);
CHECK_LE(n, out->capacity());
out->append(n);
return out;
}
QByteArray Lz4CompressionTcpSocket::compressDataWithoutHeader(const QByteArray rawData)
{
QByteArray compressedData;
compressedData.resize(maxCompressedSize(rawData.size()));
int returnSize=LZ4_compress(rawData.constData(),compressedData.data(),rawData.size());
compressedData.resize(returnSize);
return compressedData;
}
static ZEND_FUNCTION(lz4_compress)
{
zval *data;
char *output;
int output_len, data_len;
zend_bool high = 0;
char *extra = NULL;
int extra_len = -1;
int offset = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|bs", &data, &high,
&extra, &extra_len) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"lz4_compress : expects parameter to be string.");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
offset = extra_len;
} else {
offset = sizeof(int);
}
data_len = Z_STRLEN_P(data);
output = (char *)emalloc(LZ4_compressBound(data_len) + offset);
if (!output) {
zend_error(E_WARNING, "lz4_compress : memory error");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
memcpy(output, extra, offset);
} else {
/* Set the data length */
memcpy(output, &data_len, offset);
}
if (high) {
output_len = LZ4_compressHC(Z_STRVAL_P(data), output + offset, data_len);
} else {
output_len = LZ4_compress(Z_STRVAL_P(data), output + offset, data_len);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
RETVAL_STRINGL(output, output_len + offset, 1);
}
efree(output);
}
/* compression backend for LZ4 */
static int lz4_backend_compress(int level, const void* input, int length,
void* output) {
if (level == Z_BEST_COMPRESSION) {
return LZ4_compressHC(input, output, length);
}
else {
return LZ4_compress(input, output, length);
}
}
compressedSnapshot_t* dnCompressSnapshot( const snapshot_t *snapshot ) {
compressedSnapshot_t *result;
char *buffer = (char*)malloc( LZ4_compressBound( sizeof( snapshot_t ) ) );
int size = LZ4_compress( (char*)snapshot, buffer, sizeof( snapshot_t ) );
result = (compressedSnapshot_t*)malloc( sizeof( result->size ) + size );
result->size = size;
memcpy( &result->data[0], buffer, size );
free( buffer );
return result;
}
//assume dest is big enough to hold the compressed data
int RemoteDesktop::Compression_Handler::Compress(const char* source, char* dest, int inputSize, int dest_size){
if (inputSize < 1024){
assert(inputSize <= dest_size);
memcpy(dest, source, inputSize);
return -1;//no compression occurred too small to waste time trying
}
auto dstsize = (int*)dest;
auto compressedsize = LZ4_compress(source, dest + sizeof(int), inputSize);
*dstsize = inputSize;
assert(dest_size + sizeof(int) >= compressedsize);
return compressedsize + sizeof(int);//return new size of compressed data
}
size_t block_writer::write_block(size_t segment_id,
size_t column_id,
char* data,
block_info block) {
DASSERT_LT(segment_id, m_index_info.nsegments);
DASSERT_LT(column_id, m_index_info.columns.size());
DASSERT_TRUE(m_output_files[segment_id] != nullptr);
// try to compress the data
size_t compress_bound = LZ4_compressBound(block.block_size);
auto compression_buffer = m_buffer_pool.get_new_buffer();
compression_buffer->resize(compress_bound);
char* cbuffer = compression_buffer->data();
size_t clen = compress_bound;
clen = LZ4_compress(data, cbuffer, block.block_size);
char* buffer_to_write = NULL;
size_t buffer_to_write_len = 0;
if (clen < COMPRESSION_DISABLE_THRESHOLD * block.block_size) {
// compression has a benefit!
block.flags |= LZ4_COMPRESSION;
block.length = clen;
buffer_to_write = cbuffer;
buffer_to_write_len = clen;
} else {
// compression has no benefit! do not compress!
// unset LZ4
block.flags &= (~(size_t)LZ4_COMPRESSION);
block.length = block.block_size;
buffer_to_write = data;
buffer_to_write_len = block.block_size;
}
size_t padding = ((buffer_to_write_len + 4095) / 4096) * 4096 - buffer_to_write_len;
ASSERT_LT(padding, 4096);
// write!
m_output_file_locks[segment_id].lock();
block.offset = m_output_bytes_written[segment_id];
m_output_bytes_written[segment_id] += buffer_to_write_len + padding;
m_index_info.columns[column_id].segment_sizes[segment_id] += block.num_elem;
m_output_files[segment_id]->write(buffer_to_write, buffer_to_write_len);
m_output_files[segment_id]->write(padding_bytes, padding);
m_blocks[segment_id][column_id].push_back(block);
m_output_file_locks[segment_id].unlock();
m_buffer_pool.release_buffer(std::move(compression_buffer));
if (!m_output_files[segment_id]->good()) {
log_and_throw_io_failure("Fail to write. Disk may be full.");
}
return buffer_to_write_len;
}
bool Packet::compress()
{
unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2];
if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) {
int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD);
int cl = LZ4_compress((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl);
if ((cl > 0)&&(cl < pl)) {
(*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED;
setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD);
memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)cl),buf,cl);
return true;
}
}
(*this)[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED);
return false;
}
void
Compress(const std::vector<T>& data, BufferT& out)
{
ssize_t countBS = sizeof(T)*data.size();
// allocate as much memory as we need
int maxOut = LZ4_compressBound(countBS);
out.resize(maxOut);
int r = LZ4_compress((const char*)&data.front(), (char*)&out.front(), countBS);
if(r == 0) {
Util::fire_exception("lz4 compression failed");
}
// trim to really used size
out.resize(r);
}
static SquashStatus
squash_lz4_compress_buffer_unsafe (SquashCodec* codec,
size_t* compressed_size,
uint8_t compressed[SQUASH_ARRAY_PARAM(*compressed_size)],
size_t uncompressed_size,
const uint8_t uncompressed[SQUASH_ARRAY_PARAM(uncompressed_size)],
SquashOptions* options) {
int level = squash_codec_get_option_int_index (codec, options, SQUASH_LZ4_OPT_LEVEL);
#if INT_MAX < SIZE_MAX
if (SQUASH_UNLIKELY(INT_MAX < uncompressed_size) ||
SQUASH_UNLIKELY(INT_MAX < *compressed_size))
return squash_error (SQUASH_RANGE);
#endif
assert (*compressed_size >= LZ4_COMPRESSBOUND(uncompressed_size));
int lz4_r;
if (level == 7) {
lz4_r = LZ4_compress ((char*) uncompressed,
(char*) compressed,
(int) uncompressed_size);
} else if (level < 7) {
lz4_r = LZ4_compress_fast ((const char*) uncompressed,
(char*) compressed,
(int) uncompressed_size,
(int) *compressed_size,
squash_lz4_level_to_fast_mode (level));
} else {
lz4_r = LZ4_compressHC2 ((char*) uncompressed,
(char*) compressed,
(int) uncompressed_size,
squash_lz4_level_to_hc_level (level));
}
#if SIZE_MAX < INT_MAX
if (SQUASH_UNLIKELY(SIZE_MAX < lz4_r))
return squash_error (SQUASH_RANGE);
#endif
*compressed_size = lz4_r;
return (lz4_r == 0) ? SQUASH_BUFFER_FULL : SQUASH_OK;
}
void LZ4CompressedWriterAdapter::flush_buffer()
{
const size_t max_compressed_buffer_size =
static_cast<size_t>(LZ4_compressBound(static_cast<int>(m_buffer_index)));
ensure_minimum_size(m_compressed_buffer, max_compressed_buffer_size);
const int compressed_buffer_size =
LZ4_compress(
reinterpret_cast<const char*>(&m_buffer[0]),
reinterpret_cast<char*>(&m_compressed_buffer[0]),
static_cast<int>(m_buffer_index));
m_file.write(static_cast<uint64>(m_buffer_index));
m_file.write(static_cast<uint64>(compressed_buffer_size));
m_file.write(&m_compressed_buffer[0], compressed_buffer_size);
m_buffer_index = 0;
}
static ZEND_FUNCTION(horde_lz4_compress)
{
zval *data;
char *output;
int header_offset = (sizeof(headerid) + sizeof(int));
int output_len, data_len;
zend_bool high = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|b", &data, &high) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"horde_lz4_compress: uncompressed data must be a string.");
RETURN_FALSE;
}
data_len = Z_STRLEN_P(data);
output = (char *)emalloc(LZ4_compressBound(data_len) + header_offset);
if (!output) {
zend_error(E_WARNING, "horde_lz4_compress: memory error");
RETURN_FALSE;
}
*output = headerid;
memcpy(output + sizeof(headerid), &data_len, sizeof(int));
if (high) {
output_len = LZ4_compressHC(Z_STRVAL_P(data), output + header_offset, data_len);
} else {
output_len = LZ4_compress(Z_STRVAL_P(data), output + header_offset, data_len);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
RETVAL_STRINGL(output, output_len + header_offset, 1);
}
efree(output);
}
static yrmcds_error send_data(
yrmcds* c, yrmcds_command cmd, const char* key, size_t key_len,
const char* data, size_t data_len, uint32_t flags, uint32_t expire,
uint64_t cas, uint32_t* serial) {
if( c == NULL || key == NULL || key_len == 0 ||
data == NULL || data_len == 0 )
return YRMCDS_BAD_ARGUMENT;
int compressed = 0;
#ifdef LIBYRMCDS_USE_LZ4
if( (c->compress_size > 0) && (data_len > c->compress_size) ) {
if( flags & YRMCDS_FLAG_COMPRESS )
return YRMCDS_BAD_ARGUMENT;
size_t bound = (size_t)LZ4_compressBound((int)data_len);
char* new_data = (char*)malloc(bound + sizeof(uint32_t));
if( new_data == NULL )
return YRMCDS_OUT_OF_MEMORY;
uint32_t new_size =
(uint32_t)LZ4_compress(data, new_data + sizeof(uint32_t),
(int)data_len);
if( new_size == 0 ) {
free(new_data);
return YRMCDS_COMPRESS_FAILED;
}
hton32((uint32_t)data_len, new_data);
flags |= YRMCDS_FLAG_COMPRESS;
data_len = sizeof(uint32_t) + new_size;
data = new_data;
compressed = 1;
}
#endif // LIBYRMCDS_USE_LZ4
char extras[8];
hton32(flags, extras);
hton32(expire, &extras[4]);
yrmcds_error e = send_command(c, cmd, cas, serial, key_len, key,
sizeof(extras), extras, data_len, data);
if( compressed )
free((void*)data);
return e;
}
/*
* lz4_compress --
* WiredTiger LZ4 compression.
*/
static int
lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session,
uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_len,
size_t *result_lenp, int *compression_failed)
{
char *lz4buf;
size_t lz4_len;
/*
* The buffer should always be large enough due to the lz4_pre_size
* call, but be paranoid and error if it isn't.
*/
if (dst_len < src_len + sizeof(size_t))
return (lz4_error(compressor, session,
"LZ4 compress buffer too small", 0));
/* Store the length of the compressed block in the first 8 bytes. */
lz4buf = (char *)dst + sizeof(size_t);
lz4_len = (size_t)LZ4_compress((const char *)src, lz4buf, (int)src_len);
/*
* Flag no-compression if the result was larger than the original
* size or compression failed.
*/
if (lz4_len == 0 || lz4_len + sizeof(size_t) >= src_len)
*compression_failed = 1;
else {
/*
* On decompression, lz4 requires the exact compressed byte
* count (the current value of lz4_len). WiredTiger does not
* preserve that value, so save lz4_len at the beginning of the
* destination buffer.
*/
*(size_t *)dst = lz4_len;
*result_lenp = lz4_len + sizeof(size_t);
*compression_failed = 0;
}
return (0);
}
size_t lz4Compress(std::shared_ptr<uint8_t> src, size_t uncompressedBytes,
std::shared_ptr<uint8_t>& dst,
uint32_t compressionLevel)
{
if (uncompressedBytes > size_t(LZ4_MAX_INPUT_SIZE))
throw std::runtime_error("Input data too big for LZ4 (max LZ4_MAX_INPUT_SIZE)");
int const inputSize = static_cast<int>(uncompressedBytes);
int const upperBound = LZ4_compressBound(inputSize);
if (upperBound < 0)
throw std::runtime_error("Input data too big for LZ4 (max LZ4_MAX_INPUT_SIZE)");
dst.reset(new uint8_t[size_t(upperBound)], nonstd::DeleteArray<uint8_t>());
if (compressionLevel > 17)
compressionLevel = 17;
else if (compressionLevel < 1)
compressionLevel = 1;
int compressedBytes = 0;
switch (compressionLevel) {
case 1:
compressedBytes = LZ4_compress((const char*)src.get(),
(char*)dst.get(),
inputSize);
break;
default:
// compression level 0 is default mode which seems to equal level 9 and HC
compressedBytes = LZ4_compressHC2((const char*)src.get(),
(char*)dst.get(),
inputSize,
compressionLevel - 1);
break;
}
assert(compressedBytes >= 0);
if (compressedBytes <= 0)
throw std::runtime_error(std::string("LZ4_compress[HC] failed, returned value: ") +
SysTools::ToString(compressedBytes));
return compressedBytes;
}
JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_compressBytesDirect
(JNIEnv *env, jobject thisj){
const char* uncompressed_bytes;
char *compressed_bytes;
// Get members of Lz4Compressor
jobject clazz = (*env)->GetStaticObjectField(env, thisj, Lz4Compressor_clazz);
jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_uncompressedDirectBuf);
jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_uncompressedDirectBufLen);
jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_compressedDirectBuf);
jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, Lz4Compressor_directBufferSize);
// Get the input direct buffer
LOCK_CLASS(env, clazz, "Lz4Compressor");
uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
UNLOCK_CLASS(env, clazz, "Lz4Compressor");
if (uncompressed_bytes == 0) {
return (jint)0;
}
// Get the output direct buffer
LOCK_CLASS(env, clazz, "Lz4Compressor");
compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
UNLOCK_CLASS(env, clazz, "Lz4Compressor");
if (compressed_bytes == 0) {
return (jint)0;
}
compressed_direct_buf_len = LZ4_compress(uncompressed_bytes, compressed_bytes, uncompressed_direct_buf_len);
if (compressed_direct_buf_len < 0){
THROW(env, "java/lang/InternalError", "LZ4_compress failed");
}
(*env)->SetIntField(env, thisj, Lz4Compressor_uncompressedDirectBufLen, 0);
return (jint)compressed_direct_buf_len;
}
void LZ4Writer::compressAndWriteBuffer() {
//First 8 bytes is LZOBlock. Skip them
//Then there is a token which has encoded in the 0xF0 bits
//the type of compression.
compressedBuffer[8] = 32;
//Then there is the compressed size but I will write it later...
//... and finally the uncompressed size
Utils::encode_intLE(compressedBuffer, 13, uncompressedBufferLen);
int compressedSize = LZ4_compress(uncompressedBuffer, compressedBuffer + 21,
uncompressedBufferLen);
Utils::encode_intLE(compressedBuffer, 9, compressedSize);
os.write(compressedBuffer, compressedSize + 21);
if (!os.good()) {
BOOST_LOG_TRIVIAL(error)<< "Problems with writing the last file";
}
uncompressedBufferLen = 0;
}
/* Bitshuffle and compress a single block. */
int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size) {
int64_t nbytes, count;
void* tmp_buf_bshuf = malloc(size * elem_size);
if (tmp_buf_bshuf == NULL) return -1;
void* tmp_buf_lz4 = malloc(LZ4_compressBound(size * elem_size));
if (tmp_buf_lz4 == NULL){
free(tmp_buf_bshuf);
return -1;
}
size_t this_iter;
void *in = ioc_get_in(C_ptr, &this_iter);
ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size));
count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size);
if (count < 0) {
free(tmp_buf_lz4);
free(tmp_buf_bshuf);
return count;
}
nbytes = LZ4_compress((const char*) tmp_buf_bshuf, (char*) tmp_buf_lz4, size * elem_size);
free(tmp_buf_bshuf);
CHECK_ERR_FREE_LZ(nbytes, tmp_buf_lz4);
void *out = ioc_get_out(C_ptr, &this_iter);
ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4));
bshuf_write_uint32_BE(out, nbytes);
memcpy((char *) out + 4, tmp_buf_lz4, nbytes);
free(tmp_buf_lz4);
return nbytes + 4;
}
compressed_data compress(const T& in)
{
size_t byte_size = in.size() * sizeof(typename T::value_type);
if (byte_size > 0xffff)
throw std::runtime_error("too much data for compression");
compressed_data out;
if (byte_size > 0) {
auto in_ptr = reinterpret_cast<const char*>(&*in.begin());
out.resize(byte_size + 16);
std::fill(out.begin(), out.end(), 0);
out.unpacked_len = byte_size;
int compressed_length = LZ4_compress(in_ptr, out.ptr(), byte_size);
if (compressed_length <= 0)
throw std::runtime_error("lz4 compression failed");
out.resize(compressed_length);
}
return out;
}
aos_status_t *log_post_logs_with_sts_token(aos_pool_t *p, const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken, const char *project, const char *logstore, cJSON *root)
{
aos_string_t project_name, logstore_name;
aos_table_t *headers = NULL;
aos_table_t *resp_headers = NULL;
log_request_options_t *options = NULL;
aos_list_t buffer;
unsigned char *md5 = NULL;
char *buf = NULL;
int64_t buf_len;
char *b64_value = NULL;
aos_buf_t *content = NULL;
aos_status_t *s = NULL;
options = log_request_options_create(p);
options->config = log_config_create(options->pool);
aos_str_set(&(options->config->endpoint), endpoint);
aos_str_set(&(options->config->access_key_id), accesskeyId);
aos_str_set(&(options->config->access_key_secret), accessKey);
if(stsToken != NULL)
{
aos_str_set(&(options->config->sts_token), stsToken);
}
options->ctl = aos_http_controller_create(options->pool, 0);
headers = aos_table_make(p, 5);
apr_table_set(headers, LOG_API_VERSION, "0.6.0");
apr_table_set(headers, LOG_COMPRESS_TYPE, "lz4");
apr_table_set(headers, LOG_SIGNATURE_METHOD, "hmac-sha1");
apr_table_set(headers, LOG_CONTENT_TYPE, "application/json");
aos_str_set(&project_name, project);
aos_str_set(&logstore_name, logstore);
aos_list_init(&buffer);
char *body = cJSON_PrintUnformatted(root);
if(body == NULL)
{
s = aos_status_create(options->pool);
aos_status_set(s, 400, AOS_CLIENT_ERROR_CODE, "fail to format cJSON data");
return s;
}
int org_body_size = strlen(body);
apr_table_set(headers, LOG_BODY_RAW_SIZE, apr_itoa(options->pool, org_body_size));
int compress_bound = LZ4_compressBound(org_body_size);
char *compress_data = aos_pcalloc(options->pool, compress_bound);
int compressed_size = LZ4_compress(body, compress_data, org_body_size);
if(compressed_size <= 0)
{
s = aos_status_create(options->pool);
aos_status_set(s, 400, AOS_CLIENT_ERROR_CODE, "fail to compress json data");
return s;
}
content = aos_buf_pack(options->pool, compress_data, compressed_size);
aos_list_add_tail(&content->node, &buffer);
//add Content-MD5
buf_len = aos_buf_list_len(&buffer);
buf = aos_buf_list_content(options->pool, &buffer);
md5 = aos_md5(options->pool, buf, (apr_size_t)buf_len);
b64_value = aos_pcalloc(options->pool, 50);
int loop = 0;
for(; loop < 16; ++loop)
{
unsigned char a = ((*md5)>>4) & 0xF, b = (*md5) & 0xF;
b64_value[loop<<1] = a > 9 ? (a - 10 + 'A') : (a + '0');
b64_value[(loop<<1)|1] = b > 9 ? (b - 10 + 'A') : (b + '0');
++md5;
}
b64_value[loop<<1] = '\0';
apr_table_set(headers, LOG_CONTENT_MD5, b64_value);
s = log_post_logs_from_buffer(options, &project_name, &logstore_name,
&buffer, headers, &resp_headers);
free(body);
return s;
}
void save_table(ubyte *table, char color)
{
int i;
FILE *F;
char name[64];
ubyte v[256];
if (!lz4_buf) {
lz4_buf = malloc(4 + LZ4_compressBound(COPYSIZE));
if (!lz4_buf) {
printf("Out of memory.\n");
exit(0);
}
}
sprintf(name, "%s.%c.%d", tablename, color, num_saves);
if (!(F = fopen(name, "wb"))) {
printf("Could not open %s for writing.\n", name);
exit(-1);
}
for (i = 0; i < 256; i++)
v[i] = 0;
#ifndef SUICIDE
if (num_saves == 0) {
v[MATE] = 255;
for (i = 0; i < DRAW_RULE; i++) {
v[WIN_IN_ONE + i] = 1 + i;
v[LOSS_IN_ONE - i] = 254 - i;
}
for (; i < REDUCE_PLY - 2; i++) {
v[WIN_IN_ONE + i + 1] = 1 + DRAW_RULE + (i - DRAW_RULE) / 2;
v[LOSS_IN_ONE - i] = 254 - DRAW_RULE - (i - DRAW_RULE) / 2;
}
v[LOSS_IN_ONE - i] = 254 - DRAW_RULE - (i - DRAW_RULE) / 2;
} else {
for (i = 0; i < REDUCE_PLY_RED; i++) {
v[CAPT_CWIN_RED + i + 2] = 1 + ((reduce_cnt & 1) + i) / 2;
v[LOSS_IN_ONE - i - 1] = 255 - ((reduce_cnt & 1) + i + 1) / 2;
}
}
#else
if (num_saves == 0) {
for (i = 3; i <= DRAW_RULE; i++)
v[BASE_WIN + i] = 1 + (i - 3);
v[BASE_WIN + DRAW_RULE + 1] = DRAW_RULE - 1;
// should we save THREAT_CWIN1/2 (separately) ?
for (i = DRAW_RULE + 2; i < REDUCE_PLY; i++)
v[BASE_WIN + i + 2] = DRAW_RULE - 1 + (i - DRAW_RULE - 1) / 2;
for (i = 2; i <= DRAW_RULE; i++)
v[BASE_LOSS - i] = 255 - (i - 2);
for (; i < REDUCE_PLY; i++)
v[BASE_LOSS - i] = 255 - (DRAW_RULE - 2) - 1 - (i - DRAW_RULE - 1) / 2;
} else {
for (i = 0; i < REDUCE_PLY_RED; i++) {
v[BASE_WIN + i + 6] = 1 + ((reduce_cnt & 1) + i) / 2;
v[BASE_LOSS - i - 4] = 255 - ((reduce_cnt & 1) + i) / 2;
}
}
#endif
ubyte *ptr = table;
long64 total = size;
while (total > 0) {
int chunk = COPYSIZE;
if (total < chunk) chunk = total;
total -= chunk;
for (i = 0; i < chunk; i++)
copybuf[i] = v[ptr[i]];
ptr += chunk;
uint32 lz4_size = LZ4_compress((char *)copybuf, lz4_buf + 4, chunk);
*(uint32 *)lz4_buf = lz4_size;
fwrite(lz4_buf, 1, lz4_size + 4, F);
}
fclose(F);
}
int compress_file(char* input_filename, char* output_filename)
{
U64 filesize = 0;
U64 compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE;
char* in_buff;
char* out_buff;
char stdinout[] = "std";
FILE* finput;
if (!strcmp (input_filename, stdinout)) {
fprintf(stderr, "Using stdin for input\n");
finput = stdin;
} else {
finput = fopen( input_filename, "rb" );
}
FILE* foutput;
if (!strcmp (output_filename, stdinout)) {
fprintf(stderr, "Using stdout for output\n");
foutput = stdout;
} else {
foutput = fopen( output_filename, "wb" );
}
if ( finput==0 ) { printf("Pb opening %s\n", input_filename); return 2; }
if ( foutput==0) { printf("Pb opening %s\n", output_filename); return 3; }
// Allocate Memory
in_buff = malloc(CHUNKSIZE);
out_buff = malloc(OUT_CHUNKSIZE);
// Write Archive Header
*(U32*)out_buff = ARCHIVE_MAGICNUMBER;
fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput);
// Main Loop
while (1)
{
int outSize;
// Read Block
int inSize = fread(in_buff, 1, CHUNKSIZE, finput);
if( inSize<=0 ) break;
filesize += inSize;
// Compress Block
outSize = LZ4_compress(in_buff, out_buff+4, inSize);
* (U32*) out_buff = outSize;
compressedfilesize += outSize+4;
// Write Block
fwrite(out_buff, 1, outSize+4, foutput);
}
// Status
fprintf(stderr, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
(unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
fclose(finput);
fclose(foutput);
return 0;
}
int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
unsigned long long bytes = 0;
unsigned long long cbytes = 0;
unsigned long long hcbytes = 0;
unsigned long long ccbytes = 0;
void* CNBuffer;
char* compressedBuffer;
char* decodedBuffer;
# define FUZ_max LZ4_COMPRESSBOUND(LEN)
unsigned int randState=seed;
int ret, cycleNb;
# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \
printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; }
# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); }
void* stateLZ4 = malloc(LZ4_sizeofState());
void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
void* LZ4continue;
LZ4_stream_t LZ4dict;
U32 crcOrig, crcCheck;
int displayRefresh;
// init
memset(&LZ4dict, 0, sizeof(LZ4dict));
// Create compressible test buffer
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE));
decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
// display refresh rate
switch(displayLevel)
{
case 0: displayRefresh = nbCycles+1; break;
case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break;
case 2: displayRefresh=89; break;
default : displayRefresh=1;
}
// move to startCycle
for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
{
// synd rand & dict
int dictSize, blockSize, blockStart;
char* dict;
char* block;
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
LZ4_loadDict(&LZ4dict, dict, dictSize);
LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
}
// Test loop
for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++)
{
int testNb = 0;
char* dict;
char* block;
int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize;
int blockContinueCompressedSize;
if ((cycleNb%displayRefresh) == 0)
{
printf("\r%7i /%7i - ", cycleNb, nbCycles);
fflush(stdout);
}
// Select block to test
blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE;
blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
if (dictSize > blockStart) dictSize = blockStart;
block = ((char*)CNBuffer) + blockStart;
dict = block - dictSize;
/* Compression tests */
// Test compression HC
FUZ_DISPLAYTEST;
ret = LZ4_compressHC(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
// Test compression HC using external state
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
// Test compression using external state
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
// Test compression
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
compressedSize = ret;
/* Decompression tests */
crcOrig = XXH32(block, blockSize, 0);
// Test decoding with output size being exactly what's necessary => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
// Test decoding with one byte missing => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
// Test decoding with one byte too much => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
// Test decoding with output size exactly what's necessary => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with more than enough output size => must work
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
decodedBuffer[blockSize+1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
//FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
// Test decoding with output size being one byte too short => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
// Test decoding with output size being 10 bytes too short => must fail
FUZ_DISPLAYTEST;
if (blockSize>10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
}
// Test decoding with input size being one byte too short => must fail
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
// Test decoding with input size being one byte too large => must fail
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
// Test partial decoding with target output size being max/2 => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
// Test partial decoding with target output size being just below max => must work
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
/* Test Compression with limited output size */
// Test compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
// Test compression with output size being exactly what's necessary and external state (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compress_limitedOutput_withState(stateLZ4, block, compressedBuffer, blockSize, compressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
// Test HC compression with output size being exactly what's necessary (should work)
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
// Test compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compress_limitedOutput(block, compressedBuffer, blockSize, compressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compress_limitedOutput overran output buffer")
// Test HC compression with just one missing byte into output buffer => must fail
FUZ_DISPLAYTEST;
compressedBuffer[compressedSize-1] = 0;
ret = LZ4_compressHC_limitedOutput(block, compressedBuffer, blockSize, HCcompressedSize-1);
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by 1 byte)");
FUZ_CHECKTEST(compressedBuffer[compressedSize-1], "LZ4_compressHC_limitedOutput overran output buffer")
/* Dictionary tests */
// Compress using dictionary
FUZ_DISPLAYTEST;
LZ4continue = LZ4_create (dict);
LZ4_compress_continue (LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables
blockContinueCompressedSize = LZ4_compress_continue (LZ4continue, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
LZ4_free (LZ4continue);
// Decompress with dictionary as prefix
FUZ_DISPLAYTEST;
memcpy(decodedBuffer, dict, dictSize);
ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
// Compress using External dictionary
FUZ_DISPLAYTEST;
dict -= 9; // Separation, so it is an ExtDict
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_loadDict(&LZ4dict, dict, dictSize);
blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
FUZ_DISPLAYTEST;
LZ4_loadDict(&LZ4dict, dict, dictSize);
ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
// Decompress with dictionary as external
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
{
int i=0;
while (block[i]==decodedBuffer[i]) i++;
printf("Wrong Byte at position %i/%i\n", i, blockSize);
}
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
if (blockSize > 10)
{
decodedBuffer[blockSize-10] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-10, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-10 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-10 byte) (blockSize=%i)", blockSize);
}
// Fill stats
bytes += blockSize;
cbytes += compressedSize;
hcbytes += HCcompressedSize;
ccbytes += blockContinueCompressedSize;
}
printf("\r%7i /%7i - ", cycleNb, nbCycles);
printf("all tests completed successfully \n");
printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
// unalloc
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 0;
_output_error:
if(!no_prompt) getchar();
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
free(stateLZ4);
free(stateLZ4HC);
return 1;
}
