int lz4_start(int s) { int err; /* Check whether underlying socket is message-based. */ if(dsock_slow(!hquery(s, msock_type))) {err = errno; goto error1;} /* Create the object. */ struct lz4_sock *obj = malloc(sizeof(struct lz4_sock)); if(dsock_slow(!obj)) {errno = ENOMEM; goto error1;} obj->hvfs.query = lz4_hquery; obj->hvfs.close = lz4_hclose; obj->mvfs.msendv = lz4_msendv; obj->mvfs.mrecvv = lz4_mrecvv; obj->s = s; obj->outbuf = NULL; obj->outlen = 0; obj->inbuf = NULL; obj->inlen = 0; size_t ec = LZ4F_createDecompressionContext(&obj->dctx, LZ4F_VERSION); if(dsock_slow(LZ4F_isError(ec))) {err = EFAULT; goto error2;} /* Create the handle. */ int h = hcreate(&obj->hvfs); if(dsock_slow(h < 0)) {err = errno; goto error3;} return h; error3: ec = LZ4F_freeDecompressionContext(obj->dctx); dsock_assert(!LZ4F_isError(ec)); error2: free(obj); error1: errno = err; return -1; }
static SquashStatus squash_lz4f_decompress_stream (SquashStream* stream, SquashOperation operation) { SquashLZ4FStream* s = (SquashLZ4FStream*) stream; while (stream->avail_in != 0 && stream->avail_out != 0) { size_t dst_len = stream->avail_out; size_t src_len = stream->avail_in; size_t bytes_read = LZ4F_decompress (s->data.decomp.ctx, stream->next_out, &dst_len, stream->next_in, &src_len, NULL); if (LZ4F_isError (bytes_read)) { return squash_lz4f_get_status (bytes_read); } if (src_len != 0) { stream->next_in += src_len; stream->avail_in -= src_len; } if (dst_len != 0) { stream->next_out += dst_len; stream->avail_out -= dst_len; } } return (stream->avail_in == 0) ? SQUASH_OK : SQUASH_PROCESSING; }
void LZ4RunReader::decompress() { // Number of compressed bytes available size_t comp_len = comp_.fill(); // Space available for decompression size_t decomp_len = decomp_.size() - decomp_.fill(); // Attempt the decompression size_t n = LZ4F_decompress(lz4_, decomp_.base() + decomp_.hi(), &decomp_len, comp_.base() + comp_.lo(), &comp_len, NULL); if( LZ4F_isError(n) ) { throw std::runtime_error("Error during LZ4 decompression."); } // Update ring buffer indicators comp_.advance_lo(comp_len); decomp_.advance_hi(decomp_len); return; }
static int lz4_msendv(struct msock_vfs *mvfs, const struct iovec *iov, size_t iovlen, int64_t deadline) { struct lz4_sock *obj = dsock_cont(mvfs, struct lz4_sock, mvfs); /* Adjust the buffer size as needed. */ size_t len = iov_size(iov, iovlen); size_t maxlen = LZ4F_compressFrameBound(len, NULL); if(obj->outlen < maxlen) { uint8_t *newbuf = realloc(obj->outbuf, maxlen); if(dsock_slow(!newbuf)) {errno = ENOMEM; return -1;} obj->outbuf = newbuf; obj->outlen = maxlen; } /* Compress the data. */ /* TODO: Avoid the extra allocations and copies. */ uint8_t *buf = malloc(len); if(dsock_slow(!buf)) {errno = ENOMEM; return -1;} iov_copyallfrom(buf, iov, iovlen); LZ4F_preferences_t prefs = {0}; prefs.frameInfo.contentSize = len; size_t dstlen = LZ4F_compressFrame(obj->outbuf, obj->outlen, buf, len, &prefs); dsock_assert(!LZ4F_isError(dstlen)); dsock_assert(dstlen <= obj->outlen); free(buf); /* Send the compressed frame. */ return msend(obj->s, obj->outbuf, dstlen, deadline); }
static bool writeFile(llvm::StringRef file_name, llvm::StringRef data) { std::error_code error_code; llvm::raw_fd_ostream file(file_name, error_code, llvm::sys::fs::F_RW); if (error_code) return false; int uncompressed_size = data.size(); // Write the uncompressed size to the beginning of the file as a simple checksum. // It looks like each lz4 block has its own data checksum, but we need to also // make sure that we have all the blocks that we expected. // In particular, without this, an empty file seems to be a valid lz4 stream. file.write(reinterpret_cast<const char*>(&uncompressed_size), 4); LZ4F_preferences_t preferences; memset(&preferences, 0, sizeof(preferences)); preferences.frameInfo.contentChecksumFlag = contentChecksumEnabled; preferences.frameInfo.contentSize = data.size(); std::vector<char> compressed; size_t max_size = LZ4F_compressFrameBound(data.size(), &preferences); compressed.resize(max_size); size_t compressed_size = LZ4F_compressFrame(&compressed[0], max_size, data.data(), data.size(), &preferences); if (LZ4F_isError(compressed_size)) return false; file.write(compressed.data(), compressed_size); return true; }
/* lz4_compress support function * @param src constant input memory buffer of size size to be compressed * @param srz_size input memory buffer size * @param dest ouput memory buffer of size size in which to place results * @param dest_size pointer to a variable that contains the size of dest on input * and is updated to contain the size of compressed data in dest on out * @param level compression level from 1 (low/fast) to 9 (high/slow) * Returns an error code, 0 for success non-zero otherwise. */ int lz4_compress (void *src, size_t src_size, void *dest, size_t * dest_size, int level) { size_t n, len = 0; LZ4F_errorCode_t err; LZ4F_preferences_t prefs; LZ4F_compressionContext_t ctx; int retval = 0; /* Set compression parameters */ memset (&prefs, 0, sizeof (prefs)); prefs.autoFlush = 1; prefs.compressionLevel = level; /* 0...16 */ prefs.frameInfo.blockMode = 0; /* blockLinked, blockIndependent ; 0 == default */ prefs.frameInfo.blockSizeID = 0; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ prefs.frameInfo.contentChecksumFlag = 1; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ prefs.frameInfo.contentSize = (long long)src_size; /* for reference */ /* create context */ err = LZ4F_createCompressionContext (&ctx, LZ4F_VERSION); if (LZ4F_isError (err)) return -1; n = LZ4F_compressFrame (dest, *dest_size, src, src_size, &prefs); if (LZ4F_isError (n)) { retval = -2; goto cleanup; } /* update the compressed buffer size */ *dest_size = n; cleanup: if (ctx) { err = LZ4F_freeCompressionContext (ctx); if (LZ4F_isError (err)) return -5; } return retval; }
static void lz4_hclose(struct hvfs *hvfs) { struct lz4_sock *obj = (struct lz4_sock*)hvfs; size_t ec = LZ4F_freeDecompressionContext(obj->dctx); dsock_assert(!LZ4F_isError(ec)); free(obj->inbuf); free(obj->outbuf); int rc = hclose(obj->s); dsock_assert(rc == 0); free(obj); }
int lz4_stop(int s) { struct lz4_sock *obj = hquery(s, lz4_type); if(dsock_slow(!obj)) return -1; size_t ec = LZ4F_freeDecompressionContext(obj->dctx); dsock_assert(!LZ4F_isError(ec)); free(obj->inbuf); free(obj->outbuf); int u = obj->s; free(obj); return u; }
static ssize_t lz4_mrecvv(struct msock_vfs *mvfs, const struct iovec *iov, size_t iovlen, int64_t deadline) { struct lz4_sock *obj = dsock_cont(mvfs, struct lz4_sock, mvfs); /* Adjust the buffer size as needed. */ size_t len = iov_size(iov, iovlen); size_t maxlen = LZ4F_compressFrameBound(len, NULL); if(obj->inlen < maxlen) { uint8_t *newbuf = realloc(obj->inbuf, maxlen); if(dsock_slow(!newbuf)) {errno = ENOMEM; return -1;} obj->inbuf = newbuf; obj->inlen = maxlen; } /* Get the compressed message. */ ssize_t sz = mrecv(obj->s, obj->inbuf, obj->inlen, deadline); if(dsock_slow(sz < 0)) return -1; /* Extract size of the uncompressed message from LZ4 frame header. */ LZ4F_frameInfo_t info; size_t infolen = sz; size_t ec = LZ4F_getFrameInfo(obj->dctx, &info, obj->inbuf, &infolen); if(dsock_slow(LZ4F_isError(ec))) {errno = EPROTO; return -1;} /* Size is a required field. */ if(dsock_slow(info.contentSize == 0)) {errno = EPROTO; return -1;} /* Decompressed message would exceed the buffer size. */ if(dsock_slow(info.contentSize > len)) {errno = EMSGSIZE; return -1;} /* Decompress. */ /* TODO: Avoid the extra allocations and copies. */ uint8_t *buf = malloc(len); if(dsock_slow(!buf)) {errno = ENOMEM; return -1;} size_t dstlen = len; size_t srclen = sz - infolen; ec = LZ4F_decompress(obj->dctx, buf, &dstlen, obj->inbuf + infolen, &srclen, NULL); if(dsock_slow(LZ4F_isError(ec))) {errno = EPROTO; return -1;} if(dsock_slow(ec != 0)) {errno = EPROTO; return -1;} dsock_assert(srclen == sz - infolen); iov_copyallto(iov, iovlen, buf); free(buf); return dstlen; }
static std::unique_ptr<llvm::MemoryBuffer> getFile(llvm::StringRef file_name) { auto compressed_content = llvm::MemoryBuffer::getFile(file_name, -1, false); if (!compressed_content) return std::unique_ptr<llvm::MemoryBuffer>(); LZ4F_decompressionContext_t context; LZ4F_createDecompressionContext(&context, LZ4F_VERSION); LZ4F_frameInfo_t frame_info; memset(&frame_info, 0, sizeof(frame_info)); const char* start = (*compressed_content)->getBufferStart(); size_t pos = 0; size_t compressed_size = (*compressed_content)->getBufferSize(); if (compressed_size < 4) return std::unique_ptr<llvm::MemoryBuffer>(); int orig_uncompressed_size = *reinterpret_cast<const int*>(start); pos += 4; size_t remaining = compressed_size - pos; LZ4F_getFrameInfo(context, &frame_info, start + pos, &remaining); pos += remaining; std::vector<char> uncompressed; uncompressed.reserve(frame_info.contentSize); while (pos < compressed_size) { unsigned char buff[4096]; size_t buff_size = sizeof(buff); remaining = compressed_size - pos; size_t error_code = LZ4F_decompress(context, buff, &buff_size, start + pos, &remaining, NULL); if (LZ4F_isError(error_code)) { LZ4F_freeDecompressionContext(context); return std::unique_ptr<llvm::MemoryBuffer>(); } pos += remaining; if (buff_size != 0) uncompressed.insert(uncompressed.end(), buff, buff + buff_size); } LZ4F_freeDecompressionContext(context); if (uncompressed.size() != frame_info.contentSize) return std::unique_ptr<llvm::MemoryBuffer>(); if (uncompressed.size() != orig_uncompressed_size) return std::unique_ptr<llvm::MemoryBuffer>(); return llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(uncompressed.data(), uncompressed.size())); }
/* Decompression methods */ static PyObject *py_lz4f_createDecompCtx(PyObject *self, PyObject *args) { PyObject *result; LZ4F_decompressionContext_t dCtx; size_t err; (void)self; (void)args; err = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(err), "Allocation failed (error %i)", (int)err); result = PyCapsule_New(dCtx, NULL, NULL); return result; _output_error: return Py_None; }
static SquashLZ4FStream* squash_lz4f_stream_new (SquashCodec* codec, SquashStreamType stream_type, SquashOptions* options) { SquashLZ4FStream* stream; LZ4F_errorCode_t ec; assert (codec != NULL); stream = (SquashLZ4FStream*) squash_malloc (sizeof (SquashLZ4FStream)); if (SQUASH_UNLIKELY(stream == NULL)) return (squash_error (SQUASH_MEMORY), NULL); squash_lz4f_stream_init (stream, codec, stream_type, options, squash_lz4f_stream_destroy); if (stream_type == SQUASH_STREAM_COMPRESS) { ec = LZ4F_createCompressionContext(&(stream->data.comp.ctx), LZ4F_VERSION); stream->data.comp.state = SQUASH_LZ4F_STATE_INIT; stream->data.comp.output_buffer = NULL; stream->data.comp.output_buffer_pos = 0; stream->data.comp.output_buffer_size = 0; stream->data.comp.input_buffer_size = 0; stream->data.comp.prefs = (LZ4F_preferences_t) { { (LZ4F_blockSizeID_t) squash_codec_get_option_int_index (codec, options, SQUASH_LZ4F_OPT_BLOCK_SIZE), blockLinked, squash_codec_get_option_bool_index (codec, options, SQUASH_LZ4F_OPT_CHECKSUM) ? contentChecksumEnabled : noContentChecksum, }, squash_codec_get_option_int_index (codec, options, SQUASH_LZ4F_OPT_LEVEL) }; } else { ec = LZ4F_createDecompressionContext(&(stream->data.decomp.ctx), LZ4F_VERSION); } if (SQUASH_UNLIKELY(LZ4F_isError (ec))) { squash_object_unref (stream); return (squash_error (SQUASH_FAILED), NULL); } return stream; }
static void squash_lz4f_stream_destroy (void* stream) { SquashLZ4FStream* s = (SquashLZ4FStream*) stream; LZ4F_errorCode_t ec; if (((SquashStream*) stream)->stream_type == SQUASH_STREAM_COMPRESS) { ec = LZ4F_freeCompressionContext(s->data.comp.ctx); if (s->data.comp.output_buffer != NULL) free (s->data.comp.output_buffer); } else { ec = LZ4F_freeDecompressionContext(s->data.decomp.ctx); } assert (!LZ4F_isError (ec)); squash_stream_destroy (stream); }
static PyObject *py_lz4f_decompress(PyObject *self, PyObject *args, PyObject *keywds) { const char* source; char* dest; LZ4F_decompressionContext_t dCtx; int src_size; PyObject *decomp; PyObject *next; PyObject *py_dCtx; PyObject *result = PyDict_New(); size_t ssrc_size; size_t dest_size; size_t err; static char *kwlist[] = {"source", "dCtx", "blkSizeID"}; unsigned int blkID=7; (void)self; if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#O|i", kwlist, &source, &src_size, &py_dCtx, &blkID)) { return NULL; } dest_size = LZ4S_GetBlockSize_FromBlockId(blkID); dCtx = (LZ4F_decompressionContext_t)PyCapsule_GetPointer(py_dCtx, NULL); ssrc_size = (size_t)src_size; dest = (char*)malloc(dest_size); err = LZ4F_decompress(dCtx, dest, &dest_size, source, &ssrc_size, NULL); CHECK(LZ4F_isError(err), "Failed getting frameInfo. (error %i)", (int)err); //fprintf(stdout, "Dest_size: %zu Error Code:%zu \n", dest_size, err); decomp = PyBytes_FromStringAndSize(dest, dest_size); next = PyInt_FromSize_t(err); PyDict_SetItemString(result, "decomp", decomp); PyDict_SetItemString(result, "next", next); Py_XDECREF(decomp); Py_XDECREF(next); free(dest); return result; _output_error: return Py_None; }
static SquashStatus squash_lz4f_get_status (size_t res) { if (!LZ4F_isError (res)) return SQUASH_OK; switch ((LZ4F_errorCodes) (-(int)(res))) { case LZ4F_OK_NoError: return SQUASH_OK; case LZ4F_ERROR_GENERIC: return squash_error (SQUASH_FAILED); case LZ4F_ERROR_maxBlockSize_invalid: case LZ4F_ERROR_blockMode_invalid: case LZ4F_ERROR_contentChecksumFlag_invalid: case LZ4F_ERROR_headerVersion_wrong: case LZ4F_ERROR_blockChecksum_unsupported: case LZ4F_ERROR_reservedFlag_set: case LZ4F_ERROR_frameHeader_incomplete: case LZ4F_ERROR_frameType_unknown: case LZ4F_ERROR_frameSize_wrong: case LZ4F_ERROR_headerChecksum_invalid: case LZ4F_ERROR_contentChecksum_invalid: return squash_error (SQUASH_INVALID_BUFFER); case LZ4F_ERROR_compressionLevel_invalid: return squash_error (SQUASH_BAD_VALUE); case LZ4F_ERROR_allocation_failed: return squash_error (SQUASH_MEMORY); case LZ4F_ERROR_srcSize_tooLarge: case LZ4F_ERROR_dstMaxSize_tooSmall: return squash_error (SQUASH_BUFFER_FULL); case LZ4F_ERROR_decompressionFailed: case LZ4F_ERROR_srcPtr_wrong: case LZ4F_ERROR_maxCode: return squash_error (SQUASH_FAILED); default: squash_assert_unreachable (); } }
LZ4RunReader::LZ4RunReader(const std::string& run_file, const size_t buffer_size, const double trigger_fraction) : comp_(buffer_size), decomp_(buffer_size), eof_(false), trigger_(std::max(trigger_fraction, 1.0) * buffer_size) { // Check that the trigger size leaves us at least space to extract // a length from the buffer if( trigger_ < sizeof(size_t) ) { throw std::runtime_error("Buffer trigger size too small"); } // Open the input file int fd = open(run_file.c_str(), O_RDONLY); if( fd == -1 ) { throw std::runtime_error("Error opening run file " + run_file + " : " + strerror(errno)); } // Set up poll() structure fds_[0].fd = fd; fds_[0].events = POLLIN; // Set fd as nonblocking fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); // Create LZ4 decompression context LZ4F_errorCode_t r = LZ4F_createDecompressionContext(&lz4_, LZ4F_VERSION); if (LZ4F_isError(r)) { throw std::runtime_error("Error creating LZ4 compression context."); } }
static PyObject *py_lz4f_getFrameInfo(PyObject *self, PyObject *args) { const char *source; int src_size; LZ4F_decompressionContext_t dCtx; LZ4F_frameInfo_t frameInfo; PyObject *blkSize; PyObject *blkMode; PyObject *contChkFlag; PyObject *py_dCtx; PyObject *result = PyDict_New(); size_t ssrc_size; size_t err; (void)self; if (!PyArg_ParseTuple(args, "s#O", &source, &src_size, &py_dCtx)) { return NULL; } dCtx = (LZ4F_decompressionContext_t)PyCapsule_GetPointer(py_dCtx, NULL); ssrc_size = (size_t)src_size; err = LZ4F_getFrameInfo(dCtx, &frameInfo, (unsigned char*)source, &ssrc_size); CHECK(LZ4F_isError(err), "Failed getting frameInfo. (error %i)", (int)err); blkSize = PyInt_FromSize_t(frameInfo.blockSizeID); blkMode = PyInt_FromSize_t(frameInfo.blockMode); contChkFlag = PyInt_FromSize_t(frameInfo.contentChecksumFlag); PyDict_SetItemString(result, "blkSize", blkSize); PyDict_SetItemString(result, "blkMode", blkMode); PyDict_SetItemString(result, "chkFlag", contChkFlag); return result; _output_error: return Py_None; }
static void squash_lz4f_stream_init (SquashLZ4FStream* stream, SquashCodec* codec, SquashStreamType stream_type, SquashOptions* options, SquashDestroyNotify destroy_notify) { LZ4F_errorCode_t ec; squash_stream_init ((SquashStream*) stream, codec, stream_type, (SquashOptions*) options, destroy_notify); if (stream_type == SQUASH_STREAM_COMPRESS) { ec = LZ4F_createCompressionContext(&(stream->data.comp.ctx), LZ4F_VERSION); stream->data.comp.state = SQUASH_LZ4F_STATE_INIT; stream->data.comp.output_buffer = NULL; stream->data.comp.output_buffer_pos = 0; stream->data.comp.output_buffer_length = 0; stream->data.comp.input_buffer_length = 0; stream->data.comp.prefs = (LZ4F_preferences_t) { { squash_codec_get_option_int_index (codec, options, SQUASH_LZ4F_OPT_BLOCK_SIZE), blockLinked, squash_codec_get_option_bool_index (codec, options, SQUASH_LZ4F_OPT_CHECKSUM) ? contentChecksumEnabled : noContentChecksum, }, squash_codec_get_option_int_index (codec, options, SQUASH_LZ4F_OPT_LEVEL) }; } else { ec = LZ4F_createDecompressionContext(&(stream->data.decomp.ctx), LZ4F_VERSION); } assert (!LZ4F_isError (ec)); }
void Message::compress(Message::Compression type, int level) { if (isCompressed()) return; switch (type) { #ifdef BUILD_WITH_COMPRESSION_MINIZ case COMPRESS_MINIZ: { mz_ulong compressedSize = mz_compressBound(_size); int cmp_status; uint8_t *pCmp; pCmp = (mz_uint8 *)malloc((size_t)compressedSize); level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_MINIZ); // last argument is speed size tradeoff: BEST_SPEED [0-9] BEST_COMPRESSION cmp_status = mz_compress2(pCmp, &compressedSize, (const unsigned char *)_data.get(), _size, level); if (cmp_status != Z_OK) { // error free(pCmp); } _data = SharedPtr<char>((char*)pCmp); _meta["um.compressed"] = toStr(_size) + ":miniz"; _size = compressedSize; } return; #endif #ifdef BUILD_WITH_COMPRESSION_FASTLZ case COMPRESS_FASTLZ: { // The minimum input buffer size is 16. if (_size < 16) return; // The output buffer must be at least 5% larger than the input buffer and can not be smaller than 66 bytes. int compressedSize = _size + (double)_size * 0.06; if (compressedSize < 66) compressedSize = 66; char* compressedData = (char*)malloc(compressedSize); compressedSize = fastlz_compress(_data.get(), _size, compressedData); // If the input is not compressible, the return value might be larger than length if (compressedSize > _size) { free(compressedData); return; } // std::cout << _size << " -> " << compressedSize << " = " << ((float)compressedSize / (float)_size) << std::endl; _data = SharedPtr<char>((char*)compressedData); _meta["um.compressed"] = toStr(_size) + ":fastlz"; _size = compressedSize; } return; #endif #ifdef BUILD_WITH_COMPRESSION_LZ4 case COMPRESS_LZ4: { level = (level >= 0 ? level : BUILD_WITH_COMPRESSION_LEVEL_LZ4); #ifdef LZ4_FRAME LZ4F_preferences_t lz4_preferences = { { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } }, level, /* compression level */ 0, /* autoflush */ { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ }; LZ4F_errorCode_t r; LZ4F_compressionContext_t ctx; r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(r)) { // printf("Failed to create context: error %zu", r); return; } #define LZ4_HEADER_SIZE 19 #define LZ4_FOOTER_SIZE 4 size_t n, offset = 0; size_t frameSize = LZ4F_compressBound(_size, &lz4_preferences); size_t compressedSize = frameSize + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; char* compressedData = (char*)malloc(compressedSize); n = LZ4F_compressBegin(ctx, compressedData, compressedSize, &lz4_preferences); if (LZ4F_isError(n)) { // printf("Failed to start compression: error %zu", n); LZ4F_freeCompressionContext(ctx); free(compressedData); return; } offset += n; n = LZ4F_compressUpdate(ctx, compressedData + offset, compressedSize - offset, _data.get(), _size, NULL); if (LZ4F_isError(n)) { // printf("Compression failed: error %zu", n); LZ4F_freeCompressionContext(ctx); free(compressedData); return; } offset += n; n = LZ4F_compressEnd(ctx, compressedData + offset, compressedSize - offset, NULL); if (LZ4F_isError(n)) { // printf("Failed to end compression: error %zu", n); LZ4F_freeCompressionContext(ctx); free(compressedData); return; } offset += n; _data = SharedPtr<char>((char*)compressedData); _meta["um.compressed"] = toStr(_size) + ":lz4"; _size = offset; LZ4F_freeCompressionContext(ctx); #else size_t compressedSize = LZ4_compressBound(_size); char* compressedData = (char*)malloc(compressedSize); int actualSize = 0; actualSize = LZ4_compress_fast(_data.get(), compressedData, _size, compressedSize, level); if (actualSize == 0) { free(compressedData); return; } _data = SharedPtr<char>((char*)compressedData); _meta["um.compressed"] = toStr(_size) + ":lz4"; _size = actualSize; #endif } return; #endif default: break; } }
static int decompress_fd_lz4(int fdi, int fdo) { #ifdef HAVE_LZ4 enum { LZ4_DEC_BUF_SIZE = 64*1024u }; LZ4F_decompressionContext_t ctx = NULL; LZ4F_errorCode_t c; char *buf = NULL; char *src = NULL; int r = 0; struct stat fdist; c = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(c)) { log_debug("Failed to initialized LZ4: %s", LZ4F_getErrorName(c)); r = -ENOMEM; goto cleanup; } buf = malloc(LZ4_DEC_BUF_SIZE); if (!buf) { r = -errno; goto cleanup; } if (fstat(fdi, &fdist) < 0) { r = -errno; log_debug("Failed to stat the input fd"); goto cleanup; } src = mmap(NULL, fdist.st_size, PROT_READ, MAP_PRIVATE, fdi, 0); if (!src) { r = -errno; log_debug("Failed to mmap the input fd"); goto cleanup; } off_t total_in = 0; while (fdist.st_size != total_in) { size_t used = fdist.st_size - total_in; size_t produced = LZ4_DEC_BUF_SIZE; c = LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL); if (LZ4F_isError(c)) { log_debug("Failed to decode LZ4 block: %s", LZ4F_getErrorName(c)); r = -EBADMSG; goto cleanup; } r = safe_write(fdo, buf, produced); if (r < 0) { log_debug("Failed to write decoded block"); goto cleanup; } total_in += used; } r = 0; cleanup: if (ctx != NULL) LZ4F_freeDecompressionContext(ctx); if (buf != NULL) free(buf); if (src != NULL) munmap(src, fdist.st_size); return r; #else /*HAVE_LZ4*/ const char *cmd[] = { "lz4", "-cd", "-", NULL}; return decompress_using_fork_execvp(cmd, fdi, fdo); #endif /*HAVE_LZ4*/ }
static SquashStatus squash_lz4f_compress_stream (SquashStream* stream, SquashOperation operation) { SquashLZ4FStream* s = (SquashLZ4FStream*) stream; bool progress = false; if (s->data.comp.output_buffer_size != 0) { const size_t buffer_remaining = s->data.comp.output_buffer_size - s->data.comp.output_buffer_pos; const size_t cp_size = (buffer_remaining < stream->avail_out) ? buffer_remaining : stream->avail_out; memcpy (stream->next_out, s->data.comp.output_buffer + s->data.comp.output_buffer_pos, cp_size); stream->next_out += cp_size; stream->avail_out -= cp_size; s->data.comp.output_buffer_pos += cp_size; if (cp_size == buffer_remaining) { s->data.comp.output_buffer_size = 0; s->data.comp.output_buffer_pos = 0; progress = true; } else { return SQUASH_PROCESSING; } } while ((stream->avail_in != 0 || operation != SQUASH_OPERATION_PROCESS) && stream->avail_out != 0) { if (s->data.comp.state == SQUASH_LZ4F_STATE_INIT) { s->data.comp.state = SQUASH_LZ4F_STATE_ACTIVE; if (stream->avail_out < 19) { s->data.comp.output_buffer_size = LZ4F_compressBegin (s->data.comp.ctx, squash_lz4f_stream_get_output_buffer (stream), squash_lz4f_stream_get_output_buffer_size (stream), &(s->data.comp.prefs)); break; } else { size_t written = LZ4F_compressBegin (s->data.comp.ctx, stream->next_out, stream->avail_out, &(s->data.comp.prefs)); stream->next_out += written; stream->avail_out -= written; progress = true; } } else { const size_t input_buffer_size = squash_lz4f_get_input_buffer_size (stream); const size_t total_input = stream->avail_in + s->data.comp.input_buffer_size; const size_t output_buffer_max_size = squash_lz4f_stream_get_output_buffer_size (stream); if (progress && (total_input < input_buffer_size || stream->avail_out < output_buffer_max_size)) break; uint8_t* obuf; size_t olen; const size_t input_size = (total_input > input_buffer_size) ? (input_buffer_size - s->data.comp.input_buffer_size) : stream->avail_in; if (input_size > 0) { obuf = (output_buffer_max_size > stream->avail_out) ? squash_lz4f_stream_get_output_buffer (stream) : stream->next_out; olen = LZ4F_compressUpdate (s->data.comp.ctx, obuf, output_buffer_max_size, stream->next_in, input_size, NULL); if (!LZ4F_isError (olen)) { if (input_size + s->data.comp.input_buffer_size == input_buffer_size) { s->data.comp.input_buffer_size = 0; } else { s->data.comp.input_buffer_size += input_size; assert (olen == 0); } stream->next_in += input_size; stream->avail_in -= input_size; } else { squash_assert_unreachable(); } } else if (operation == SQUASH_OPERATION_FLUSH) { assert (stream->avail_in == 0); olen = squash_lz4f_stream_get_output_buffer_size (stream); obuf = (olen > stream->avail_out) ? squash_lz4f_stream_get_output_buffer (stream) : stream->next_out; olen = LZ4F_flush (s->data.comp.ctx, obuf, olen, NULL); s->data.comp.input_buffer_size = 0; } else if (operation == SQUASH_OPERATION_FINISH) { assert (stream->avail_in == 0); olen = squash_lz4f_stream_get_output_buffer_size (stream); obuf = (olen > stream->avail_out) ? squash_lz4f_stream_get_output_buffer (stream) : stream->next_out; olen = LZ4F_compressEnd (s->data.comp.ctx, obuf, olen, NULL); s->data.comp.input_buffer_size = 0; } else if (progress) { break; } else { squash_assert_unreachable(); } if (SQUASH_UNLIKELY(LZ4F_isError (olen))) { squash_assert_unreachable(); return squash_error (SQUASH_FAILED); } else { if (olen != 0) { if (obuf == s->data.comp.output_buffer) { s->data.comp.output_buffer_size = olen; break; } else { stream->next_out += olen; stream->avail_out -= olen; } } } if (operation != SQUASH_OPERATION_PROCESS) break; } } if (s->data.comp.output_buffer_size != 0) { const size_t buffer_remaining = s->data.comp.output_buffer_size - s->data.comp.output_buffer_pos; const size_t cp_size = (buffer_remaining < stream->avail_out) ? buffer_remaining : stream->avail_out; memcpy (stream->next_out, s->data.comp.output_buffer + s->data.comp.output_buffer_pos, cp_size); stream->next_out += cp_size; stream->avail_out -= cp_size; s->data.comp.output_buffer_pos += cp_size; if (cp_size == buffer_remaining) { s->data.comp.output_buffer_size = 0; s->data.comp.output_buffer_pos = 0; progress = true; } else { return SQUASH_PROCESSING; } } return (stream->avail_in == 0 && s->data.comp.output_buffer_size == 0) ? SQUASH_OK : SQUASH_PROCESSING; }
void Message::uncompress() { if (!isCompressed()) return; std::string errorStr = "Unsupported compression"; size_t actualSize; std::string comprType; std::string comprIdent = _meta["um.compressed"]; size_t colon = comprIdent.find_first_of(':'); if (colon == std::string::npos) { errorStr = "No colon found in um.compressed meta field"; goto DECOMPRESS_ERROR; } actualSize = strTo<size_t>(comprIdent.substr(0, colon)); comprType = comprIdent.substr(colon + 1); _meta["um.compressRatio"] = toStr((double)_size / (double)actualSize); // std::cout << _size << " vs " << actualSize << std::endl; if (false) {} #ifdef BUILD_WITH_COMPRESSION_MINIZ else if (comprType == "miniz") { int cmp_status; uint8_t *pUncmp; pUncmp = (mz_uint8 *)malloc((size_t)actualSize); cmp_status = mz_uncompress(pUncmp, (mz_ulong*)&actualSize, (const unsigned char *)_data.get(), _size); if (cmp_status != MZ_OK) { errorStr = mz_error(cmp_status); goto DECOMPRESS_ERROR; } _size = actualSize; _data = SharedPtr<char>((char*)pUncmp); _meta.erase("um.compressed"); return; } #endif #ifdef BUILD_WITH_COMPRESSION_FASTLZ else if (comprType == "fastlz") { void* uncompressed = malloc((size_t)actualSize); // returns the size of the decompressed block. actualSize = fastlz_decompress(_data.get(), _size, uncompressed, actualSize); // If error occurs, e.g. the compressed data is corrupted or the output buffer is not large enough, then 0 if (actualSize == 0) { errorStr = "fastlz_decompress returned 0"; goto DECOMPRESS_ERROR; } _size = actualSize; _data = SharedPtr<char>((char*)uncompressed); _meta.erase("um.compressed"); return; } #endif #ifdef BUILD_WITH_COMPRESSION_LZ4 else if (comprType == "lz4") { #ifdef LZ4_FRAME LZ4F_errorCode_t n; LZ4F_decompressionContext_t ctx; void* uncompressed = malloc((size_t)actualSize); n = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(n)) { errorStr = LZ4F_getErrorName(n); goto DECOMPRESS_ERROR; } n = LZ4F_decompress(ctx, uncompressed, &actualSize, _data.get(), &_size, NULL); if (LZ4F_isError(n)) { errorStr = LZ4F_getErrorName(n); goto DECOMPRESS_ERROR; } _size = actualSize; _data = SharedPtr<char>((char*)uncompressed); _meta.erase("um.compressed"); LZ4F_freeDecompressionContext(ctx); #else char* uncompressed = (char*)malloc((size_t)actualSize); int n = LZ4_decompress_fast(_data.get(), uncompressed, actualSize); if (n < 0) { errorStr = "Decompression failed"; goto DECOMPRESS_ERROR; } _size = actualSize; _data = SharedPtr<char>((char*)uncompressed); _meta.erase("um.compressed"); #endif return; } #endif DECOMPRESS_ERROR: UM_LOG_WARN("Could not decompress message: %s", errorStr.c_str()); }
SEXP do_lzDecompress (SEXP FROM) { SEXP ANS; LZ4F_decompressionContext_t ctx; LZ4F_frameInfo_t info; char *from; char *ans; void *src; size_t m, n, output_size, input_size = xlength(FROM); size_t ibuf, obuf, icum, ocum; if(TYPEOF(FROM) != RAWSXP) error("'from' must be raw or character"); from = (char *)RAW(FROM); /* An implementation following the standard API would do this: * LZ4F_errorCode_t err = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); * if (LZ4F_isError (err)) error("could not create LZ4 decompression context"); * ... * LZ4F_freeDecompressionContext(ctx); * The problem with that approach is that LZ4F_createDecompressionContext * allocates memory with calloc internally. Later, if R's allocVector fails, * for example, or if R interrupts this function somewhere in '...' then * those internal allocations in LZ4F_createDecompressionContext leak--that is * they aren't ever de-allocated. * * We explicitly allocat the LZ4F_decompressionContext_t pointer using a * replica of the internal LZ4F_dctx_t structure defined near the top of this * file, see the note and corresponding warning! We allocate on the heap (via * R_alloc) here instead of a seemingly simpler stack allocation because LZ4 * indicates that the address needs to be aligned to 8-byte boundaries which is * provided by R_alloc, see: * https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Transient-storage-allocation */ LZ4F_dctx_t *dctxPtr = (LZ4F_dctx_t *)R_alloc(1, sizeof(LZ4F_dctx_t)); memset(dctxPtr, 0, sizeof(LZ4F_dctx_t)); dctxPtr->version = LZ4F_VERSION; ctx = (LZ4F_decompressionContext_t)dctxPtr; m = input_size; n = LZ4F_getFrameInfo(ctx, &info, (void *)from, &input_size); if (LZ4F_isError (n)) error("LZ4F_getFrameInfo"); src = from + input_size; // lz4 frame header offset output_size = (size_t) info.contentSize; ANS = allocVector(RAWSXP, output_size); ans = (char *)RAW(ANS); input_size = m - input_size; icum = 0; ibuf = lzframe_chunksize; if(ibuf > input_size) ibuf = input_size; ocum = 0; obuf = output_size; for(;;) { n = LZ4F_decompress(ctx, ans, &obuf, src, &ibuf, NULL); if (LZ4F_isError (n)) error("LZ4F_decompress"); icum = icum + ibuf; ocum = ocum + obuf; if(icum >= input_size) break; ans = ans + obuf; src = src + ibuf; ibuf = lzframe_chunksize; if(ibuf > (input_size - icum)) ibuf = input_size - icum; obuf = output_size - ocum; } return ANS; }
static int verify_basic_LZ4F_decompression(const void* compressedBuffer, int cSize, U64 crcOrig, void* decodedBuffer, const size_t decodedBufferSize) { DISPLAYLEVEL(3, "%s (cSize %i, crcOrig %08x, decodedBufferSize %i)\n", __FUNCTION__, cSize, (unsigned int)crcOrig, (int)decodedBufferSize); LZ4F_decompressionContext_t dCtx = NULL; U64 crcDest; size_t compressedBufferSize = cSize; BYTE* op = (BYTE*) decodedBuffer; BYTE* const oend = (BYTE*) decodedBuffer + decodedBufferSize; const BYTE* ip = (const BYTE*) compressedBuffer; const BYTE* const iend = (const BYTE*) compressedBuffer + cSize; LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__); memset(decodedBuffer, 0, decodedBufferSize); DISPLAYLEVEL(3, "Single Block : \n"); size_t destSize = decodedBufferSize; errorCode = LZ4F_decompress(dCtx, decodedBuffer, &destSize, compressedBuffer, &compressedBufferSize, NULL); UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__); crcDest = XXH64(decodedBuffer, decodedBufferSize, 1); DISPLAYLEVEL(3, "Regenerated %i bytes (%08x)\n", (int )decodedBufferSize, (unsigned int)crcDest); UT_VERIFY(crcDest == crcOrig, return __LINE__); memset(decodedBuffer, 0, decodedBufferSize); DISPLAYLEVEL(4, "Reusing decompression context \n"); { size_t iSize = compressedBufferSize - 4; const BYTE* cBuff = (const BYTE*) compressedBuffer; DISPLAYLEVEL(3, "Missing last 4 bytes : "); destSize = decodedBufferSize; errorCode = LZ4F_decompress(dCtx, decodedBuffer, &destSize, cBuff, &iSize, NULL); UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__); UT_VERIFY(errorCode, return __LINE__); crcDest = XXH64(decodedBuffer, destSize, 1); DISPLAYLEVEL(3, "crcDest (%08x)\n", (unsigned int)crcDest); DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned )errorCode); cBuff += iSize; iSize = errorCode; errorCode = LZ4F_decompress(dCtx, decodedBuffer, &destSize, cBuff, &iSize, NULL); UT_VERIFY(errorCode == 0, return __LINE__); crcDest = XXH64(decodedBuffer, decodedBufferSize, 1); DISPLAYLEVEL(3, "crcDest (%08x)\n", (unsigned int)crcDest); UT_VERIFY(crcDest == crcOrig, return __LINE__); } { size_t oSize = 0; size_t iSize = 0; LZ4F_frameInfo_t fi; DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__); DISPLAYLEVEL(3, " %u \n", (unsigned )errorCode); DISPLAYLEVEL(3, "get FrameInfo on null input : "); errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); UT_VERIFY(errorCode == (size_t) -LZ4F_ERROR_frameHeader_incomplete, return __LINE__); DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); iSize = 6; errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); UT_VERIFY(errorCode == (size_t) -LZ4F_ERROR_frameHeader_incomplete, return __LINE__); DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); ip += iSize; DISPLAYLEVEL(3, "get FrameInfo on enough input : "); iSize = 15 - iSize; errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__); DISPLAYLEVEL(3, " correctly decoded \n"); ip += iSize; } DISPLAYLEVEL(3, "Byte after byte : \n"); while (ip < iend) { size_t oSize = oend - op; size_t iSize = 1; errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__); op += oSize; ip += iSize; } crcDest = XXH64(decodedBuffer, decodedBufferSize, 1); UT_VERIFY(crcDest == crcOrig, return __LINE__); DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned )(op - (BYTE* )decodedBuffer), (unsigned )decodedBufferSize); errorCode = LZ4F_freeDecompressionContext(dCtx); UT_VERIFY(!LZ4F_isError(errorCode), return __LINE__); dCtx = NULL; return 0; }
static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { LZ4F_errorCode_t r; LZ4F_compressionContext_t ctx; char *src, *buf = NULL; size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size; r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(r)) { printf("Failed to create context: error %zu", r); return 1; } r = 1; src = malloc(BUF_SIZE); if (!src) { printf("Not enough memory"); goto cleanup; } frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences); size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; buf = malloc(size); if (!buf) { printf("Not enough memory"); goto cleanup; } n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); if (LZ4F_isError(n)) { printf("Failed to start compression: error %zu", n); goto cleanup; } printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n); for (;;) { k = fread(src, 1, BUF_SIZE, in); if (k == 0) break; count_in += k; n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); if (LZ4F_isError(n)) { printf("Compression failed: error %zu", n); goto cleanup; } offset += n; count_out += n; if (size - offset < frame_size + LZ4_FOOTER_SIZE) { printf("Writing %zu bytes\n", offset); k = fwrite(buf, 1, offset, out); if (k < offset) { if (ferror(out)) printf("Write failed"); else printf("Short write"); goto cleanup; } offset = 0; } } n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); if (LZ4F_isError(n)) { printf("Failed to end compression: error %zu", n); goto cleanup; } offset += n; count_out += n; printf("Writing %zu bytes\n", offset); k = fwrite(buf, 1, offset, out); if (k < offset) { if (ferror(out)) printf("Write failed"); else printf("Short write"); goto cleanup; } *size_in = count_in; *size_out = count_out; r = 0; cleanup: if (ctx) LZ4F_freeCompressionContext(ctx); free(src); free(buf); return r; }
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration) { unsigned testResult = 0; unsigned testNb = 0; void* srcBuffer = NULL; void* compressedBuffer = NULL; void* decodedBuffer = NULL; U32 coreRand = seed; LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cCtx = NULL; size_t result; const U32 startTime = FUZ_GetMilliStart(); XXH64_state_t xxh64; # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } // backup all allocated addresses, from which we will later select buffers const size_t max_buf_size = 131 KB; size_t num_buf_size_distribution_deviations = 0; LZ4SG_in_t sg_in_buf_potential [2*MAX_SG_BUFFERS]; LZ4SG_out_t sg_out_buf_potential[2*MAX_SG_BUFFERS]; LZ4SG_in_t sg_cin [MAX_SG_BUFFERS]; LZ4SG_out_t sg_cout[MAX_SG_BUFFERS]; LZ4SG_in_t sg_din [MAX_SG_BUFFERS]; LZ4SG_out_t sg_dout[MAX_SG_BUFFERS]; size_t sg_cin_len, sg_cout_len, sg_din_len, sg_dout_len; const size_t maxDstSize = LZ4_SG_compressBound(srcDataLength, NELEMS(sg_cin), NELEMS(sg_cout)); unsigned int i; for (i = 0; i < NELEMS(sg_in_buf_potential); i++) { sg_in_buf_potential [i].sg_base = malloc(max_buf_size); sg_in_buf_potential [i].sg_len = max_buf_size; sg_out_buf_potential[i].sg_base = malloc(max_buf_size); sg_out_buf_potential[i].sg_len = max_buf_size; } /* Init */ duration *= 1000; /* Create buffers */ result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); srcBuffer = malloc(srcDataLength); CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); const size_t compressedBufferLength = maxDstSize; compressedBuffer = malloc(compressedBufferLength); CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); /* jump to requested testNb */ for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer /* main fuzzer test loop */ for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++) { U32 randState = coreRand ^ prime1; (void)FUZ_rand(&coreRand); /* update seed */ srand48(FUZ_rand(&randState)); DISPLAYUPDATE(2, "\r%5u ", testNb); const size_t max_src_buf_size = (4 MB > srcDataLength) ? srcDataLength : 4 MB; unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(max_src_buf_size-1) - 1)) + 1; const size_t min_src_size = 20; const size_t min_first_dest_buf_size = 21; const size_t min_src_buf_size = 1; const size_t min_dst_buf_size = 10; size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + min_src_size; size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize); size_t cSize; size_t dstSize; size_t dstSizeBound; U64 crcOrig, crcDecoded; unsigned int test_selection = FUZ_rand(&randState); //TODO: enable lz4f_compress_compatibility_test with LZ4_SG_decompress int lz4f_compress_compatibility_test = 0;//(test_selection % 4) == 0; if (!lz4f_compress_compatibility_test) { // SGL compress unsigned int buffer_selection = FUZ_rand(&randState); if ((buffer_selection & 0xF) == 1) { // SG compress single source and single target buffers sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart; sg_cin[0].sg_len = srcSize; sg_cin_len = 1; sg_cout[0].sg_base = compressedBuffer; sg_cout[0].sg_len = compressedBufferLength; sg_cout_len = 1; dstSizeBound = dstSize = compressedBufferLength; } else { // SG compress random number and size source and target buffers sg_cin_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS); sg_cout_len = 1 + (FUZ_rand(&randState) % MAX_SG_BUFFERS); // single source buffer if (1 == sg_cin_len) { sg_cin[0].sg_base = (BYTE*)srcBuffer+srcStart; sg_cin[0].sg_len = srcSize; DISPLAYUPDATE(4, "INFO: single source buf size %i\n", (int)srcSize); } else { // multiple source buffers if (srcSize > sg_cin_len*max_buf_size/2) { srcSize = sg_cin_len*max_buf_size/2; num_buf_size_distribution_deviations++; DISPLAYUPDATE(4, "NOTE: source buffer total size deviation %i\n", (int)num_buf_size_distribution_deviations); } size_t exact_src_size = 0; unsigned int buf_size_mean = srcSize / sg_cin_len; for (i = 0; i < sg_cin_len; i++) { size_t buf_size = rnd_exponential(buf_size_mean, min_src_buf_size, max_buf_size); DISPLAYUPDATE(4, "INFO: source buf %i size %i\n", i, (int)buf_size); if (srcStart+exact_src_size+buf_size > srcDataLength) { buf_size = srcDataLength-(srcStart+exact_src_size); } sg_cin[i].sg_base = sg_in_buf_potential[i*2+1].sg_base; sg_cin[i].sg_len = buf_size; memcpy((void *)sg_cin[i].sg_base, (BYTE*)srcBuffer+srcStart+exact_src_size, buf_size); exact_src_size += buf_size; if (srcStart+exact_src_size == srcDataLength) { num_buf_size_distribution_deviations++; sg_cin_len = i+1; DISPLAYUPDATE(4, "NOTE: final source buffer size deviation %i (buffers number limited to %i)\n", (int)num_buf_size_distribution_deviations, (int)sg_cin_len); } } srcSize = exact_src_size; } // we can now derive the required limit for output dstSizeBound = LZ4_SG_compressBound(srcSize, sg_cin_len, sg_cout_len); // single target buffer if (1 == sg_cout_len) { sg_cout[0].sg_base = compressedBuffer; sg_cout[0].sg_len = compressedBufferLength; } else { // multiple target buffers int finalBufferTruncated = 0; dstSize = 0; unsigned int buf_size_mean = dstSizeBound / sg_cout_len; for (i = 0; i < sg_cout_len; i++) { const size_t min_buf_size = (i == 0) ? min_first_dest_buf_size : min_dst_buf_size; size_t buf_size = rnd_exponential(buf_size_mean, min_buf_size, max_buf_size); DISPLAYUPDATE(4, "INFO: target buf %i size %i\n", (int)i, (int)buf_size); if (dstSize+buf_size > dstSizeBound) { buf_size = dstSizeBound-dstSize; finalBufferTruncated = 1; } dstSize += buf_size; sg_cout[i].sg_base = sg_out_buf_potential[i*2+1].sg_base; sg_cout[i].sg_len = buf_size; if (finalBufferTruncated) { num_buf_size_distribution_deviations++; if (buf_size < min_buf_size) { // merge truncated with previous? if (i > 0) { sg_cout[i-1].sg_len += buf_size; if (sg_cout[i-1].sg_len > max_buf_size) { // skip, too much hassle DISPLAYUPDATE(4, "NOTE: unable to truncate final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations); sg_cout_len = 0; break; } } else { // can this happen? DISPLAYUPDATE(4, "NOTE: unable to truncate first and final target buffer size (deviations %i), skipping\n", (int)num_buf_size_distribution_deviations); sg_cout_len = 0; break; } sg_cout_len = i; } else { sg_cout_len = i+1; } DISPLAYUPDATE(4, "NOTE: final target buffer size truncated (%i), buffers number limited to %i, final's size is now %i (deviations %i)\n", (int)buf_size, (int)sg_cout_len, (int)sg_cout[sg_cout_len-1].sg_len, (int)num_buf_size_distribution_deviations); } } // skip/abort condition if (0 == sg_cout_len) continue; } if ((buffer_selection & 0xF) == 0) { //TODO: select a random input and output buffer and split it in two, // feeding consecutive addresses as consecutive entries in SGL } } crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); size_t sourceSizeOut = srcSize; result = LZ4_SG_compress(&sg_cin[0], sg_cin_len, &sg_cout[0], sg_cout_len, &sourceSizeOut, maxDstSize, DEFAULT_ACCEL); if (((result == 0) || (sourceSizeOut != srcSize)) && (dstSize < dstSizeBound)) { // forgive compression failure when output total size is lower than bound num_buf_size_distribution_deviations++; DISPLAYUPDATE(4, "NOTE: dstSize %i < %i dstSizeBound, compression attempt failed, not totally unexpected (deviations %i), skipping\n", (int)dstSize, (int)dstSizeBound, (int)num_buf_size_distribution_deviations); continue; } CHECK(result <= 0, "Compression failed (error %i)", (int)result); CHECK(sourceSizeOut != srcSize, "Compression stopped at %i out of %i", (int)sourceSizeOut, (int)srcSize); cSize = result; } else { // LZ4F compression - use it in order to verify SGL decompress compatibility with it DISPLAYUPDATE(4, "INFO: LZ4F compression\n"); // alternative // size_t dstMaxSize = LZ4F_compressFrameBound(srcSize, prefsPtr); // DISPLAYLEVEL(3, "compressFrame srcSize %zu dstMaxSize %zu\n", // srcSize, dstMaxSize); // cSize = LZ4F_compressFrame(compressedBuffer, dstMaxSize, (char*)srcBuffer + srcStart, srcSize, prefsPtr); // CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1); unsigned BSId = 4 + (FUZ_rand(&randState) & 3); unsigned BMId = FUZ_rand(&randState) & 1; unsigned CCflag = FUZ_rand(&randState) & 1; unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; LZ4F_preferences_t prefs; LZ4F_compressOptions_t cOptions; LZ4F_preferences_t* prefsPtr = &prefs; memset(&prefs, 0, sizeof(prefs)); memset(&cOptions, 0, sizeof(cOptions)); prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId; prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId; prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag; prefs.frameInfo.contentSize = frameContentSize; prefs.autoFlush = autoflush; prefs.compressionLevel = FUZ_rand(&randState) % 5; if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; const BYTE* ip = (const BYTE*)srcBuffer + srcStart; const BYTE* const iend = ip + srcSize; BYTE* op = (BYTE*)compressedBuffer; BYTE* const oend = op + LZ4F_compressFrameBound(srcDataLength, NULL); unsigned maxBits = FUZ_highbit((U32)srcSize); result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); op += result; while (ip < iend) { unsigned nbBitsSeg = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; size_t oSize = LZ4F_compressBound(iSize, prefsPtr); unsigned forceFlush = ((FUZ_rand(&randState) & 3) == 1); if (iSize > (size_t)(iend-ip)) iSize = iend-ip; cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); DISPLAYLEVEL(3, "compressUpdate ip %d iSize %zu oSize %zu forceFlush %d\n", (int)(ip-((const BYTE*)srcBuffer + srcStart)), iSize, oSize, forceFlush); result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; ip += iSize; if (forceFlush) { result = LZ4F_flush(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); op += result; } } result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); CHECK(LZ4F_isError(result), "Compression completion failed (error %i)", (int)result); op += result; cSize = op-(BYTE*)compressedBuffer; } //DECOMPRESS test_selection = FUZ_rand(&randState); if (lz4f_compress_compatibility_test || ((test_selection % 2) == 0)) { //TODO: SGL decompress with random buffer sizes // SGL decompress with same buffer sizes used for compression // prepare din with cout's data sg_din_len = sg_cout_len; for (i = 0; i < sg_din_len; i++) { sg_din[i].sg_len = sg_cout[i].sg_len; if (sg_cout[i].sg_len <= max_buf_size) { // enough room to copy - do it sg_din[i].sg_base = sg_in_buf_potential[i*2+0].sg_base; if (sg_din[i].sg_base != sg_cout[i].sg_base) { memcpy((void *)sg_din[i].sg_base, sg_cout[i].sg_base, sg_cout[i].sg_len); } } else { // this is probably single output buffer - skip copy, use directly sg_din[i].sg_base = sg_cout[i].sg_base; } } // prepare dout to receive decompressed data sg_dout_len = sg_cin_len; for (i = 0; i < sg_dout_len; i++) { sg_dout[i].sg_len = sg_cin[i].sg_len; if (sg_cin[i].sg_len <= max_buf_size) { // enough room to decompress into independent buffer sg_dout[i].sg_base = sg_out_buf_potential[i*2+0].sg_base; } else { // this is probably single input buffer, use an external output buffer sg_dout[i].sg_base = decodedBuffer; } } size_t sourceSizeOut = cSize; size_t maxOutputSize = srcSize; int decomp_result = LZ4_SG_decompress(&sg_din[0], sg_din_len, &sg_dout[0], sg_dout_len, &sourceSizeOut, maxOutputSize); CHECK(decomp_result <= 0, "SG decompression failed (error %i)", (int)decomp_result); CHECK(decomp_result != (int)srcSize, "SG decompression stopped at %i", (int)decomp_result); // verify result checksum size_t total_checked = 0; XXH64_reset(&xxh64, 1); for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) { size_t cur_size = sg_dout[i].sg_len; size_t rem = decomp_result - total_checked; if (rem < cur_size) cur_size = rem; total_checked += cur_size; XXH64_update(&xxh64, sg_dout[i].sg_base, cur_size); } crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) { DISPLAYLEVEL(1, "checked %i out of %i (crcDecoded %08x, crcOrig %08x)\n", (int)total_checked, decomp_result, (unsigned)crcDecoded, (unsigned)crcOrig); // locate error if any total_checked = 0; for (i = 0; (i < sg_dout_len) && ((int)total_checked < decomp_result); i++) { size_t cur_size = sg_dout[i].sg_len; size_t rem = decomp_result - total_checked; if (rem < cur_size) cur_size = rem; total_checked += cur_size; U64 crc_in = XXH64(sg_cin [i].sg_base, cur_size, 1); U64 crc_out = XXH64(sg_dout[i].sg_base, cur_size, 1); if (crc_in != crc_out) { locateBuffDiff(sg_cin[i].sg_base, sg_dout[i].sg_base, cur_size); break; } } DISPLAYLEVEL(1, "checked %i out of %i\n", (int)total_checked, decomp_result); } CHECK(crcDecoded != crcOrig, "Decompression corruption"); } else { // prepare compressedBuffer from SGL size_t total_copied = 0; for (i = 0; i < sg_cout_len; i++) { size_t buf_size_bytes = cSize - total_copied; if (buf_size_bytes == 0) break; if (buf_size_bytes > sg_cout[i].sg_len) buf_size_bytes = sg_cout[i].sg_len; if (((char *)compressedBuffer)+total_copied != sg_cout[i].sg_base) { memcpy(((char *)compressedBuffer)+total_copied, sg_cout[i].sg_base, buf_size_bytes); } total_copied += buf_size_bytes; } LZ4F_decompressOptions_t dOptions; memset(&dOptions, 0, sizeof(dOptions)); const BYTE* ip = (const BYTE*)compressedBuffer; const BYTE* const iend = ip + cSize; BYTE* op = (BYTE*)decodedBuffer; BYTE* const oend = op + srcDataLength; size_t totalOut = 0; unsigned maxBits = FUZ_highbit((U32)cSize); XXH64_reset(&xxh64, 1); if (maxBits < 3) maxBits = 3; while (ip < iend) { unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; size_t oSize = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; if (iSize > (size_t)(iend-ip)) iSize = iend-ip; if (oSize > (size_t)(oend-op)) oSize = oend-op; dOptions.stableDst = FUZ_rand(&randState) & 1; result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize); CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s ip %d)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result), (int)(ip-(const BYTE*)compressedBuffer)); XXH64_update(&xxh64, op, (U32)oSize); totalOut += oSize; op += oSize; ip += iSize; } CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); if (totalOut) /* otherwise, it's a skippable frame */ { crcDecoded = XXH64_digest(&xxh64); if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize); CHECK(crcDecoded != crcOrig, "Decompression corruption"); } } } DISPLAYLEVEL(2, "\rAll tests completed \n"); _end: LZ4F_freeDecompressionContext(dCtx); LZ4F_freeCompressionContext(cCtx); free(srcBuffer); free(compressedBuffer); free(decodedBuffer); for (i = 0; i < NELEMS(sg_in_buf_potential); i++) { free((void *)(sg_in_buf_potential [i].sg_base)); free( sg_out_buf_potential[i].sg_base); } if (num_buf_size_distribution_deviations > 0) { DISPLAYLEVEL(2, "NOTE: %i buffer size deviations \n", (int)num_buf_size_distribution_deviations); } if (pause) { DISPLAY("press enter to finish \n"); (void)getchar(); } return testResult; _output_error: testResult = 1; goto _end; // unreachable return -1; #undef CHECK }