void LZ4CompressStream::FlushOut()
{
if(ptr == (byte *)~buffer)
return;
CoWork co;
int osz = LZ4_compressBound(BLOCK_BYTES);
byte *t = ~outbuf;
int ii = 0;
for(byte *s = ~buffer; s < ptr; s += BLOCK_BYTES) {
int origsize = min((int)BLOCK_BYTES, int(ptr - s));
#ifdef _MULTITHREADED
if(concurrent)
co & [=] {
outsz[ii] = LZ4_compress_default((char *)s, (char *)t, origsize, osz);
};
else
#endif
outsz[ii] = LZ4_compress_default((char *)s, (char *)t, origsize, osz);
ii++;
t += osz;
}
if(concurrent)
co.Finish();
byte *s = ~buffer;
t = ~outbuf;
for(int i = 0; i < ii; i++) {
int origsize = min((int)BLOCK_BYTES, int(ptr - s));
int clen = outsz[i];
if(clen < 0) {
SetError();
return;
}
if(clen >= origsize || clen == 0) {
out->Put32le(0x80000000 | origsize);
out->Put(s, origsize);
}
else {
out->Put32le(clen);
out->Put(t, clen);
}
s += BLOCK_BYTES;
t += osz;
}
int origsize = int(ptr - ~buffer);
xxh.Put(~buffer, origsize);
pos += origsize;
ptr = ~buffer;
}
void* CBAWMeshWriter::compressWithLz4AndTryOnStack(const void* _input, size_t _inputSize, void* _stack, size_t _stackSize, size_t& _outComprSize) const
{
void* data = _stack;
size_t dstSize = _stackSize;
size_t compressedSize = 0;
const int lz4CompressBound = LZ4_compressBound(_inputSize);
if (lz4CompressBound) // if input is not too large
{
if (lz4CompressBound > _stackSize)
{
dstSize = asset::BlobHeaderV1::calcEncSize(lz4CompressBound);
data = _IRR_ALIGNED_MALLOC(dstSize,_IRR_SIMD_ALIGNMENT);
}
compressedSize = LZ4_compress_default((const char*)_input, (char*)data, _inputSize, dstSize);
}
if (!compressedSize) // if compression did not succeed
{
if (data != _stack)
_IRR_ALIGNED_FREE(data);
compressedSize = _inputSize;
data = const_cast<void*>(_input);
#ifdef _DEBUG
os::Printer::log("Failed to compress (lz4). Blob exported without compression.", ELL_WARNING);
#endif
}
_outComprSize = compressedSize;
return data;
}
void compress_data(Data *d) {
int slen = d->datalen;
int olen = LZ4_COMPRESSBOUND(slen);
char cbuf[olen];
#if LZ4_VERSION_MINOR >= 7
int ocnt = LZ4_compress_default((const char*)d->data, cbuf, slen, olen);
#else
/* In this API version, $cbuf is required to be large enough to handle
* up to LZ4_COMPRESSBOUND($slen) bytes. Since we already allocate
* this large of a buffer anyway, we just drop $olen.
*
* This is required for building against the liblz4 version in Debian
* Jessie.
*/
int ocnt = LZ4_compress((const char*)d->data, cbuf, slen);
#endif
if(ocnt == 0) /* should never happen */
LERROR(EXIT_FAILURE, 0, "LZ4_compress_default() failed");
free(d->data);
d->data = malloc(ocnt);
GUARD_MALLOC(d->data);
d->datalen = (size_t)ocnt;
memcpy(d->data, (const void*)cbuf, d->datalen);
}
size_t lz4_output_stream::compress(void* sink, size_t sink_size) {
VAST_ENTER_WITH(VAST_ARG(sink, sink_size));
VAST_ASSERT(sink_size >= valid_bytes_);
auto n = LZ4_compress_default(
reinterpret_cast<char const*>(uncompressed_.data()),
reinterpret_cast<char*>(sink),
static_cast<int>(valid_bytes_),
static_cast<int>(sink_size));
VAST_ASSERT(n > 0);
VAST_RETURN(n);
}
static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen, void *ctx)
{
int out_len = LZ4_compress_default(src, dst,
slen, *dlen, ctx);
if (!out_len)
return -EINVAL;
*dlen = out_len;
return 0;
}
std::pair<std::unique_ptr<char[]>, size_t> compress(const void* data, size_t dataSize, int level)
{
if (dataSize == 0) return std::make_pair(std::unique_ptr<char[]>(), 0);
int csizeBound = LZ4_compressBound(dataSize);
std::unique_ptr<char[]> cdata(new char[csizeBound]);
int csize = (level == 0)
? LZ4_compress_default(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound)
: LZ4_compress_HC(static_cast<const char*>(data), cdata.get(), dataSize, csizeBound, level);
assert(csize >= 0 && csize <= csizeBound);
return std::make_pair(std::move(cdata), csize);
}
size_t LZ4Encoder::OnCode(const MemoryByteData& input, MemoryByteData& output) const
{
MemoryStream outputStream(output);
outputStream.Write((uint)input.Size());
int size = LZ4_compress_default((const char*)input.Data(), (char*)output.MutableData() + sizeof(uint), (int)input.Size(), (int)output.Size() - sizeof(uint));
if (size>0)
{
size += sizeof(uint);
output.ForceSetSize(size);
}
else
{
output.ForceSetSize(0);
}
return size;
}
int
G_lz4_compress(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz)
{
int err, nbytes, buf_sz;
unsigned char *buf;
/* Catch errors early */
if (src == NULL || dst == NULL)
return -1;
/* Don't do anything if either of these are true */
if (src_sz <= 0 || dst_sz <= 0)
return 0;
/* Output buffer has to be larger for single pass compression */
buf_sz = LZ4_compressBound(src_sz);
if (NULL == (buf = (unsigned char *)
G_calloc(buf_sz, sizeof(unsigned char))))
return -1;
/* Do single pass compression */
err = LZ4_compress_default((char *)src, (char *)buf, src_sz, buf_sz);
if (err <= 0) {
G_free(buf);
return -1;
}
if (err >= src_sz) {
/* compression not possible */
G_free(buf);
return -2;
}
/* bytes of compressed data is return value */
nbytes = err;
/* Copy the data from buf to dst */
for (err = 0; err < nbytes; err++)
dst[err] = buf[err];
G_free(buf);
return nbytes;
}
static mrb_value
mrb_LZ4_compress_default(mrb_state *mrb, mrb_value self)
{
char *source;
mrb_int source_size;
mrb_get_args(mrb, "s", &source, &source_size);
int maxDestSize = LZ4_COMPRESSBOUND(source_size);
if (likely(maxDestSize)) {
mrb_value dest = mrb_str_buf_new(mrb, maxDestSize);
int destSize = LZ4_compress_default(source, RSTRING_PTR(dest), source_size, RSTRING_CAPA(dest));
if (likely(destSize)) {
return mrb_str_resize(mrb, dest, destSize);
} else {
mrb_raise(mrb, E_LZ4_ERROR, "cannot compress");
}
} else {
mrb_raise(mrb, E_ARGUMENT_ERROR, "source_size is too large");
}
}
lz4_log_buf* serialize_to_proto_buf_with_malloc_lz4(log_group_builder* bder)
{
size_t length = log_get_packed_size(bder->grp);
unsigned char * buf = (unsigned char *)malloc(length);
log_pack(bder->grp, buf);
int compress_bound = LZ4_compressBound(length);
char *compress_data = (char *)malloc(compress_bound);
int compressed_size = LZ4_compress_default((char *)buf, compress_data, length, compress_bound);
if(compressed_size <= 0)
{
free(buf);
free(compress_data);
return NULL;
}
lz4_log_buf* pLogbuf = (lz4_log_buf*)malloc(sizeof(lz4_log_buf) + compressed_size);
pLogbuf->length = compressed_size;
pLogbuf->raw_length = length;
memcpy(pLogbuf->data, compress_data, compressed_size);
free(buf);
free(compress_data);
return pLogbuf;
}
/* Write a frame or frames to the log. */
int sqlite3WalFrames(Wal *pWal, int szPage, PgHdr *pList, Pgno nTruncate, int isCommit, int sync_flags)
{
PgHdr *p;
MDB_val key, data;
int rc;
mdbinf* mdb;
MDB_txn* txn;
#if ATOMIC
db_thread *thr = g_tsd_thread;
db_connection* pCon = g_tsd_conn;
#else
db_thread* thr = enif_tsd_get(g_tsd_thread);
db_connection* pCon = enif_tsd_get(g_tsd_conn);
#endif
#if ATOMIC
if (!g_tsd_wmdb)
lock_wtxn(thr->nEnv);
mdb = g_tsd_wmdb;
#else
mdb = enif_tsd_get(g_tsd_wmdb);
if (!mdb)
lock_wtxn(thr->nEnv);
mdb = enif_tsd_get(g_tsd_wmdb);
#endif
txn = mdb->txn;
if (!mdb)
return SQLITE_ERROR;
key.mv_size = sizeof(u64);
key.mv_data = (void*)&pWal->index;
// Term/evnum must always be increasing
if ((pWal->inProgressTerm > 0 && pWal->inProgressTerm < pWal->lastCompleteTerm) ||
(pWal->inProgressEvnum > 0 && pWal->inProgressEvnum < pWal->lastCompleteEvnum))
return SQLITE_ERROR;
track_time(2,thr);
// ** - Pages DB: {<<ActorIndex:64, Pgno:32/unsigned>>, <<Evterm:64,Evnum:64,Fragment,CompressedPage/binary>>}
for(p=pList; p; p=p->pDirty)
{
u8 pagesKeyBuf[sizeof(u64)+sizeof(u32)];
u8 pagesBuf[PAGE_BUFF_SIZE];
int full_size = 0;
int page_size = LZ4_compress_default((char*)p->pData,(char*)pagesBuf+sizeof(u64)*2+1,szPage,sizeof(pagesBuf));
char fragment_index = 0;
int skipped = 0;
track_time(3,thr);
DBG("Insert frame, actor=%lld, pgno=%u, "
"term=%lld, evnum=%lld, commit=%d, truncate=%d, compressedsize=%d",
pWal->index,p->pgno,pWal->inProgressTerm,pWal->inProgressEvnum,
isCommit,nTruncate,page_size);
if (pCon->doReplicate)
{
u8 hdr[sizeof(u64)*2+sizeof(u32)*2];
put8byte(hdr, pWal->inProgressTerm);
put8byte(hdr+sizeof(u64), pWal->inProgressEvnum);
put4byte(hdr+sizeof(u64)*2, p->pgno);
if (p->pDirty)
put4byte(hdr+sizeof(u64)*2+sizeof(u32), 0);
else
put4byte(hdr+sizeof(u64)*2+sizeof(u32), nTruncate);
#ifndef _TESTAPP_
wal_page_hook(thr,pagesBuf+sizeof(u64)*2+1, page_size, hdr, sizeof(hdr));
#endif
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
// Check if there are pages with the same or higher evnum/evterm. If there are, delete them.
// This can happen if sqlite flushed some page to disk before commiting, because there were
// so many pages that they could not be held in memory. Or it could happen if pages need to be
// overwritten because there was a write that did not pass raft consensus.
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_SET_KEY);
if (rc == MDB_SUCCESS)
{
size_t ndupl;
mdb_cursor_count(mdb->cursorPages,&ndupl);
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_LAST_DUP);
if (rc == MDB_SUCCESS)
{
MDB_val pgDelKey = {0,NULL}, pgDelVal = {0,NULL};
u64 evnum, evterm;
u8 frag = *((u8*)data.mv_data+sizeof(u64)*2);
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
while ((evterm > pWal->inProgressTerm || evnum >= pWal->inProgressEvnum))
//(pWal->inProgressTerm + pWal->inProgressEvnum) > 0)
{
DBG("Deleting pages higher or equal to current. "
"Evterm=%llu, evnum=%llu, curterm=%llu, curevn=%llu, dupl=%ld",
evterm,evnum,pWal->inProgressTerm,pWal->inProgressEvnum,ndupl);
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
mdb_cursor_get(mdb->cursorPages,&pgDelKey,&pgDelVal,MDB_GET_CURRENT);
// if (mdb_cursor_del(mdb->cursorPages,0) != MDB_SUCCESS)
// {
// DBG("Cant delete!");
// break;
// }
if (frag == 0)
pWal->allPages--;
ndupl--;
if (!ndupl)
break;
rc = mdb_cursor_get(mdb->cursorPages,&key,&data,MDB_PREV_DUP);
if (rc != MDB_SUCCESS)
break;
memcpy(&evterm, data.mv_data, sizeof(u64));
memcpy(&evnum, (u8*)data.mv_data + sizeof(u64), sizeof(u64));
frag = *((u8*)data.mv_data+sizeof(u64)*2);
}
if (pgDelKey.mv_data != NULL)
{
if ((rc = mdb_del(txn,mdb->pagesdb,&pgDelKey,&pgDelVal)) != MDB_SUCCESS)
{
DBG("Unable to cleanup page from pagedb %d",rc);
break;
}
pgDelKey.mv_data = NULL;
}
}
memcpy(pagesKeyBuf, &pWal->index,sizeof(u64));
memcpy(pagesKeyBuf + sizeof(u64), &p->pgno, sizeof(u32));
key.mv_size = sizeof(pagesKeyBuf);
key.mv_data = pagesKeyBuf;
}
track_time(4,thr);
memcpy(pagesBuf, &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + sizeof(u64), &pWal->inProgressEvnum, sizeof(u64));
full_size = page_size + sizeof(u64)*2 + 1;
if (full_size < thr->maxvalsize)
fragment_index = 0;
else
{
full_size = page_size;
skipped = thr->maxvalsize - sizeof(u64)*2 - 1;
full_size -= skipped;
while(full_size > 0)
{
full_size -= (thr->maxvalsize - sizeof(u64)*2 - 1);
fragment_index++;
}
full_size = page_size + sizeof(u64)*2 +1;
}
pagesBuf[sizeof(u64)*2] = fragment_index;
data.mv_size = fragment_index == 0 ? full_size : thr->maxvalsize;
data.mv_data = pagesBuf;
// fragment_index == 0 ? MDB_APPENDDUP : 0
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
// printf("Cursor put failed to pages %d",rc);
DBG("ERROR: cursor put failed=%d, datasize=%d",rc,full_size);
return SQLITE_ERROR;
}
fragment_index--;
skipped = data.mv_size;
while (fragment_index >= 0)
{
DBG("Insert fragment %d",(int)fragment_index);
if (fragment_index == 0)
data.mv_size = full_size - skipped + sizeof(u64)*2 + 1;
else
data.mv_size = thr->maxvalsize;
data.mv_data = pagesBuf + skipped - (sizeof(u64)*2+1);
memcpy(pagesBuf + skipped - (sizeof(u64)*2+1), &pWal->inProgressTerm, sizeof(u64));
memcpy(pagesBuf + skipped - (sizeof(u64)+1), &pWal->inProgressEvnum, sizeof(u64));
pagesBuf[skipped-1] = fragment_index;
if ((rc = mdb_cursor_put(mdb->cursorPages,&key,&data,0)) != MDB_SUCCESS)
{
DBG("ERROR: cursor secondary put failed: err=%d, datasize=%d, skipped=%d, frag=%d",
rc,full_size, skipped, (int)fragment_index);
return SQLITE_ERROR;
}
skipped += data.mv_size - sizeof(u64)*2 - 1;
fragment_index--;
}
thr->pagesChanged++;
}
// printf("");
// ** - Log DB: {<<ActorIndex:64, Evterm:64, Evnum:64>>, <<Pgno:32/unsigned>>}
if (pWal->inProgressTerm > 0)
{
for(p=pList; p; p=p->pDirty)
{
u8 logKeyBuf[sizeof(u64)*3];
DBG("Inserting to log");
memcpy(logKeyBuf, &pWal->index, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64), &pWal->inProgressTerm, sizeof(u64));
memcpy(logKeyBuf + sizeof(u64)*2, &pWal->inProgressEvnum, sizeof(u64));
key.mv_size = sizeof(logKeyBuf);
key.mv_data = logKeyBuf;
data.mv_size = sizeof(u32);
data.mv_data = &p->pgno;
if (mdb_cursor_put(mdb->cursorLog,&key,&data,0) != MDB_SUCCESS)
{
// printf("Cursor put failed to log");
DBG("ERROR: cursor put to log failed: %d",rc);
return SQLITE_ERROR;
}
pWal->allPages++;
}
}
else
{
DBG("Skipping log");
for(p=pList; p; p=p->pDirty)
pWal->allPages++;
}
/** - Info DB: {<<ActorIndex:64>>, <<V,FirstCompleteTerm:64,FirstCompleteEvnum:64,
LastCompleteTerm:64,LastCompleteEvnum:64,
InprogressTerm:64,InProgressEvnum:64>>} */
{
if (isCommit)
{
DBG("Commit actor=%llu fct=%llu, fcev=%llu, lct=%llu, lcev=%llu, int=%llu, inev=%llu",
pWal->index,
pWal->firstCompleteTerm, pWal->firstCompleteEvnum, pWal->lastCompleteTerm,
pWal->lastCompleteEvnum, pWal->inProgressTerm,pWal->inProgressEvnum);
#ifndef _TESTAPP_
enif_mutex_lock(pWal->mtx);
#endif
pWal->lastCompleteTerm = pWal->inProgressTerm > 0 ?
pWal->inProgressTerm : pWal->lastCompleteTerm;
pWal->lastCompleteEvnum = pWal->inProgressEvnum > 0 ?
pWal->inProgressEvnum : pWal->lastCompleteEvnum;
if (pWal->firstCompleteTerm == 0)
{
pWal->firstCompleteTerm = pWal->inProgressTerm;
pWal->firstCompleteEvnum = pWal->inProgressEvnum;
}
pWal->inProgressTerm = pWal->inProgressEvnum = 0;
pWal->mxPage = pWal->mxPage > nTruncate ? pWal->mxPage : nTruncate;
// pWal->changed = 0;
thr->forceCommit = 1;
pCon->dirty = 0;
#ifndef _TESTAPP_
enif_mutex_unlock(pWal->mtx);
#endif
DBG("cur mxpage=%u",pWal->mxPage);
}
else
{
// pWal->changed = 1;
pCon->dirty = 1;
}
thr->pagesChanged++;
rc = storeinfo(pWal,0,0,NULL);
if (rc != SQLITE_OK)
return rc;
track_time(5,thr);
}
return SQLITE_OK;
}
void store_cb (flux_t *h, flux_msg_handler_t *mh,
const flux_msg_t *msg, void *arg)
{
sqlite_ctx_t *ctx = arg;
const void *data;
int size, hash_len;
uint8_t hash[BLOBREF_MAX_DIGEST_SIZE];
char blobref[BLOBREF_MAX_STRING_SIZE] = "-";
int uncompressed_size = -1;
int rc = -1;
int old_state;
//delay cancellation to ensure lock-correctness in sqlite
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state);
if (flux_request_decode_raw (msg, NULL, &data, &size) < 0) {
flux_log_error (h, "store: request decode failed");
goto done;
}
if (size > ctx->blob_size_limit) {
errno = EFBIG;
goto done;
}
if (blobref_hash (ctx->hashfun, (uint8_t *)data, size, blobref,
sizeof (blobref)) < 0)
goto done;
if ((hash_len = blobref_strtohash (blobref, hash, sizeof (hash))) < 0)
goto done;
if (size >= compression_threshold) {
int r;
int out_len = LZ4_compressBound(size);
if (ctx->lzo_bufsize < out_len && grow_lzo_buf (ctx, out_len) < 0)
goto done;
r = LZ4_compress_default (data, ctx->lzo_buf, size, out_len);
if (r == 0) {
errno = EINVAL;
goto done;
}
uncompressed_size = size;
size = r;
data = ctx->lzo_buf;
}
if (sqlite3_bind_text (ctx->store_stmt, 1, (char *)hash, hash_len,
SQLITE_STATIC) != SQLITE_OK) {
log_sqlite_error (ctx, "store: binding key");
set_errno_from_sqlite_error (ctx);
goto done;
}
if (sqlite3_bind_int (ctx->store_stmt, 2, uncompressed_size) != SQLITE_OK) {
log_sqlite_error (ctx, "store: binding size");
set_errno_from_sqlite_error (ctx);
goto done;
}
if (sqlite3_bind_blob (ctx->store_stmt, 3,
data, size, SQLITE_STATIC) != SQLITE_OK) {
log_sqlite_error (ctx, "store: binding data");
set_errno_from_sqlite_error (ctx);
goto done;
}
if (sqlite3_step (ctx->store_stmt) != SQLITE_DONE
&& sqlite3_errcode (ctx->db) != SQLITE_CONSTRAINT) {
log_sqlite_error (ctx, "store: executing stmt");
set_errno_from_sqlite_error (ctx);
goto done;
}
rc = 0;
done:
if (rc < 0) {
if (flux_respond_error (h, msg, errno, NULL) < 0)
flux_log_error (h, "store: flux_respond_error");
}
else {
if (flux_respond_raw (h, msg, blobref, strlen (blobref) + 1) < 0)
flux_log_error (h, "store: flux_respond_raw");
}
(void) sqlite3_reset (ctx->store_stmt);
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
}
static ZEND_FUNCTION(lz4_compress)
{
zval *data;
char *output;
int output_len, data_len, dst_len;
long level = 0;
long maxLevel = (long)PHP_LZ4_CLEVEL_MAX;
char *extra = NULL;
#if ZEND_MODULE_API_NO >= 20141001
size_t extra_len = -1;
#else
int extra_len = -1;
#endif
int offset = 0;
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC,
"z|ls", &data, &level,
&extra, &extra_len) == FAILURE) {
RETURN_FALSE;
}
if (Z_TYPE_P(data) != IS_STRING) {
zend_error(E_WARNING,
"lz4_compress : expects parameter to be string.");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
offset = extra_len;
} else {
offset = sizeof(int);
}
data_len = Z_STRLEN_P(data);
dst_len = LZ4_compressBound(data_len) + offset;
output = (char *)emalloc(dst_len);
if (!output) {
zend_error(E_WARNING, "lz4_compress : memory error");
RETURN_FALSE;
}
if (extra && extra_len > 0) {
memcpy(output, extra, offset);
} else {
/* Set the data length */
memcpy(output, &data_len, offset);
}
if (level == 0) {
output_len = LZ4_compress_default(Z_STRVAL_P(data), output + offset, data_len, dst_len - offset - 1);
} else {
if (level > maxLevel || level < 0) {
zend_error(E_WARNING, "lz4_compress: compression level (%ld)"
" must be within 1..%ld", level, maxLevel);
efree(output);
RETURN_FALSE;
}
output_len = LZ4_compress_HC(Z_STRVAL_P(data), output + offset, data_len, dst_len - offset - 1, level);
}
if (output_len <= 0) {
RETVAL_FALSE;
} else {
#if ZEND_MODULE_API_NO >= 20141001
RETVAL_STRINGL(output, output_len + offset);
#else
RETVAL_STRINGL(output, output_len + offset, 1);
#endif
}
efree(output);
}
void CompressedWriteBuffer::nextImpl()
{
if (!offset())
return;
size_t uncompressed_size = offset();
size_t compressed_size = 0;
char * compressed_buffer_ptr = nullptr;
/** The format of compressed block - see CompressedStream.h
*/
switch (method)
{
case CompressionMethod::LZ4:
case CompressionMethod::LZ4HC:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
compressed_buffer.resize(header_size + LZ4_COMPRESSBOUND(uncompressed_size));
#pragma GCC diagnostic pop
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::LZ4);
if (method == CompressionMethod::LZ4)
compressed_size = header_size + LZ4_compress_default(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size));
else
compressed_size = header_size + LZ4_compress_HC(
working_buffer.begin(),
&compressed_buffer[header_size],
uncompressed_size,
LZ4_COMPRESSBOUND(uncompressed_size),
0);
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
case CompressionMethod::ZSTD:
{
static constexpr size_t header_size = 1 + sizeof(UInt32) + sizeof(UInt32);
compressed_buffer.resize(header_size + ZSTD_compressBound(uncompressed_size));
compressed_buffer[0] = static_cast<UInt8>(CompressionMethodByte::ZSTD);
size_t res = ZSTD_compress(
&compressed_buffer[header_size],
compressed_buffer.size(),
working_buffer.begin(),
uncompressed_size,
1);
if (ZSTD_isError(res))
throw Exception("Cannot compress block with ZSTD: " + std::string(ZSTD_getErrorName(res)), ErrorCodes::CANNOT_COMPRESS);
compressed_size = header_size + res;
UInt32 compressed_size_32 = compressed_size;
UInt32 uncompressed_size_32 = uncompressed_size;
unalignedStore(&compressed_buffer[1], compressed_size_32);
unalignedStore(&compressed_buffer[5], uncompressed_size_32);
compressed_buffer_ptr = &compressed_buffer[0];
break;
}
default:
throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD);
}
CityHash_v1_0_2::uint128 checksum = CityHash_v1_0_2::CityHash128(compressed_buffer_ptr, compressed_size);
out.write(reinterpret_cast<const char *>(&checksum), sizeof(checksum));
out.write(compressed_buffer_ptr, compressed_size);
}
size_t compress(const char* in, size_t in_size, char* out, size_t out_size) {
return LZ4_compress_default(in, out, in_size, out_size);
}
static int ct_archive_data(struct hsm_copyaction_private *hcp, const char *src,
const char *dst, int src_fd,
const struct hsm_action_item *hai, long hal_flags)
{
struct hsm_extent he;
__u64 file_offset = hai->hai_extent.offset;
struct stat src_st;
char *uncompress_buf = NULL;
char *compress_buf = NULL;
__u64 write_total = 0;
__u64 length = hai->hai_extent.length;
time_t last_report_time;
int rc = 0;
double start_ct_now = ct_now();
time_t now;
int compression_bound = LZ4_compressBound(chunk_size);
// Archiving a file from Lustre to the object store
CT_TRACE("Archiving %s to %s", src, dst);
if (fstat(src_fd, &src_st) < 0) {
rc = -errno;
CT_ERROR(rc, "cannot stat '%s'", src);
return rc;
}
if (!S_ISREG(src_st.st_mode)) {
rc = -EINVAL;
CT_ERROR(rc, "'%s' is not a regular file", src);
return rc;
}
if (hai->hai_extent.offset > (__u64)src_st.st_size) {
rc = -EINVAL;
CT_ERROR(rc, "Trying to start reading past end ("LPU64" > "
"%jd) of '%s' source file", hai->hai_extent.offset,
(intmax_t)src_st.st_size, src);
return rc;
}
strippingInfo stripping_params;
stripping_params.lmm_stripe_count = 1;
stripping_params.lmm_stripe_size = ONE_MB;
if (ct_save_stripe(src_fd, src, &stripping_params)) {
return -1;
}
/* Don't read beyond a given extent */
if (length > src_st.st_size - hai->hai_extent.offset)
length = src_st.st_size - hai->hai_extent.offset;
last_report_time = time(NULL);
he.offset = file_offset;
he.length = 0;
rc = llapi_hsm_action_progress(hcp, &he, length, 0);
if (rc < 0) {
/* Action has been canceled or something wrong
* is happening. Stop copying data. */
CT_ERROR(rc, "progress ioctl for copy '%s'->'%s' failed",
src, dst);
goto out;
}
errno = 0;
uncompress_buf = malloc(chunk_size);
if (uncompress_buf == NULL) {
rc = -ENOMEM;
goto out;
}
compress_buf = malloc(compression_bound);
if (compress_buf == NULL) {
rc = -ENOMEM;
goto out;
}
int chunk_id = -1;
const char totalLength[] = "totallength";
const char chunksize[] = "chunksize";
const char stripe_size[] = "stripesize";
const char stripe_count[] = "stripecount";
const char path[] = "path";
const char uid[] = "uid";
const char gid[] = "gid";
char totalLength_s[TOTALLENGTH];
char chunksize_s[TOTALLENGTH];
char stripe_size_s[TOTALLENGTH];
char stripe_count_s[TOTALLENGTH];
char path_s[PATH_MAX];
char uid_s[TOTALLENGTH];
char gid_s[TOTALLENGTH];
snprintf(totalLength_s, sizeof(totalLength_s), "%llu", length);
snprintf(chunksize_s, sizeof(chunksize_s), "%i", chunk_size);
snprintf(stripe_size_s, sizeof(stripe_size_s), "%i", stripping_params.lmm_stripe_size);
snprintf(stripe_count_s, sizeof(stripe_count_s), "%i", stripping_params.lmm_stripe_count);
snprintf(path_s, sizeof(path_s), "%s", src); // FIXME should use fid2path to get the normal path
snprintf(uid_s, sizeof(uid_s), "%i", src_st.st_uid);
snprintf(gid_s, sizeof(gid_s), "%i", src_st.st_gid);
// Saving some metadata for disaster recovery
S3NameValue metadata[7] =
{
{
totalLength,
totalLength_s,
},
{
chunksize,
chunksize_s,
},
{
stripe_size,
stripe_size_s,
},
{
stripe_count,
stripe_count_s,
},
{
path,
path_s
},
{
uid,
uid_s
},
{
gid,
gid_s
}
};
S3PutProperties putProperties =
{
// application/x-lz4 does not officially exist
"application/x-lz4", // contentType
NULL, // md5
NULL, // cacheControl
NULL, // contentDispositionFilename
NULL, // contentEncoding
-1, // expires
0, // cannedAcl
sizeof(metadata) / sizeof(S3NameValue), // metaDataCount
metadata, // S3NameValue *metaData
0, // useServerSideEncryption
};
do {
// Uploading to object store
if (chunk_id == -1) {
CT_TRACE("start copy of "LPU64" bytes from '%s' to '%s'",
length, src, dst);
}
// size of the current chunk, limited by chunk_size
long long unsigned int chunk;
if (length - write_total > chunk_size) {
// upper bound is the chunk_size
chunk = chunk_size;
}
else {
// limited by the file
chunk = length - write_total;
}
chunk_id = file_offset / chunk_size;
put_object_callback_data data;
data.buffer_offset = 0;
double before_lustre_read = ct_now();
pread(src_fd, uncompress_buf, chunk, file_offset);
CT_TRACE("Reading a chunk from %s of %llu bytes offset %llu from lustre took %fs",
src, chunk, file_offset, ct_now() - before_lustre_read);
double before_compression = ct_now();
int compressed_size = LZ4_compress_default(uncompress_buf, compress_buf, chunk, compression_bound);
CT_TRACE("Compressing a chunk from %s took %fs and the compressed size is %i bytes",
src, ct_now() - before_compression, compressed_size);
if (compressed_size <= 0) {
rc = -1;
CT_ERROR(rc, "Compression error");
goto out;
}
data.contentLength = compressed_size;
data.buffer = compress_buf;
S3PutObjectHandler putObjectHandler =
{
putResponseHandler,
&putObjectDataCallback
};
char dst_chunk_s[S3_MAX_KEY_SIZE];
snprintf(dst_chunk_s, sizeof(dst_chunk_s), "%s.%i", dst, chunk_id);
char bucket_name[S3_MAX_BUCKET_NAME_SIZE];
getBucketName(sizeof(bucket_name), bucket_name, dst_chunk_s);
// Get a local copy of the general bucketContext than overwrite the
// pointer to the bucket_name
S3BucketContext localbucketContext;
memcpy(&localbucketContext, &bucketContext, sizeof(S3BucketContext));
localbucketContext.bucketName = bucket_name;
double before_s3_put = ct_now();
int retry_count = RETRYCOUNT;
do {
S3_put_object(&localbucketContext, dst_chunk_s, compressed_size, &putProperties, NULL, &putObjectHandler, &data);
} while (S3_status_is_retryable(data.status) && should_retry(&retry_count));
CT_TRACE("S3 put of %s took %fs",
dst_chunk_s, ct_now() - before_s3_put);
if (data.status != S3StatusOK) {
rc = -EIO;
CT_ERROR(rc, "S3Error %s", S3_get_status_name(data.status));
goto out;
}
he.offset = file_offset;
he.length = chunk;
now = time(NULL);
if (now >= last_report_time + ct_opt.o_report_int) {
last_report_time = now;
CT_TRACE("sending progress report for archiving %s", src);
rc = llapi_hsm_action_progress(hcp, &he, length, 0);
if (rc < 0) {
/* Action has been canceled or something wrong
* is happening. Stop copying data. */
CT_ERROR(rc, "progress ioctl for copy '%s'->'%s' failed",
src, dst);
goto out;
}
}
write_total += chunk;
file_offset += chunk;
} while (file_offset < length);
rc = 0;
// We need to delete every chunk of higher chunk_id if they
// exists, this can happen if the new file is smaller
// TODO only delete objects if this is a dirty write
chunk_id += 1;
do {
char dst_s[S3_MAX_KEY_SIZE];
int retry_count;
snprintf(dst_s, sizeof(dst_s), "%s.%i", dst, chunk_id);
get_object_callback_data head_data;
get_object_callback_data delete_data;
char bucket_name[S3_MAX_BUCKET_NAME_SIZE];
getBucketName(sizeof(bucket_name), bucket_name, dst_s);
// Get a local copy of the general bucketContext than overwrite the
// pointer to the bucket_name
S3BucketContext localbucketContext;
memcpy(&localbucketContext, &bucketContext, sizeof(S3BucketContext));
localbucketContext.bucketName = bucket_name;
CT_TRACE("Checking if chunk %i exists", chunk_id);
retry_count = RETRYCOUNT;
do {
S3_head_object(&localbucketContext, dst_s, NULL, &headResponseHandler, &head_data);
} while (S3_status_is_retryable(head_data.status) && should_retry(&retry_count));
if (head_data.status == S3StatusHttpErrorNotFound) {
// Object do not exist, this mean we stop deleting chunks
CT_TRACE("Chunk %i do not exists", chunk_id);
break;
}
if (head_data.status != S3StatusOK) {
rc = -EIO;
CT_ERROR(rc, "S3Error %s", S3_get_status_name(head_data.status));
goto out;
}
CT_TRACE("Deleting chunk %i", chunk_id);
retry_count = RETRYCOUNT;
do {
S3_delete_object(&localbucketContext, dst_s, NULL, &deleteResponseHandler, &delete_data);
} while (S3_status_is_retryable(delete_data.status) && should_retry(&retry_count));
if (delete_data.status != S3StatusOK) {
rc = -EIO;
CT_ERROR(rc, "S3Error %s", S3_get_status_name(delete_data.status));
goto out;
}
chunk_id++;
} while (true);
out:
if (uncompress_buf != NULL)
free(uncompress_buf);
if (compress_buf != NULL)
free(compress_buf);
CT_TRACE("copied "LPU64" bytes in %f seconds",
length, ct_now() - start_ct_now);
return rc;
}
int64_t lzbench_lz4_compress(char *inbuf, size_t insize, char *outbuf, size_t outsize, size_t level, size_t, char*)
{
return LZ4_compress_default(inbuf, outbuf, insize, outsize);
}
/*
* main
*/
int main(void) {
/* Introduction */
// Below we will have a Compression and Decompression section to demonstrate.
// There are a few important notes before we start:
// 1) The return codes of LZ4_ functions are important.
// Read lz4.h if you're unsure what a given code means.
// 2) LZ4 uses char* pointers in all LZ4_ functions.
// This is baked into the API and probably not going to change.
// If your program uses pointers that are unsigned char*, void*, or otherwise different,
// you may need to do some casting or set the right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
/* Compression */
// We'll store some text into a variable pointed to by *src to be compressed later.
const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
// The compression function needs to know how many bytes exist. Since we're using a string, we can use strlen() + 1 (for \0).
const int src_size = (int)(strlen(src) + 1);
// LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
const int max_dst_size = LZ4_compressBound(src_size);
// We will use that size for our destination boundary when allocating space.
char* compressed_data = malloc(max_dst_size);
if (compressed_data == NULL)
run_screaming("Failed to allocate memory for *compressed_data.", 1);
// That's all the information and preparation LZ4 needs to compress *src into *compressed_data.
// Invoke LZ4_compress_default now with our size values and pointers to our memory locations.
// Save the return value for error checking.
const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
// Check return_value to determine what happened.
if (compressed_data_size < 0)
run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", compressed_data_size);
if (compressed_data_size == 0)
run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1);
if (compressed_data_size > 0)
printf("We successfully compressed some data!\n");
// Not only does a positive return_value mean success, the value returned == the number of bytes required.
// You can use this to realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept.
compressed_data = (char *)realloc(compressed_data, compressed_data_size);
if (compressed_data == NULL)
run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1);
/* Decompression */
// Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a
// *new_src location of size src_size since we know that value.
char* const regen_buffer = malloc(src_size);
if (regen_buffer == NULL)
run_screaming("Failed to allocate memory for *regen_buffer.", 1);
// The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is,
// where the regen_buffer memory location is, and how large regen_buffer (uncompressed) output will be.
// Again, save the return_value.
const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressed_data_size, src_size);
free(compressed_data); /* no longer useful */
if (decompressed_size < 0)
run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", decompressed_size);
if (decompressed_size == 0)
run_screaming("I'm not sure this function can ever return 0. Documentation in lz4.h doesn't indicate so.", 1);
if (decompressed_size > 0)
printf("We successfully decompressed some data!\n");
// Not only does a positive return value mean success,
// value returned == number of bytes regenerated from compressed_data stream.
/* Validation */
// We should be able to compare our original *src with our *new_src and be byte-for-byte identical.
if (memcmp(src, regen_buffer, src_size) != 0)
run_screaming("Validation failed. *src and *new_src are not identical.", 1);
printf("Validation done. The string we ended up with is:\n%s\n", regen_buffer);
return 0;
}
