// don't inline, to be friendly to js engine osr
void __attribute__ ((noinline)) doit(char *buffer, int size, int i) {
static char *buffer2 = NULL;
static char *buffer3 = NULL;
unsigned long maxCompressedSize = lzma_stream_buffer_bound(size);
if (!buffer2) buffer2 = (char*)malloc(maxCompressedSize);
if (!buffer3) buffer3 = (char*)malloc(size);
size_t compressedSize = 0;
int ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64, NULL, (const uint8_t*)buffer, size, (uint8_t*)buffer2, &compressedSize, maxCompressedSize);
assert(ret == LZMA_OK);
if (i == 0) printf("sizes: %d,%d\n", size, compressedSize);
lzma_stream strm = LZMA_STREAM_INIT;
ret = lzma_stream_decoder(&strm, UINT64_MAX, 0);
assert(ret == LZMA_OK);
strm.next_in = (const uint8_t*)buffer2;
strm.avail_in = compressedSize;
strm.next_out = (uint8_t*)buffer3;
strm.avail_out = size;
ret = lzma_code (&strm, LZMA_FINISH);
assert(ret == LZMA_OK || ret == LZMA_STREAM_END);
size_t decompressedSize = size - strm.avail_out;
assert(decompressedSize == size);
if (i == 0) assert(strcmp(buffer, buffer3) == 0);
}
void XzCompress::Encode(void)
{
direct = 0; // set direction needed by parent [Get|Send]Chars()
// get buffer
char chunk[1024];
char *buf = (char *)calloc(1, 1024);
char *chunkbuf = buf;
unsigned long chunklen;
unsigned long len = 0;
while((chunklen = GetChars(chunk, 1023))) {
memcpy(chunkbuf, chunk, chunklen);
len += chunklen;
if (chunklen < 1023)
break;
else buf = (char *)realloc(buf, len + 1024);
chunkbuf = buf+len;
}
zlen = (long)lzma_stream_buffer_bound(len);
char *zbuf = new char[zlen+1];
size_t zpos = 0;
if (len)
{
//printf("Doing compress\n");
switch (lzma_easy_buffer_encode(level | LZMA_PRESET_EXTREME, LZMA_CHECK_CRC64, NULL, (const uint8_t*)buf, (size_t)len, (uint8_t*)zbuf, &zpos, (size_t)zlen)) {
case LZMA_OK: SendChars(zbuf, zpos); break;
case LZMA_BUF_ERROR: fprintf(stderr, "ERROR: not enough room in the out buffer during compression.\n"); break;
case LZMA_UNSUPPORTED_CHECK: fprintf(stderr, "ERROR: unsupported_check error encountered during decompression.\n"); break;
case LZMA_OPTIONS_ERROR: fprintf(stderr, "ERROR: options error encountered during decompression.\n"); break;
case LZMA_MEM_ERROR: fprintf(stderr, "ERROR: not enough memory during compression.\n"); break;
case LZMA_DATA_ERROR: fprintf(stderr, "ERROR: corrupt data during compression.\n"); break;
case LZMA_PROG_ERROR: fprintf(stderr, "ERROR: program error encountered during decompression.\n"); break;
default: fprintf(stderr, "ERROR: an unknown error occured during compression.\n"); break;
}
}
else
{
fprintf(stderr, "ERROR: no buffer to compress\n");
}
delete [] zbuf;
free (buf);
}
Array<uint8_t> compress(RawArray<const uint8_t> data, int level, event_t event) {
thread_time_t time(compress_kind,event);
if (level<20) { // zlib
size_t dest_size = compressBound(data.size());
Array<uint8_t> compressed(CHECK_CAST_INT(dest_size),uninit);
int z = compress2(compressed.data(),&dest_size,(uint8_t*)data.data(),data.size(),level);
if (z!=Z_OK)
THROW(IOError,"zlib failure in compress_and_write: %s",zlib_error(z));
return compressed.slice_own(0,CHECK_CAST_INT(dest_size));
} else { // lzma
size_t dest_size = lzma_stream_buffer_bound(data.size());
Array<uint8_t> compressed(CHECK_CAST_INT(dest_size),uninit);
size_t pos = 0;
lzma_ret r = lzma_easy_buffer_encode(level-20,LZMA_CHECK_CRC64,0,data.data(),data.size(),compressed.data(),&pos,dest_size);
if (r!=LZMA_OK)
THROW(RuntimeError,"lzma compression error: %s (%d)",lzma_error(r),r);
return compressed.slice_own(0,CHECK_CAST_INT(pos));
}
}
std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) {
lzma_ret rc;
lzma_stream stream = LZMA_STREAM_INIT;
rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_easy_encoder error: ", rc));
}
SCOPE_EXIT { lzma_end(&stream); };
uint64_t uncompressedLength = data->computeChainDataLength();
uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength);
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
auto out = addOutputBuffer(
&stream,
(maxCompressedLength <= maxSingleStepLength ?
maxCompressedLength :
defaultBufferLength));
if (encodeSize()) {
auto size = IOBuf::createCombined(kMaxVarintLength64);
encodeVarintToIOBuf(uncompressedLength, size.get());
size->appendChain(std::move(out));
out = std::move(size);
}
for (auto& range : *data) {
if (range.empty()) {
continue;
}
stream.next_in = const_cast<uint8_t*>(range.data());
stream.avail_in = range.size();
while (stream.avail_in != 0) {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_RUN);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code error: ", rc));
}
}
}
do {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_FINISH);
} while (rc == LZMA_OK);
if (rc != LZMA_STREAM_END) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code ended with error: ", rc));
}
out->prev()->trimEnd(stream.avail_out);
return out;
}
size_t new_cmp(size_t in_sz)
{
uint32_t preset = COMPRESSION_LEVEL | (COMPRESSION_EXTREME ? LZMA_PRESET_EXTREME : 0);
blk_sz=in_sz;
return bound=lzma_stream_buffer_bound(in_sz);//BUG: this is wrong..
}
