std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {
std::unique_ptr<IOBuf> clone;
if (data->isChained()) {
// LZ4 doesn't support streaming, so we have to coalesce
clone = data->clone();
clone->coalesce();
data = clone.get();
}
uint32_t extraSize = encodeSize() ? kMaxVarintLength64 : 0;
auto out = IOBuf::create(extraSize + LZ4_compressBound(data->length()));
if (encodeSize()) {
encodeVarintToIOBuf(data->length(), out.get());
}
int n;
if (highCompression_) {
n = LZ4_compressHC(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
} else {
n = LZ4_compress(reinterpret_cast<const char*>(data->data()),
reinterpret_cast<char*>(out->writableTail()),
data->length());
}
CHECK_GE(n, 0);
CHECK_LE(n, out->capacity());
out->append(n);
return out;
}
void addAttribute (char * name, char * value) {
int id = getTagId (name); // create the entry if necessary
if (m_dumpTable) {
encodeSize (id+1);
encodeString (value);
}
}
void open (char * t, bool selfClosing) {
int id = getTagId (t); // create the entry if necessary
if (m_dumpTable) {
encodeChar (selfClosing ? 2 : 1);
encodeSize (id);
}
}
void dumpTable () {
Tag * tag = m_tags;
encodeSize (m_nbTags);
while (tag != NULL) {
encodeString (tag->m_name);
tag = tag->m_next;
}
}
std::unique_ptr<IOBuf> LZ4Codec::doUncompress(
const IOBuf* data,
uint64_t uncompressedLength) {
std::unique_ptr<IOBuf> clone;
if (data->isChained()) {
// LZ4 doesn't support streaming, so we have to coalesce
clone = data->clone();
clone->coalesce();
data = clone.get();
}
folly::io::Cursor cursor(data);
uint64_t actualUncompressedLength;
if (encodeSize()) {
actualUncompressedLength = decodeVarintFromCursor(cursor);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("LZ4Codec: invalid uncompressed length");
}
} else {
actualUncompressedLength = uncompressedLength;
if (actualUncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH ||
actualUncompressedLength > maxUncompressedLength()) {
throw std::runtime_error("LZ4Codec: invalid uncompressed length");
}
}
auto sp = StringPiece{cursor.peekBytes()};
auto out = IOBuf::create(actualUncompressedLength);
int n = LZ4_decompress_safe(
sp.data(),
reinterpret_cast<char*>(out->writableTail()),
sp.size(),
actualUncompressedLength);
if (n < 0 || uint64_t(n) != actualUncompressedLength) {
throw std::runtime_error(to<std::string>(
"LZ4 decompression returned invalid value ", n));
}
out->append(actualUncompressedLength);
return out;
}
std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data,
uint64_t uncompressedLength) {
lzma_ret rc;
lzma_stream stream = LZMA_STREAM_INIT;
rc = lzma_auto_decoder(&stream, std::numeric_limits<uint64_t>::max(), 0);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_auto_decoder error: ", rc));
}
SCOPE_EXIT { lzma_end(&stream); };
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
folly::io::Cursor cursor(data);
uint64_t actualUncompressedLength;
if (encodeSize()) {
actualUncompressedLength = decodeVarintFromCursor(cursor);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("LZMA2Codec: invalid uncompressed length");
}
} else {
actualUncompressedLength = uncompressedLength;
DCHECK_NE(actualUncompressedLength, UNKNOWN_UNCOMPRESSED_LENGTH);
}
auto out = addOutputBuffer(
&stream,
(actualUncompressedLength <= maxSingleStepLength ?
actualUncompressedLength :
defaultBufferLength));
bool streamEnd = false;
auto buf = cursor.peekBytes();
while (!buf.empty()) {
stream.next_in = const_cast<uint8_t*>(buf.data());
stream.avail_in = buf.size();
while (stream.avail_in != 0) {
if (streamEnd) {
throw std::runtime_error(to<std::string>(
"LZMA2Codec: junk after end of data"));
}
streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
}
cursor.skip(buf.size());
buf = cursor.peekBytes();
}
while (!streamEnd) {
streamEnd = doInflate(&stream, out.get(), defaultBufferLength);
}
out->prev()->trimEnd(stream.avail_out);
if (actualUncompressedLength != stream.total_out) {
throw std::runtime_error(to<std::string>(
"LZMA2Codec: invalid uncompressed length"));
}
return out;
}
std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) {
lzma_ret rc;
lzma_stream stream = LZMA_STREAM_INIT;
rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_easy_encoder error: ", rc));
}
SCOPE_EXIT { lzma_end(&stream); };
uint64_t uncompressedLength = data->computeChainDataLength();
uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength);
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
auto out = addOutputBuffer(
&stream,
(maxCompressedLength <= maxSingleStepLength ?
maxCompressedLength :
defaultBufferLength));
if (encodeSize()) {
auto size = IOBuf::createCombined(kMaxVarintLength64);
encodeVarintToIOBuf(uncompressedLength, size.get());
size->appendChain(std::move(out));
out = std::move(size);
}
for (auto& range : *data) {
if (range.empty()) {
continue;
}
stream.next_in = const_cast<uint8_t*>(range.data());
stream.avail_in = range.size();
while (stream.avail_in != 0) {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_RUN);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code error: ", rc));
}
}
}
do {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_FINISH);
} while (rc == LZMA_OK);
if (rc != LZMA_STREAM_END) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code ended with error: ", rc));
}
out->prev()->trimEnd(stream.avail_out);
return out;
}
bool LZMA2Codec::doNeedsUncompressedLength() const {
return !encodeSize();
}