IOBuf IOBuf::cloneAsValue() const {
auto tmp = cloneOneAsValue();
for (IOBuf* current = next_; current != this; current = current->next_) {
tmp.prependChain(current->cloneOne());
}
return tmp;
}
static status
prependChainTable(ChainTable ct, Any name, Any value)
{ Chain ch;
if ( (ch = getMemberHashTable((HashTable) ct, name)) )
prependChain(ch, value);
else
appendHashTable((HashTable) ct, name, newObject(ClassChain, value, EAV));
succeed;
}
static status
initialiseWinPrinter(WinPrinter prt, Name jobname)
{ if ( isDefault(jobname) )
jobname = CtoName("XPCE");
assign(prt, map_mode, DEFAULT);
assign(prt, device, DEFAULT);
assign(prt, job_name, jobname);
assign(prt, job, NIL);
assign(prt, resolution, DEFAULT);
assign(prt, origin, newObject(ClassPoint, EAV));
prt->ws_ref = alloc(sizeof(ws_printer));
memset(prt->ws_ref, 0, sizeof(ws_printer));
return prependChain(WinPrinters, prt);
}
void CompressionVarintTest::runSimpleTest(const DataHolder& dh) {
auto original = IOBuf::wrapBuffer(dh.data(uncompressedLength_));
auto compressed = codec_->compress(original.get());
auto breakPoint =
1UL +
Random::rand64(std::max(9UL, oneBasedMsbPos(uncompressedLength_)) / 9UL);
auto tinyBuf = IOBuf::copyBuffer(compressed->data(),
std::min(compressed->length(), breakPoint));
compressed->trimStart(breakPoint);
tinyBuf->prependChain(std::move(compressed));
compressed = std::move(tinyBuf);
auto uncompressed = codec_->uncompress(compressed.get());
EXPECT_EQ(uncompressedLength_, uncompressed->computeChainDataLength());
EXPECT_EQ(dh.hash(uncompressedLength_), hashIOBuf(uncompressed.get()));
}
std::unique_ptr<IOBuf> LZMA2Codec::doCompress(const IOBuf* data) {
lzma_ret rc;
lzma_stream stream = LZMA_STREAM_INIT;
rc = lzma_easy_encoder(&stream, level_, LZMA_CHECK_NONE);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_easy_encoder error: ", rc));
}
SCOPE_EXIT { lzma_end(&stream); };
uint64_t uncompressedLength = data->computeChainDataLength();
uint64_t maxCompressedLength = lzma_stream_buffer_bound(uncompressedLength);
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
auto out = addOutputBuffer(
&stream,
(maxCompressedLength <= maxSingleStepLength ?
maxCompressedLength :
defaultBufferLength));
if (encodeSize()) {
auto size = IOBuf::createCombined(kMaxVarintLength64);
encodeVarintToIOBuf(uncompressedLength, size.get());
size->appendChain(std::move(out));
out = std::move(size);
}
for (auto& range : *data) {
if (range.empty()) {
continue;
}
stream.next_in = const_cast<uint8_t*>(range.data());
stream.avail_in = range.size();
while (stream.avail_in != 0) {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_RUN);
if (rc != LZMA_OK) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code error: ", rc));
}
}
}
do {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = lzma_code(&stream, LZMA_FINISH);
} while (rc == LZMA_OK);
if (rc != LZMA_STREAM_END) {
throw std::runtime_error(folly::to<std::string>(
"LZMA2Codec: lzma_code ended with error: ", rc));
}
out->prev()->trimEnd(stream.avail_out);
return out;
}
std::unique_ptr<IOBuf> ZlibCodec::doCompress(const IOBuf* data) {
z_stream stream;
stream.zalloc = nullptr;
stream.zfree = nullptr;
stream.opaque = nullptr;
// Using deflateInit2() to support gzip. "The windowBits parameter is the
// base two logarithm of the maximum window size (...) The default value is
// 15 (...) Add 16 to windowBits to write a simple gzip header and trailer
// around the compressed data instead of a zlib wrapper. The gzip header
// will have no file name, no extra data, no comment, no modification time
// (set to zero), no header crc, and the operating system will be set to 255
// (unknown)."
int windowBits = 15 + (type() == CodecType::GZIP ? 16 : 0);
// All other parameters (method, memLevel, strategy) get default values from
// the zlib manual.
int rc = deflateInit2(&stream,
level_,
Z_DEFLATED,
windowBits,
/* memLevel */ 8,
Z_DEFAULT_STRATEGY);
if (rc != Z_OK) {
throw std::runtime_error(to<std::string>(
"ZlibCodec: deflateInit error: ", rc, ": ", stream.msg));
}
stream.next_in = stream.next_out = nullptr;
stream.avail_in = stream.avail_out = 0;
stream.total_in = stream.total_out = 0;
bool success = false;
SCOPE_EXIT {
int rc = deflateEnd(&stream);
// If we're here because of an exception, it's okay if some data
// got dropped.
CHECK(rc == Z_OK || (!success && rc == Z_DATA_ERROR))
<< rc << ": " << stream.msg;
};
uint64_t uncompressedLength = data->computeChainDataLength();
uint64_t maxCompressedLength = deflateBound(&stream, uncompressedLength);
// Max 64MiB in one go
constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
auto out = addOutputBuffer(
&stream,
(maxCompressedLength <= maxSingleStepLength ?
maxCompressedLength :
defaultBufferLength));
for (auto& range : *data) {
uint64_t remaining = range.size();
uint64_t written = 0;
while (remaining) {
uint32_t step = (remaining > maxSingleStepLength ?
maxSingleStepLength : remaining);
stream.next_in = const_cast<uint8_t*>(range.data() + written);
stream.avail_in = step;
remaining -= step;
written += step;
while (stream.avail_in != 0) {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = deflate(&stream, Z_NO_FLUSH);
CHECK_EQ(rc, Z_OK) << stream.msg;
}
}
}
do {
if (stream.avail_out == 0) {
out->prependChain(addOutputBuffer(&stream, defaultBufferLength));
}
rc = deflate(&stream, Z_FINISH);
} while (rc == Z_OK);
CHECK_EQ(rc, Z_STREAM_END) << stream.msg;
out->prev()->trimEnd(stream.avail_out);
success = true; // we survived
return out;
}
Future<void> LengthFieldPrepender::write(
Context* ctx, std::unique_ptr<IOBuf> buf) {
int length = lengthAdjustment_ + buf->computeChainDataLength();
if (lengthIncludesLengthField_) {
length += lengthFieldLength_;
}
if (length < 0) {
throw std::runtime_error("Length field < 0");
}
auto len = IOBuf::create(lengthFieldLength_);
len->append(lengthFieldLength_);
folly::io::RWPrivateCursor c(len.get());
switch (lengthFieldLength_) {
case 1: {
if (length >= 256) {
throw std::runtime_error("length does not fit byte");
}
if (networkByteOrder_) {
c.writeBE((uint8_t)length);
} else {
c.writeLE((uint8_t)length);
}
break;
}
case 2: {
if (length >= 65536) {
throw std::runtime_error("length does not fit byte");
}
if (networkByteOrder_) {
c.writeBE((uint16_t)length);
} else {
c.writeLE((uint16_t)length);
}
break;
}
case 4: {
if (networkByteOrder_) {
c.writeBE((uint32_t)length);
} else {
c.writeLE((uint32_t)length);
}
break;
}
case 8: {
if (networkByteOrder_) {
c.writeBE((uint64_t)length);
} else {
c.writeLE((uint64_t)length);
}
break;
}
default: {
throw std::runtime_error("Invalid lengthFieldLength");
}
}
len->prependChain(std::move(buf));
return ctx->fireWrite(std::move(len));
}
