StatusWith<std::size_t> SnappyMessageCompressor::decompressData(ConstDataRange input,
DataRange output) {
size_t expectedLength = 0;
if (!snappy::GetUncompressedLength(input.data(), input.length(), &expectedLength) ||
expectedLength != output.length()) {
return {ErrorCodes::BadValue, "Compressed message was invalid or corrupted"};
}
if (!snappy::RawUncompress(input.data(), input.length(), const_cast<char*>(output.data()))) {
return Status{ErrorCodes::BadValue, "Compressed message was invalid or corrupted"};
}
counterHitDecompress(input.length(), output.length());
return output.length();
}
Status FTDCFileWriter::writeArchiveFileBuffer(ConstDataRange buf) {
_archiveStream.write(buf.data(), buf.length());
if (_archiveStream.fail()) {
return {
ErrorCodes::FileStreamFailed,
str::stream()
<< "Failed to write to archive file buffer for full-time diagnostic data capture: "
<< _archiveFile.generic_string()};
}
// Flush the stream explictly, this is preferred over "pubsetbuf(0,0)" which has implementation
// defined behavior of "before any I/O has occurred".
_archiveStream.flush();
if (_archiveStream.fail()) {
return {
ErrorCodes::FileStreamFailed,
str::stream()
<< "Failed to flush to archive file buffer for full-time diagnostic data capture: "
<< _archiveFile.generic_string()};
}
_size += buf.length();
return Status::OK();
}
void BinDataInfo::Functions::UUID::call(JSContext* cx, JS::CallArgs args) {
boost::optional<mongo::UUID> uuid;
if (args.length() == 0) {
uuid = mongo::UUID::gen();
} else {
uassert(ErrorCodes::BadValue, "UUID needs 0 or 1 arguments", args.length() == 1);
auto arg = args.get(0);
std::string str = ValueWriter(cx, arg).toString();
// For backward compatibility quietly accept and convert 32-character hex strings to
// BinData(3, ...) as used for the deprecated UUID v3 BSON type.
if (str.length() == 32) {
hexToBinData(cx, bdtUUID, arg, args.rval());
return;
}
uuid = uassertStatusOK(mongo::UUID::parse(str));
};
ConstDataRange cdr = uuid->toCDR();
std::string encoded = mongo::base64::encode(cdr.data(), cdr.length());
JS::AutoValueArray<2> newArgs(cx);
newArgs[0].setInt32(newUUID);
ValueReader(cx, newArgs[1]).fromStringData(encoded);
getScope(cx)->getProto<BinDataInfo>().newInstance(newArgs, args.rval());
}
BSONObj createBSONMetricChunkDocument(ConstDataRange buf, Date_t date) {
BSONObjBuilder builder;
builder.appendDate(kFTDCIdField, date);
builder.appendNumber(kFTDCTypeField, static_cast<int>(FTDCType::kMetricChunk));
builder.appendBinData(kFTDCDataField, buf.length(), BinDataType::BinDataGeneral, buf.data());
return builder.obj();
}
StatusWith<std::size_t> SnappyMessageCompressor::compressData(ConstDataRange input,
DataRange output) {
size_t outLength = output.length();
if (output.length() < getMaxCompressedSize(input.length())) {
return {ErrorCodes::BadValue, "Output too small for max size of compressed input"};
}
snappy::RawCompress(input.data(), input.length(), const_cast<char*>(output.data()), &outLength);
counterHitCompress(input.length(), outLength);
return {outLength};
}
Status FTDCFileWriter::writeArchiveFileBuffer(ConstDataRange buf) {
_archiveStream.write(buf.data(), buf.length());
if (_archiveStream.fail()) {
return {
ErrorCodes::FileStreamFailed,
str::stream()
<< "Failed to write to archive file buffer for full-time diagnostic data capture: "
<< _archiveFile.generic_string()};
}
_size += buf.length();
return Status::OK();
}
StatusWith<ConstDataRange> BlockCompressor::compress(ConstDataRange source) {
z_stream stream;
int level = Z_DEFAULT_COMPRESSION;
stream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(source.data()));
stream.avail_in = source.length();
// In compress.c in the zlib source, they recommend that the compression buffer be
// at least 0.1% larger + 12 bytes then the source length.
// We make the buffer 1% larger then the source length buffer to be on the safe side. If we are
// too small, deflate returns an error.
_buffer.resize(source.length() * 1.01 + 12);
stream.next_out = _buffer.data();
stream.avail_out = _buffer.size();
stream.zalloc = nullptr;
stream.zfree = nullptr;
stream.opaque = nullptr;
int err = deflateInit(&stream, level);
if (err != Z_OK) {
return {ErrorCodes::ZLibError, str::stream() << "deflateInit failed with " << err};
}
err = deflate(&stream, Z_FINISH);
if (err != Z_STREAM_END) {
(void)deflateEnd(&stream);
if (err != Z_OK) {
return {ErrorCodes::ZLibError, str::stream() << "deflate failed with " << err};
}
}
err = deflateEnd(&stream);
if (err != Z_OK) {
return {ErrorCodes::ZLibError, str::stream() << "deflateEnd failed with " << err};
}
return ConstDataRange(reinterpret_cast<char*>(_buffer.data()), stream.total_out);
}
Status FTDCFileWriter::writeInterimFileBuffer(ConstDataRange buf) {
// Fixed size interim stream
std::ofstream interimStream;
// Disable file buffering
interimStream.rdbuf()->pubsetbuf(0, 0);
// Open up a temporary interim file
interimStream.open(_interimTempFile.c_str(),
std::ios_base::out | std::ios_base::binary | std::ios_base::trunc);
if (!interimStream.is_open()) {
return Status(ErrorCodes::FileNotOpen,
"Failed to open interim file " + _interimTempFile.generic_string());
}
interimStream.write(buf.data(), buf.length());
if (interimStream.fail()) {
return {
ErrorCodes::FileStreamFailed,
str::stream()
<< "Failed to write to interim file buffer for full-time diagnostic data capture: "
<< _interimTempFile.generic_string()};
}
interimStream.close();
// Now that the temp interim file is closed, rename the temp interim file to the real one.
boost::system::error_code ec;
boost::filesystem::rename(_interimTempFile, _interimFile, ec);
if (ec) {
return Status(ErrorCodes::FileRenameFailed, ec.message());
}
_sizeInterim = buf.length();
return Status::OK();
}
StatusWith<ConstDataRange> BlockCompressor::uncompress(ConstDataRange source,
size_t uncompressedLength) {
z_stream stream;
stream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(source.data()));
stream.avail_in = source.length();
_buffer.resize(uncompressedLength);
stream.next_out = _buffer.data();
stream.avail_out = _buffer.size();
stream.zalloc = nullptr;
stream.zfree = nullptr;
stream.opaque = nullptr;
int err = inflateInit(&stream);
if (err != Z_OK) {
return {ErrorCodes::ZLibError, str::stream() << "inflateInit failed with " << err};
}
err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END) {
(void)inflateEnd(&stream);
if (err != Z_OK) {
return {ErrorCodes::ZLibError, str::stream() << "inflate failed with " << err};
}
}
err = inflateEnd(&stream);
if (err != Z_OK) {
return {ErrorCodes::ZLibError, str::stream() << "inflateEnd failed with " << err};
}
return ConstDataRange(reinterpret_cast<char*>(_buffer.data()), stream.total_out);
}
StatusWith<std::tuple<ConstDataRange, Date_t>> FTDCCompressor::getCompressedSamples() {
_uncompressedChunkBuffer.setlen(0);
// Append reference document - BSON Object
_uncompressedChunkBuffer.appendBuf(_referenceDoc.objdata(), _referenceDoc.objsize());
// Append count of metrics - uint32 little endian
_uncompressedChunkBuffer.appendNum(static_cast<std::uint32_t>(_metricsCount));
// Append count of samples - uint32 little endian
_uncompressedChunkBuffer.appendNum(static_cast<std::uint32_t>(_deltaCount));
if (_metricsCount != 0 && _deltaCount != 0) {
// On average, we do not need all 10 bytes for every sample, worst case, we grow the buffer
DataBuilder db(_metricsCount * _deltaCount * FTDCVarInt::kMaxSizeBytes64 / 2);
std::uint32_t zeroesCount = 0;
// For each set of samples for a particular metric,
// we think of it is simple array of 64-bit integers we try to compress into a byte array.
// This is done in three steps for each metric
// 1. Delta Compression
// - i.e., we store the difference between pairs of samples, not their absolute values
// - this is done in addSamples
// 2. Run Length Encoding of zeros
// - We find consecutive sets of zeros and represent them as a tuple of (0, count - 1).
// - Each memeber is stored as VarInt packed integer
// 3. Finally, for non-zero members, we store these as VarInt packed
//
// These byte arrays are added to a buffer which is then concatenated with other chunks and
// compressed with ZLIB.
for (std::uint32_t i = 0; i < _metricsCount; i++) {
for (std::uint32_t j = 0; j < _deltaCount; j++) {
std::uint64_t delta = _deltas[getArrayOffset(_maxDeltas, j, i)];
if (delta == 0) {
++zeroesCount;
continue;
}
// If we have a non-zero sample, then write out all the accumulated zero samples.
if (zeroesCount > 0) {
auto s1 = db.writeAndAdvance(FTDCVarInt(0));
if (!s1.isOK()) {
return s1;
}
auto s2 = db.writeAndAdvance(FTDCVarInt(zeroesCount - 1));
if (!s2.isOK()) {
return s2;
}
zeroesCount = 0;
}
auto s3 = db.writeAndAdvance(FTDCVarInt(delta));
if (!s3.isOK()) {
return s3;
}
}
// If we are on the last metric, and the previous loop ended in a zero, write out the
// RLE
// pair of zero information.
if ((i == (_metricsCount - 1)) && zeroesCount) {
auto s1 = db.writeAndAdvance(FTDCVarInt(0));
if (!s1.isOK()) {
return s1;
}
auto s2 = db.writeAndAdvance(FTDCVarInt(zeroesCount - 1));
if (!s2.isOK()) {
return s2;
}
}
}
// Append the entire compacted metric chunk into the uncompressed buffer
ConstDataRange cdr = db.getCursor();
_uncompressedChunkBuffer.appendBuf(cdr.data(), cdr.length());
}
auto swDest = _compressor.compress(
ConstDataRange(_uncompressedChunkBuffer.buf(), _uncompressedChunkBuffer.len()));
// The only way for compression to fail is if the buffer size calculations are wrong
if (!swDest.isOK()) {
return swDest.getStatus();
}
_compressedChunkBuffer.setlen(0);
_compressedChunkBuffer.appendNum(static_cast<std::uint32_t>(_uncompressedChunkBuffer.len()));
_compressedChunkBuffer.appendBuf(swDest.getValue().data(), swDest.getValue().length());
return std::tuple<ConstDataRange, Date_t>(
ConstDataRange(_compressedChunkBuffer.buf(),
static_cast<size_t>(_compressedChunkBuffer.len())),
_referenceDocDate);
}