/**
* Round trips the pipeline through serialization by calling serialize(), then Pipeline::parse().
* fasserts if it fails to parse after being serialized.
*/
boost::intrusive_ptr<Pipeline> reparsePipeline(
const boost::intrusive_ptr<Pipeline>& pipeline,
const AggregationRequest& request,
const boost::intrusive_ptr<ExpressionContext>& expCtx) {
auto serialized = pipeline->serialize();
// Convert vector<Value> to vector<BSONObj>.
std::vector<BSONObj> parseableSerialization;
parseableSerialization.reserve(serialized.size());
for (auto&& serializedStage : serialized) {
invariant(serializedStage.getType() == BSONType::Object);
parseableSerialization.push_back(serializedStage.getDocument().toBson());
}
auto reparsedPipeline = Pipeline::parse(parseableSerialization, expCtx);
if (!reparsedPipeline.isOK()) {
error() << "Aggregation command did not round trip through parsing and serialization "
"correctly. Input pipeline: "
<< Value(request.getPipeline()).toString()
<< ", serialized pipeline: " << Value(serialized).toString();
fassertFailedWithStatusNoTrace(40175, reparsedPipeline.getStatus());
}
reparsedPipeline.getValue()->injectExpressionContext(expCtx);
reparsedPipeline.getValue()->optimizePipeline();
return reparsedPipeline.getValue();
}
int index_collator_customize(WT_COLLATOR *coll,
WT_SESSION *s,
const char *uri,
WT_CONFIG_ITEM *metadata,
WT_COLLATOR **collp) {
try {
invariant(metadata->type == WT_CONFIG_ITEM::WT_CONFIG_ITEM_STRUCT);
WiredTigerConfigParser config(*metadata);
WT_CONFIG_ITEM version;
if (config.get("formatVersion", &version) != 0) {
fassertFailedWithStatusNoTrace(28580,
Status(ErrorCodes::UnsupportedFormat, str::stream()
<< "Found an index from an unsupported RC version."
<< " Please restart with --repair to fix."));
}
WT_CONFIG_ITEM infoObj;
const int infoObjRet = config.get("infoObj", &infoObj);
if (infoObjRet != 0) {
// This must be from a future version from when we removed infoObj. Print something
// reasonable.
infoObj.str = "(unknown)";
infoObj.len = strlen(infoObj.str);
}
if (version.type != WT_CONFIG_ITEM::WT_CONFIG_ITEM_NUM
|| version.val < kMinimumIndexVersion || version.val > kMaximumIndexVersion) {
fassertFailedWithStatusNoTrace(28579,
Status(ErrorCodes::UnsupportedFormat, str::stream()
<< "Index " << infoObj.str
<< " has unsupported formatVersion " << version.str
<< ". Please restart with --repair to fix."));
}
invariant(infoObjRet == 0);
IndexDescriptor desc(0, "unknown", fromjson(std::string(infoObj.str, infoObj.len)));
*collp = new WiredTigerIndexCollator(Ordering::make(desc.keyPattern()));
return 0;
}
catch (...) {
std::terminate();
}
}
void ReplicationRecoveryImpl::_truncateOplogTo(OperationContext* opCtx,
Timestamp truncateTimestamp) {
Timer timer;
const NamespaceString oplogNss(NamespaceString::kRsOplogNamespace);
AutoGetDb autoDb(opCtx, oplogNss.db(), MODE_IX);
Lock::CollectionLock oplogCollectionLoc(opCtx->lockState(), oplogNss.ns(), MODE_X);
Collection* oplogCollection = autoDb.getDb()->getCollection(opCtx, oplogNss);
if (!oplogCollection) {
fassertFailedWithStatusNoTrace(
34418,
Status(ErrorCodes::NamespaceNotFound,
str::stream() << "Can't find " << NamespaceString::kRsOplogNamespace.ns()));
}
// Scan through oplog in reverse, from latest entry to first, to find the truncateTimestamp.
RecordId oldestIDToDelete; // Non-null if there is something to delete.
auto oplogRs = oplogCollection->getRecordStore();
auto oplogReverseCursor = oplogRs->getCursor(opCtx, /*forward=*/false);
size_t count = 0;
while (auto next = oplogReverseCursor->next()) {
const BSONObj entry = next->data.releaseToBson();
const RecordId id = next->id;
count++;
const auto tsElem = entry["ts"];
if (count == 1) {
if (tsElem.eoo())
LOG(2) << "Oplog tail entry: " << redact(entry);
else
LOG(2) << "Oplog tail entry ts field: " << tsElem;
}
if (tsElem.timestamp() < truncateTimestamp) {
// If count == 1, that means that we have nothing to delete because everything in the
// oplog is < truncateTimestamp.
if (count != 1) {
invariant(!oldestIDToDelete.isNull());
oplogCollection->cappedTruncateAfter(opCtx, oldestIDToDelete, /*inclusive=*/true);
}
log() << "Replication recovery oplog truncation finished in: " << timer.millis()
<< "ms";
return;
}
oldestIDToDelete = id;
}
severe() << "Reached end of oplog looking for oplog entry before " << truncateTimestamp.toBSON()
<< " but couldn't find any after looking through " << count << " entries.";
fassertFailedNoTrace(40296);
}
WiredTigerRecordStore::WiredTigerRecordStore(OperationContext* ctx,
StringData ns,
StringData uri,
bool isCapped,
int64_t cappedMaxSize,
int64_t cappedMaxDocs,
CappedDocumentDeleteCallback* cappedDeleteCallback,
WiredTigerSizeStorer* sizeStorer)
: RecordStore(ns),
_uri(uri.toString()),
_tableId(WiredTigerSession::genTableId()),
_isCapped(isCapped),
_isOplog(NamespaceString::oplog(ns)),
_cappedMaxSize(cappedMaxSize),
_cappedMaxSizeSlack(std::min(cappedMaxSize / 10, int64_t(16 * 1024 * 1024))),
_cappedMaxDocs(cappedMaxDocs),
_cappedSleep(0),
_cappedSleepMS(0),
_cappedDeleteCallback(cappedDeleteCallback),
_cappedDeleteCheckCount(0),
_useOplogHack(shouldUseOplogHack(ctx, _uri)),
_sizeStorer(sizeStorer),
_sizeStorerCounter(0),
_shuttingDown(false) {
Status versionStatus = WiredTigerUtil::checkApplicationMetadataFormatVersion(
ctx, uri, kMinimumRecordStoreVersion, kMaximumRecordStoreVersion);
if (!versionStatus.isOK()) {
fassertFailedWithStatusNoTrace(28548, versionStatus);
}
if (_isCapped) {
invariant(_cappedMaxSize > 0);
invariant(_cappedMaxDocs == -1 || _cappedMaxDocs > 0);
} else {
invariant(_cappedMaxSize == -1);
invariant(_cappedMaxDocs == -1);
}
// Find the largest RecordId currently in use and estimate the number of records.
Cursor cursor(ctx, *this, /*forward=*/false);
if (auto record = cursor.next()) {
int64_t max = _makeKey(record->id);
_oplog_highestSeen = record->id;
_nextIdNum.store(1 + max);
if (_sizeStorer) {
long long numRecords;
long long dataSize;
_sizeStorer->loadFromCache(uri, &numRecords, &dataSize);
_numRecords.store(numRecords);
_dataSize.store(dataSize);
_sizeStorer->onCreate(this, numRecords, dataSize);
} else {
LOG(1) << "Doing scan of collection " << ns << " to get size and count info";
_numRecords.store(0);
_dataSize.store(0);
do {
_numRecords.fetchAndAdd(1);
_dataSize.fetchAndAdd(record->data.size());
} while ((record = cursor.next()));
}
} else {
_dataSize.store(0);
_numRecords.store(0);
// Need to start at 1 so we are always higher than RecordId::min()
_nextIdNum.store(1);
if (sizeStorer)
_sizeStorer->onCreate(this, 0, 0);
}
_hasBackgroundThread = WiredTigerKVEngine::initRsOplogBackgroundThread(ns);
}
