// find all oplog entries for a given OID in the oplog.refs collection and apply them
// TODO this should be a range query on oplog.refs where _id.oid == oid and applyOps to
// each entry found. The locking of the query interleaved with the locking in the applyOps
// did not work, so it a sequence of point queries.
// TODO verify that the query plan is a indexed lookup.
// TODO verify that the query plan does not fetch too many docs and then only process one of them.
void applyRefOp(BSONObj entry) {
OID oid = entry["ref"].OID();
LOG(3) << "apply ref " << entry << " oid " << oid << endl;
long long seq = 0; // note that 0 is smaller than any of the seq numbers
while (1) {
BSONObj entry;
{
LOCK_REASON(lockReason, "repl: finding oplog.refs entry to apply");
Client::ReadContext ctx(rsOplogRefs, lockReason);
// TODO: Should this be using rsOplogRefsDetails, verifying non-null?
Collection *cl = getCollection(rsOplogRefs);
if (cl == NULL || !cl->findOne(BSON("_id" << BSON("$gt" << BSON("oid" << oid << "seq" << seq))), entry, true)) {
break;
}
}
BSONElement e = entry.getFieldDotted("_id.seq");
seq = e.Long();
BSONElement eOID = entry.getFieldDotted("_id.oid");
if (oid != eOID.OID()) {
break;
}
LOG(3) << "apply " << entry << " seq=" << seq << endl;
applyOps(entry["ops"].Array());
}
}
/* static */ KVPrefix KVPrefix::fromBSONElement(const BSONElement value) {
if (value.eoo()) {
return kNotPrefixed;
}
return KVPrefix(static_cast<int64_t>(value.Long()));
}
StatusWith<CursorResponse> CursorResponse::parseFromBSON(const BSONObj& cmdResponse) {
Status cmdStatus = getStatusFromCommandResult(cmdResponse);
if (!cmdStatus.isOK()) {
return cmdStatus;
}
std::string fullns;
BSONObj batchObj;
CursorId cursorId;
BSONElement cursorElt = cmdResponse[kCursorField];
if (cursorElt.type() != BSONType::Object) {
return {ErrorCodes::TypeMismatch,
str::stream() << "Field '" << kCursorField
<< "' must be a nested object in: " << cmdResponse};
}
BSONObj cursorObj = cursorElt.Obj();
BSONElement idElt = cursorObj[kIdField];
if (idElt.type() != BSONType::NumberLong) {
return {ErrorCodes::TypeMismatch,
str::stream() << "Field '" << kIdField
<< "' must be of type long in: " << cmdResponse};
}
cursorId = idElt.Long();
BSONElement nsElt = cursorObj[kNsField];
if (nsElt.type() != BSONType::String) {
return {ErrorCodes::TypeMismatch,
str::stream() << "Field '" << kNsField
<< "' must be of type string in: " << cmdResponse};
}
fullns = nsElt.String();
BSONElement batchElt = cursorObj[kBatchField];
if (batchElt.eoo()) {
batchElt = cursorObj[kBatchFieldInitial];
}
if (batchElt.type() != BSONType::Array) {
return {ErrorCodes::TypeMismatch,
str::stream() << "Must have array field '" << kBatchFieldInitial << "' or '"
<< kBatchField << "' in: " << cmdResponse};
}
batchObj = batchElt.Obj();
std::vector<BSONObj> batch;
for (BSONElement elt : batchObj) {
if (elt.type() != BSONType::Object) {
return {
ErrorCodes::BadValue,
str::stream() << "getMore response batch contains a non-object element: " << elt};
}
batch.push_back(elt.Obj().getOwned());
}
return {{NamespaceString(fullns), cursorId, batch}};
}
void run() {
BSONObj spec( BSON("key" << BSON( "a" << "hashed" ) ));
BSONObj nullObj = BSON( "a" << BSONNULL );
// Call getKeys on the nullObj.
BSONObjSet nullFieldKeySet;
ExpressionKeysPrivate::getHashKeys(nullObj, "a", 0, 0, false, &nullFieldKeySet);
BSONElement nullFieldFromKey = nullFieldKeySet.begin()->firstElement();
ASSERT_EQUALS( ExpressionKeysPrivate::makeSingleHashKey( nullObj.firstElement(), 0, 0 ),
nullFieldFromKey.Long() );
BSONObj missingField = IndexLegacy::getMissingField(NULL,spec);
ASSERT_EQUALS( NumberLong, missingField.firstElement().type() );
ASSERT_EQUALS( nullFieldFromKey, missingField.firstElement());
}
void run() {
BSONObj spec( BSON("key" << BSON( "a" << "hashed" ) << "seed" << 0x5eed ));
BSONObj nullObj = BSON( "a" << BSONNULL );
BSONObjSet nullFieldKeySet;
ExpressionKeysPrivate::getHashKeys(nullObj, "a", 0x5eed, 0, false, &nullFieldKeySet);
BSONElement nullFieldFromKey = nullFieldKeySet.begin()->firstElement();
ASSERT_EQUALS( ExpressionKeysPrivate::makeSingleHashKey( nullObj.firstElement(), 0x5eed, 0 ),
nullFieldFromKey.Long() );
// Ensure that getMissingField recognizes that the seed is different (and returns
// the right key).
BSONObj missingField = IndexLegacy::getMissingField(NULL,spec);
ASSERT_EQUALS( NumberLong, missingField.firstElement().type());
ASSERT_EQUALS( nullFieldFromKey, missingField.firstElement());
}
SafeNum::SafeNum(const BSONElement& element) {
switch (element.type()) {
case NumberInt:
_type = NumberInt;
_value.int32Val = element.Int();
break;
case NumberLong:
_type = NumberLong;
_value.int64Val = element.Long();
break;
case NumberDouble:
_type = NumberDouble;
_value.doubleVal = element.Double();
break;
default:
_type = EOO;
}
}
void rollbackRefOp(BSONObj entry) {
OID oid = entry["ref"].OID();
LOG(3) << "rollback ref " << entry << " oid " << oid << endl;
long long seq = LLONG_MAX;
while (1) {
BSONObj currEntry;
{
LOCK_REASON(lockReason, "repl: rolling back entry from oplog.refs");
Client::ReadContext ctx(rsOplogRefs, lockReason);
verify(rsOplogRefsDetails != NULL);
shared_ptr<Cursor> c(
Cursor::make(
rsOplogRefsDetails,
rsOplogRefsDetails->getPKIndex(),
KeyPattern::toKeyFormat(BSON( "_id" << BSON("oid" << oid << "seq" << seq))), // right endpoint
KeyPattern::toKeyFormat(BSON( "_id" << BSON("oid" << oid << "seq" << 0))), // left endpoint
false,
-1 // direction
)
);
if (c->ok()) {
currEntry = c->current().copy();
}
else {
break;
}
}
BSONElement e = currEntry.getFieldDotted("_id.seq");
seq = e.Long();
BSONElement eOID = currEntry.getFieldDotted("_id.oid");
if (oid != eOID.OID()) {
break;
}
LOG(3) << "apply " << currEntry << " seq=" << seq << endl;
rollbackOps(currEntry["ops"].Array());
// decrement seq so next query gets the next value
seq--;
}
}
// find all oplog entries for a given OID in the oplog.refs collection and apply them
// TODO this should be a range query on oplog.refs where _id.oid == oid and applyOps to
// each entry found. The locking of the query interleaved with the locking in the applyOps
// did not work, so it a sequence of point queries.
// TODO verify that the query plan is a indexed lookup.
// TODO verify that the query plan does not fetch too many docs and then only process one of them.
void applyRefOp(BSONObj entry) {
OID oid = entry["ref"].OID();
LOG(3) << "apply ref " << entry << " oid " << oid << endl;
long long seq = 0; // note that 0 is smaller than any of the seq numbers
while (1) {
BSONObj entry;
{
Client::ReadContext ctx(rsOplogRefs);
// TODO: Should this be using rsOplogRefsDetails, verifying non-null?
NamespaceDetails *d = nsdetails(rsOplogRefs);
if (d == NULL || !d->findOne(BSON("_id" << BSON("$gt" << BSON("oid" << oid << "seq" << seq))), entry, true)) {
break;
}
}
BSONElement e = entry.getFieldDotted("_id.seq");
seq = e.Long();
BSONElement eOID = entry.getFieldDotted("_id.oid");
if (oid != eOID.OID()) {
break;
}
LOG(3) << "apply " << entry << " seq=" << seq << endl;
applyOps(entry["ops"].Array());
}
}
StatusWith<CursorResponse> CursorResponse::parseFromBSON(const BSONObj& cmdResponse) {
Status cmdStatus = getStatusFromCommandResult(cmdResponse);
if (!cmdStatus.isOK()) {
if (ErrorCodes::isStaleShardVersionError(cmdStatus.code())) {
auto vWanted = ChunkVersion::fromBSON(cmdResponse, "vWanted");
auto vReceived = ChunkVersion::fromBSON(cmdResponse, "vReceived");
if (!vWanted.hasEqualEpoch(vReceived)) {
return Status(ErrorCodes::StaleEpoch, cmdStatus.reason());
}
}
return cmdStatus;
}
std::string fullns;
BSONObj batchObj;
CursorId cursorId;
BSONElement cursorElt = cmdResponse[kCursorField];
if (cursorElt.type() != BSONType::Object) {
return {ErrorCodes::TypeMismatch,
str::stream() << "Field '" << kCursorField << "' must be a nested object in: "
<< cmdResponse};
}
BSONObj cursorObj = cursorElt.Obj();
BSONElement idElt = cursorObj[kIdField];
if (idElt.type() != BSONType::NumberLong) {
return {
ErrorCodes::TypeMismatch,
str::stream() << "Field '" << kIdField << "' must be of type long in: " << cmdResponse};
}
cursorId = idElt.Long();
BSONElement nsElt = cursorObj[kNsField];
if (nsElt.type() != BSONType::String) {
return {ErrorCodes::TypeMismatch,
str::stream() << "Field '" << kNsField << "' must be of type string in: "
<< cmdResponse};
}
fullns = nsElt.String();
BSONElement batchElt = cursorObj[kBatchField];
if (batchElt.eoo()) {
batchElt = cursorObj[kBatchFieldInitial];
}
if (batchElt.type() != BSONType::Array) {
return {ErrorCodes::TypeMismatch,
str::stream() << "Must have array field '" << kBatchFieldInitial << "' or '"
<< kBatchField
<< "' in: "
<< cmdResponse};
}
batchObj = batchElt.Obj();
std::vector<BSONObj> batch;
for (BSONElement elt : batchObj) {
if (elt.type() != BSONType::Object) {
return {ErrorCodes::BadValue,
str::stream() << "getMore response batch contains a non-object element: "
<< elt};
}
batch.push_back(elt.Obj());
}
for (auto& doc : batch) {
doc.shareOwnershipWith(cmdResponse);
}
auto latestOplogTimestampElem = cmdResponse[kInternalLatestOplogTimestampField];
if (latestOplogTimestampElem && latestOplogTimestampElem.type() != BSONType::bsonTimestamp) {
return {
ErrorCodes::BadValue,
str::stream()
<< "invalid _internalLatestOplogTimestamp format; expected timestamp but found: "
<< latestOplogTimestampElem.type()};
}
auto writeConcernError = cmdResponse["writeConcernError"];
if (writeConcernError && writeConcernError.type() != BSONType::Object) {
return {ErrorCodes::BadValue,
str::stream() << "invalid writeConcernError format; expected object but found: "
<< writeConcernError.type()};
}
return {{NamespaceString(fullns),
cursorId,
std::move(batch),
boost::none,
latestOplogTimestampElem ? latestOplogTimestampElem.timestamp()
: boost::optional<Timestamp>{},
writeConcernError ? writeConcernError.Obj().getOwned() : boost::optional<BSONObj>{}}};
}
boost::intrusive_ptr<DocumentSource> DocumentSourceMergeCursors::createFromBson(
BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx) {
if (elem.type() == BSONType::Object) {
// This is the modern serialization format. We de-serialize using the IDL.
auto ownedObj = elem.embeddedObject().getOwned();
auto armParams =
AsyncResultsMergerParams::parse(IDLParserErrorContext(kStageName), ownedObj);
return new DocumentSourceMergeCursors(
Grid::get(expCtx->opCtx)->getExecutorPool()->getArbitraryExecutor(),
std::move(armParams),
expCtx,
std::move(ownedObj));
}
// This is the old serialization format which can still be generated by mongos processes
// older than 4.0.
// TODO SERVER-34009 Remove support for this format.
uassert(17026,
"$mergeCursors stage expected either an array or an object as argument",
elem.type() == BSONType::Array);
const auto serializedRemotes = elem.Array();
uassert(50729,
"$mergeCursors stage expected array with at least one entry",
serializedRemotes.size() > 0);
boost::optional<NamespaceString> nss;
std::vector<RemoteCursor> remotes;
for (auto&& cursor : serializedRemotes) {
BSONElement nsElem;
BSONElement hostElem;
BSONElement idElem;
uassert(17027,
"$mergeCursors stage requires each cursor in array to be an object",
cursor.type() == BSONType::Object);
for (auto&& cursorElem : cursor.Obj()) {
const auto fieldName = cursorElem.fieldNameStringData();
if (fieldName == "ns"_sd) {
nsElem = cursorElem;
} else if (fieldName == "host"_sd) {
hostElem = cursorElem;
} else if (fieldName == "id"_sd) {
idElem = cursorElem;
} else {
uasserted(50730,
str::stream() << "Unrecognized option " << fieldName
<< " within cursor provided to $mergeCursors: "
<< cursor);
}
}
uassert(
50731,
"$mergeCursors stage requires \'ns\' field with type string for each cursor in array",
nsElem.type() == BSONType::String);
// We require each cursor to have the same namespace. This isn't a fundamental limit of the
// system, but needs to be true due to the implementation of AsyncResultsMerger, which
// tracks one namespace for all cursors.
uassert(50720,
"$mergeCursors requires each cursor to have the same namespace",
!nss || nss->ns() == nsElem.valueStringData());
nss = NamespaceString(nsElem.String());
uassert(
50721,
"$mergeCursors stage requires \'host\' field with type string for each cursor in array",
hostElem.type() == BSONType::String);
auto host = uassertStatusOK(HostAndPort::parse(hostElem.valueStringData()));
uassert(50722,
"$mergeCursors stage requires \'id\' field with type long for each cursor in array",
idElem.type() == BSONType::NumberLong);
auto cursorId = idElem.Long();
// We are assuming that none of the cursors have been iterated at all, and so will not have
// any data in the initial batch.
// TODO SERVER-33323 We use a fake shard id because the AsyncResultsMerger won't use it for
// anything, and finding the real one is non-trivial.
RemoteCursor remoteCursor;
remoteCursor.setShardId(ShardId("fakeShardIdForMergeCursors"));
remoteCursor.setHostAndPort(std::move(host));
std::vector<BSONObj> emptyBatch;
remoteCursor.setCursorResponse(CursorResponse{*nss, cursorId, emptyBatch});
remotes.push_back(std::move(remoteCursor));
}
invariant(nss); // We know there is at least one cursor in 'serializedRemotes', and we require
// each cursor to have a 'ns' field.
AsyncResultsMergerParams params;
params.setRemotes(std::move(remotes));
params.setNss(*nss);
return new DocumentSourceMergeCursors(
Grid::get(expCtx->opCtx)->getExecutorPool()->getArbitraryExecutor(),
std::move(params),
expCtx,
elem.embeddedObject().getOwned());
}
