// find all oplog entries for a given OID in the oplog.refs collection and apply them // TODO this should be a range query on oplog.refs where _id.oid == oid and applyOps to // each entry found. The locking of the query interleaved with the locking in the applyOps // did not work, so it a sequence of point queries. // TODO verify that the query plan is a indexed lookup. // TODO verify that the query plan does not fetch too many docs and then only process one of them. void applyRefOp(BSONObj entry) { OID oid = entry["ref"].OID(); LOG(3) << "apply ref " << entry << " oid " << oid << endl; long long seq = 0; // note that 0 is smaller than any of the seq numbers while (1) { BSONObj entry; { LOCK_REASON(lockReason, "repl: finding oplog.refs entry to apply"); Client::ReadContext ctx(rsOplogRefs, lockReason); // TODO: Should this be using rsOplogRefsDetails, verifying non-null? Collection *cl = getCollection(rsOplogRefs); if (cl == NULL || !cl->findOne(BSON("_id" << BSON("$gt" << BSON("oid" << oid << "seq" << seq))), entry, true)) { break; } } BSONElement e = entry.getFieldDotted("_id.seq"); seq = e.Long(); BSONElement eOID = entry.getFieldDotted("_id.oid"); if (oid != eOID.OID()) { break; } LOG(3) << "apply " << entry << " seq=" << seq << endl; applyOps(entry["ops"].Array()); } }
/* static */ KVPrefix KVPrefix::fromBSONElement(const BSONElement value) { if (value.eoo()) { return kNotPrefixed; } return KVPrefix(static_cast<int64_t>(value.Long())); }
StatusWith<CursorResponse> CursorResponse::parseFromBSON(const BSONObj& cmdResponse) { Status cmdStatus = getStatusFromCommandResult(cmdResponse); if (!cmdStatus.isOK()) { return cmdStatus; } std::string fullns; BSONObj batchObj; CursorId cursorId; BSONElement cursorElt = cmdResponse[kCursorField]; if (cursorElt.type() != BSONType::Object) { return {ErrorCodes::TypeMismatch, str::stream() << "Field '" << kCursorField << "' must be a nested object in: " << cmdResponse}; } BSONObj cursorObj = cursorElt.Obj(); BSONElement idElt = cursorObj[kIdField]; if (idElt.type() != BSONType::NumberLong) { return {ErrorCodes::TypeMismatch, str::stream() << "Field '" << kIdField << "' must be of type long in: " << cmdResponse}; } cursorId = idElt.Long(); BSONElement nsElt = cursorObj[kNsField]; if (nsElt.type() != BSONType::String) { return {ErrorCodes::TypeMismatch, str::stream() << "Field '" << kNsField << "' must be of type string in: " << cmdResponse}; } fullns = nsElt.String(); BSONElement batchElt = cursorObj[kBatchField]; if (batchElt.eoo()) { batchElt = cursorObj[kBatchFieldInitial]; } if (batchElt.type() != BSONType::Array) { return {ErrorCodes::TypeMismatch, str::stream() << "Must have array field '" << kBatchFieldInitial << "' or '" << kBatchField << "' in: " << cmdResponse}; } batchObj = batchElt.Obj(); std::vector<BSONObj> batch; for (BSONElement elt : batchObj) { if (elt.type() != BSONType::Object) { return { ErrorCodes::BadValue, str::stream() << "getMore response batch contains a non-object element: " << elt}; } batch.push_back(elt.Obj().getOwned()); } return {{NamespaceString(fullns), cursorId, batch}}; }
void run() { BSONObj spec( BSON("key" << BSON( "a" << "hashed" ) )); BSONObj nullObj = BSON( "a" << BSONNULL ); // Call getKeys on the nullObj. BSONObjSet nullFieldKeySet; ExpressionKeysPrivate::getHashKeys(nullObj, "a", 0, 0, false, &nullFieldKeySet); BSONElement nullFieldFromKey = nullFieldKeySet.begin()->firstElement(); ASSERT_EQUALS( ExpressionKeysPrivate::makeSingleHashKey( nullObj.firstElement(), 0, 0 ), nullFieldFromKey.Long() ); BSONObj missingField = IndexLegacy::getMissingField(NULL,spec); ASSERT_EQUALS( NumberLong, missingField.firstElement().type() ); ASSERT_EQUALS( nullFieldFromKey, missingField.firstElement()); }
void run() { BSONObj spec( BSON("key" << BSON( "a" << "hashed" ) << "seed" << 0x5eed )); BSONObj nullObj = BSON( "a" << BSONNULL ); BSONObjSet nullFieldKeySet; ExpressionKeysPrivate::getHashKeys(nullObj, "a", 0x5eed, 0, false, &nullFieldKeySet); BSONElement nullFieldFromKey = nullFieldKeySet.begin()->firstElement(); ASSERT_EQUALS( ExpressionKeysPrivate::makeSingleHashKey( nullObj.firstElement(), 0x5eed, 0 ), nullFieldFromKey.Long() ); // Ensure that getMissingField recognizes that the seed is different (and returns // the right key). BSONObj missingField = IndexLegacy::getMissingField(NULL,spec); ASSERT_EQUALS( NumberLong, missingField.firstElement().type()); ASSERT_EQUALS( nullFieldFromKey, missingField.firstElement()); }
SafeNum::SafeNum(const BSONElement& element) { switch (element.type()) { case NumberInt: _type = NumberInt; _value.int32Val = element.Int(); break; case NumberLong: _type = NumberLong; _value.int64Val = element.Long(); break; case NumberDouble: _type = NumberDouble; _value.doubleVal = element.Double(); break; default: _type = EOO; } }
void rollbackRefOp(BSONObj entry) { OID oid = entry["ref"].OID(); LOG(3) << "rollback ref " << entry << " oid " << oid << endl; long long seq = LLONG_MAX; while (1) { BSONObj currEntry; { LOCK_REASON(lockReason, "repl: rolling back entry from oplog.refs"); Client::ReadContext ctx(rsOplogRefs, lockReason); verify(rsOplogRefsDetails != NULL); shared_ptr<Cursor> c( Cursor::make( rsOplogRefsDetails, rsOplogRefsDetails->getPKIndex(), KeyPattern::toKeyFormat(BSON( "_id" << BSON("oid" << oid << "seq" << seq))), // right endpoint KeyPattern::toKeyFormat(BSON( "_id" << BSON("oid" << oid << "seq" << 0))), // left endpoint false, -1 // direction ) ); if (c->ok()) { currEntry = c->current().copy(); } else { break; } } BSONElement e = currEntry.getFieldDotted("_id.seq"); seq = e.Long(); BSONElement eOID = currEntry.getFieldDotted("_id.oid"); if (oid != eOID.OID()) { break; } LOG(3) << "apply " << currEntry << " seq=" << seq << endl; rollbackOps(currEntry["ops"].Array()); // decrement seq so next query gets the next value seq--; } }
// find all oplog entries for a given OID in the oplog.refs collection and apply them // TODO this should be a range query on oplog.refs where _id.oid == oid and applyOps to // each entry found. The locking of the query interleaved with the locking in the applyOps // did not work, so it a sequence of point queries. // TODO verify that the query plan is a indexed lookup. // TODO verify that the query plan does not fetch too many docs and then only process one of them. void applyRefOp(BSONObj entry) { OID oid = entry["ref"].OID(); LOG(3) << "apply ref " << entry << " oid " << oid << endl; long long seq = 0; // note that 0 is smaller than any of the seq numbers while (1) { BSONObj entry; { Client::ReadContext ctx(rsOplogRefs); // TODO: Should this be using rsOplogRefsDetails, verifying non-null? NamespaceDetails *d = nsdetails(rsOplogRefs); if (d == NULL || !d->findOne(BSON("_id" << BSON("$gt" << BSON("oid" << oid << "seq" << seq))), entry, true)) { break; } } BSONElement e = entry.getFieldDotted("_id.seq"); seq = e.Long(); BSONElement eOID = entry.getFieldDotted("_id.oid"); if (oid != eOID.OID()) { break; } LOG(3) << "apply " << entry << " seq=" << seq << endl; applyOps(entry["ops"].Array()); } }
StatusWith<CursorResponse> CursorResponse::parseFromBSON(const BSONObj& cmdResponse) { Status cmdStatus = getStatusFromCommandResult(cmdResponse); if (!cmdStatus.isOK()) { if (ErrorCodes::isStaleShardVersionError(cmdStatus.code())) { auto vWanted = ChunkVersion::fromBSON(cmdResponse, "vWanted"); auto vReceived = ChunkVersion::fromBSON(cmdResponse, "vReceived"); if (!vWanted.hasEqualEpoch(vReceived)) { return Status(ErrorCodes::StaleEpoch, cmdStatus.reason()); } } return cmdStatus; } std::string fullns; BSONObj batchObj; CursorId cursorId; BSONElement cursorElt = cmdResponse[kCursorField]; if (cursorElt.type() != BSONType::Object) { return {ErrorCodes::TypeMismatch, str::stream() << "Field '" << kCursorField << "' must be a nested object in: " << cmdResponse}; } BSONObj cursorObj = cursorElt.Obj(); BSONElement idElt = cursorObj[kIdField]; if (idElt.type() != BSONType::NumberLong) { return { ErrorCodes::TypeMismatch, str::stream() << "Field '" << kIdField << "' must be of type long in: " << cmdResponse}; } cursorId = idElt.Long(); BSONElement nsElt = cursorObj[kNsField]; if (nsElt.type() != BSONType::String) { return {ErrorCodes::TypeMismatch, str::stream() << "Field '" << kNsField << "' must be of type string in: " << cmdResponse}; } fullns = nsElt.String(); BSONElement batchElt = cursorObj[kBatchField]; if (batchElt.eoo()) { batchElt = cursorObj[kBatchFieldInitial]; } if (batchElt.type() != BSONType::Array) { return {ErrorCodes::TypeMismatch, str::stream() << "Must have array field '" << kBatchFieldInitial << "' or '" << kBatchField << "' in: " << cmdResponse}; } batchObj = batchElt.Obj(); std::vector<BSONObj> batch; for (BSONElement elt : batchObj) { if (elt.type() != BSONType::Object) { return {ErrorCodes::BadValue, str::stream() << "getMore response batch contains a non-object element: " << elt}; } batch.push_back(elt.Obj()); } for (auto& doc : batch) { doc.shareOwnershipWith(cmdResponse); } auto latestOplogTimestampElem = cmdResponse[kInternalLatestOplogTimestampField]; if (latestOplogTimestampElem && latestOplogTimestampElem.type() != BSONType::bsonTimestamp) { return { ErrorCodes::BadValue, str::stream() << "invalid _internalLatestOplogTimestamp format; expected timestamp but found: " << latestOplogTimestampElem.type()}; } auto writeConcernError = cmdResponse["writeConcernError"]; if (writeConcernError && writeConcernError.type() != BSONType::Object) { return {ErrorCodes::BadValue, str::stream() << "invalid writeConcernError format; expected object but found: " << writeConcernError.type()}; } return {{NamespaceString(fullns), cursorId, std::move(batch), boost::none, latestOplogTimestampElem ? latestOplogTimestampElem.timestamp() : boost::optional<Timestamp>{}, writeConcernError ? writeConcernError.Obj().getOwned() : boost::optional<BSONObj>{}}}; }
boost::intrusive_ptr<DocumentSource> DocumentSourceMergeCursors::createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx) { if (elem.type() == BSONType::Object) { // This is the modern serialization format. We de-serialize using the IDL. auto ownedObj = elem.embeddedObject().getOwned(); auto armParams = AsyncResultsMergerParams::parse(IDLParserErrorContext(kStageName), ownedObj); return new DocumentSourceMergeCursors( Grid::get(expCtx->opCtx)->getExecutorPool()->getArbitraryExecutor(), std::move(armParams), expCtx, std::move(ownedObj)); } // This is the old serialization format which can still be generated by mongos processes // older than 4.0. // TODO SERVER-34009 Remove support for this format. uassert(17026, "$mergeCursors stage expected either an array or an object as argument", elem.type() == BSONType::Array); const auto serializedRemotes = elem.Array(); uassert(50729, "$mergeCursors stage expected array with at least one entry", serializedRemotes.size() > 0); boost::optional<NamespaceString> nss; std::vector<RemoteCursor> remotes; for (auto&& cursor : serializedRemotes) { BSONElement nsElem; BSONElement hostElem; BSONElement idElem; uassert(17027, "$mergeCursors stage requires each cursor in array to be an object", cursor.type() == BSONType::Object); for (auto&& cursorElem : cursor.Obj()) { const auto fieldName = cursorElem.fieldNameStringData(); if (fieldName == "ns"_sd) { nsElem = cursorElem; } else if (fieldName == "host"_sd) { hostElem = cursorElem; } else if (fieldName == "id"_sd) { idElem = cursorElem; } else { uasserted(50730, str::stream() << "Unrecognized option " << fieldName << " within cursor provided to $mergeCursors: " << cursor); } } uassert( 50731, "$mergeCursors stage requires \'ns\' field with type string for each cursor in array", nsElem.type() == BSONType::String); // We require each cursor to have the same namespace. This isn't a fundamental limit of the // system, but needs to be true due to the implementation of AsyncResultsMerger, which // tracks one namespace for all cursors. uassert(50720, "$mergeCursors requires each cursor to have the same namespace", !nss || nss->ns() == nsElem.valueStringData()); nss = NamespaceString(nsElem.String()); uassert( 50721, "$mergeCursors stage requires \'host\' field with type string for each cursor in array", hostElem.type() == BSONType::String); auto host = uassertStatusOK(HostAndPort::parse(hostElem.valueStringData())); uassert(50722, "$mergeCursors stage requires \'id\' field with type long for each cursor in array", idElem.type() == BSONType::NumberLong); auto cursorId = idElem.Long(); // We are assuming that none of the cursors have been iterated at all, and so will not have // any data in the initial batch. // TODO SERVER-33323 We use a fake shard id because the AsyncResultsMerger won't use it for // anything, and finding the real one is non-trivial. RemoteCursor remoteCursor; remoteCursor.setShardId(ShardId("fakeShardIdForMergeCursors")); remoteCursor.setHostAndPort(std::move(host)); std::vector<BSONObj> emptyBatch; remoteCursor.setCursorResponse(CursorResponse{*nss, cursorId, emptyBatch}); remotes.push_back(std::move(remoteCursor)); } invariant(nss); // We know there is at least one cursor in 'serializedRemotes', and we require // each cursor to have a 'ns' field. AsyncResultsMergerParams params; params.setRemotes(std::move(remotes)); params.setNss(*nss); return new DocumentSourceMergeCursors( Grid::get(expCtx->opCtx)->getExecutorPool()->getArbitraryExecutor(), std::move(params), expCtx, elem.embeddedObject().getOwned()); }