// find all oplog entries for a given OID in the oplog.refs collection and apply them // TODO this should be a range query on oplog.refs where _id.oid == oid and applyOps to // each entry found. The locking of the query interleaved with the locking in the applyOps // did not work, so it a sequence of point queries. // TODO verify that the query plan is a indexed lookup. // TODO verify that the query plan does not fetch too many docs and then only process one of them. void applyRefOp(BSONObj entry) { OID oid = entry["ref"].OID(); LOG(3) << "apply ref " << entry << " oid " << oid << endl; long long seq = 0; // note that 0 is smaller than any of the seq numbers while (1) { BSONObj entry; { LOCK_REASON(lockReason, "repl: finding oplog.refs entry to apply"); Client::ReadContext ctx(rsOplogRefs, lockReason); // TODO: Should this be using rsOplogRefsDetails, verifying non-null? Collection *cl = getCollection(rsOplogRefs); if (cl == NULL || !cl->findOne(BSON("_id" << BSON("$gt" << BSON("oid" << oid << "seq" << seq))), entry, true)) { break; } } BSONElement e = entry.getFieldDotted("_id.seq"); seq = e.Long(); BSONElement eOID = entry.getFieldDotted("_id.oid"); if (oid != eOID.OID()) { break; } LOG(3) << "apply " << entry << " seq=" << seq << endl; applyOps(entry["ops"].Array()); } }
Status bsonExtractOIDField(const BSONObj& object, StringData fieldName, OID* out) { BSONElement element; Status status = bsonExtractTypedField(object, fieldName, jstOID, &element); if (status.isOK()) *out = element.OID(); return status; }
StatusWith<ShardingMetadata> ShardingMetadata::readFromMetadata(const BSONObj& metadataObj) { BSONElement smElem; auto smExtractStatus = bsonExtractTypedField(metadataObj, kGLEStatsFieldName, mongo::Object, &smElem); if (!smExtractStatus.isOK()) { return smExtractStatus; } if (smElem.embeddedObject().nFields() != 2) { return Status(ErrorCodes::InvalidOptions, str::stream() << "The $gleStats object can only have 2 fields, but got " << smElem.embeddedObject().toString()); } BSONElement lastOpTimeElem; auto lastOpTimeExtractStatus = bsonExtractTypedField(smElem.embeddedObject(), kGLEStatsLastOpTimeFieldName, mongo::bsonTimestamp, &lastOpTimeElem); if (!lastOpTimeExtractStatus.isOK()) { return lastOpTimeExtractStatus; } BSONElement lastElectionIdElem; auto lastElectionIdExtractStatus = bsonExtractTypedField( smElem.embeddedObject(), kGLEStatsElectionIdFieldName, mongo::jstOID, &lastElectionIdElem); if (!lastElectionIdExtractStatus.isOK()) { return lastElectionIdExtractStatus; } return ShardingMetadata(lastOpTimeElem.timestamp(), lastElectionIdElem.OID()); }
Status HandshakeArgs::initialize(const BSONObj& argsObj) { Status status = bsonCheckOnlyHasFields("HandshakeArgs", argsObj, kLegalHandshakeFieldNames); if (!status.isOK()) return status; BSONElement oid; status = bsonExtractTypedField(argsObj, kRIDFieldName, jstOID, &oid); if (!status.isOK()) return status; _rid = oid.OID(); _hasRid = true; status = bsonExtractIntegerField(argsObj, kMemberIdFieldName, &_memberId); if (!status.isOK()) { // field not necessary for master slave, do not return NoSuchKey Error if (status != ErrorCodes::NoSuchKey) { return status; } _memberId = -1; } else { _hasMemberId = true; } return Status::OK(); }
StatusWith<LocksType> LocksType::fromBSON(const BSONObj& source) { LocksType lock; { std::string lockName; Status status = bsonExtractStringField(source, name.name(), &lockName); if (!status.isOK()) return status; lock._name = lockName; } { long long lockStateInt; Status status = bsonExtractIntegerField(source, state.name(), &lockStateInt); if (!status.isOK()) return status; lock._state = static_cast<State>(lockStateInt); } if (source.hasField(process.name())) { std::string lockProcess; Status status = bsonExtractStringField(source, process.name(), &lockProcess); if (!status.isOK()) return status; lock._process = lockProcess; } if (source.hasField(lockID.name())) { BSONElement lockIDElem; Status status = bsonExtractTypedField(source, lockID.name(), BSONType::jstOID, &lockIDElem); if (!status.isOK()) return status; lock._lockID = lockIDElem.OID(); } if (source.hasField(who.name())) { std::string lockWho; Status status = bsonExtractStringField(source, who.name(), &lockWho); if (!status.isOK()) return status; lock._who = lockWho; } if (source.hasField(why.name())) { std::string lockWhy; Status status = bsonExtractStringField(source, why.name(), &lockWhy); if (!status.isOK()) return status; lock._why = lockWhy; } return lock; }
// Copy a range of documents to the local oplog.refs collection static void copyOplogRefsRange(OplogReader &r, OID oid) { shared_ptr<DBClientCursor> c = r.getOplogRefsCursor(oid); Client::ReadContext ctx(rsOplogRefs); while (c->more()) { BSONObj b = c->next(); BSONElement eOID = b.getFieldDotted("_id.oid"); if (oid != eOID.OID()) { break; } LOG(6) << "copyOplogRefsRange " << b << endl; writeEntryToOplogRefs(b); } }
// Gets the string representation of a BSON object that can be correctly written to a CSV file string csvString (const BSONElement& object) { const char* binData; // Only used with BinData type switch (object.type()) { case MinKey: return "$MinKey"; case MaxKey: return "$MaxKey"; case NumberInt: case NumberDouble: case NumberLong: case Bool: return object.toString(false); case String: case Symbol: return csvEscape(object.toString(false), true); case Object: return csvEscape(object.jsonString(Strict, false)); case Array: return csvEscape(object.jsonString(Strict, false)); case BinData: int len; binData = object.binDataClean(len); return toHex(binData, len); case jstOID: return "ObjectID(" + object.OID().toString() + ")"; // OIDs are always 24 bytes case Date: return timeToISOString(object.Date() / 1000); case Timestamp: return csvEscape(object.jsonString(Strict, false)); case RegEx: return csvEscape("/" + string(object.regex()) + "/" + string(object.regexFlags())); case Code: return csvEscape(object.toString(false)); case CodeWScope: if (string(object.codeWScopeScopeDataUnsafe()) == "") { return csvEscape(object.toString(false)); } else { return csvEscape(object.jsonString(Strict, false)); } case EOO: case Undefined: case DBRef: case jstNULL: cerr << "Invalid BSON object type for CSV output: " << object.type() << endl; return ""; } // Can never get here verify(false); return ""; }
Status bsonExtractOIDFieldWithDefault(const BSONObj& object, StringData fieldName, const OID& defaultValue, OID* out) { BSONElement element; Status status = bsonExtractTypedFieldImpl(object, fieldName, jstOID, &element, true); if (status == ErrorCodes::NoSuchKey) { *out = defaultValue; return Status::OK(); } if (status.isOK()) *out = element.OID(); return status; }
void ClientInfo::_addWriteBack( vector<WBInfo>& all , const BSONObj& gle ) { BSONElement w = gle["writeback"]; if ( w.type() != jstOID ) return; BSONElement cid = gle["connectionId"]; if ( cid.eoo() ) { error() << "getLastError writeback can't work because of version mis-match" << endl; return; } all.push_back( WBInfo( cid.numberLong() , w.OID() ) ); }
StatusWith<ShardingMetadata> ShardingMetadata::readFromMetadata(const BSONObj& metadataObj) { BSONElement smElem; auto smExtractStatus = bsonExtractTypedField(metadataObj, kGLEStatsFieldName, mongol::Object, &smElem); if (!smExtractStatus.isOK()) { return smExtractStatus; } if (smElem.embeddedObject().nFields() != 2) { return Status(ErrorCodes::InvalidOptions, str::stream() << "The $gleStats object can only have 2 fields, but got " << smElem.embeddedObject().toString()); } repl::OpTime opTime; const BSONElement opTimeElement = smElem.embeddedObject()[kGLEStatsLastOpTimeFieldName]; if (opTimeElement.eoo()) { return Status(ErrorCodes::NoSuchKey, "lastOpTime field missing"); } else if (opTimeElement.type() == bsonTimestamp) { opTime = repl::OpTime(opTimeElement.timestamp(), repl::OpTime::kUninitializedTerm); } else if (opTimeElement.type() == Date) { opTime = repl::OpTime(Timestamp(opTimeElement.date()), repl::OpTime::kUninitializedTerm); } else if (opTimeElement.type() == Object) { Status status = bsonExtractOpTimeField(smElem.embeddedObject(), kGLEStatsLastOpTimeFieldName, &opTime); if (!status.isOK()) { return status; } } else { return Status(ErrorCodes::TypeMismatch, str::stream() << "Expected \"" << kGLEStatsLastOpTimeFieldName << "\" field in response to replSetHeartbeat " "command to have type Date or Timestamp, but found type " << typeName(opTimeElement.type())); } BSONElement lastElectionIdElem; auto lastElectionIdExtractStatus = bsonExtractTypedField( smElem.embeddedObject(), kGLEStatsElectionIdFieldName, mongol::jstOID, &lastElectionIdElem); if (!lastElectionIdExtractStatus.isOK()) { return lastElectionIdExtractStatus; } return ShardingMetadata(opTime, lastElectionIdElem.OID()); }
StatusWith<ChunkVersion> ChunkVersion::parseFromBSONWithFieldForCommands(const BSONObj& obj, StringData field) { BSONElement versionElem; Status status = bsonExtractField(obj, field, &versionElem); if (!status.isOK()) return status; if (versionElem.type() != Array) { return {ErrorCodes::TypeMismatch, str::stream() << "Invalid type " << versionElem.type() << " for shardVersion element. Expected an array"}; } BSONObjIterator it(versionElem.Obj()); if (!it.more()) return {ErrorCodes::BadValue, "Unexpected empty version"}; ChunkVersion version; // Expect the timestamp { BSONElement tsPart = it.next(); if (tsPart.type() != bsonTimestamp) return {ErrorCodes::TypeMismatch, str::stream() << "Invalid type " << tsPart.type() << " for version timestamp part."}; version._combined = tsPart.timestamp().asULL(); } // Expect the epoch OID { BSONElement epochPart = it.next(); if (epochPart.type() != jstOID) return {ErrorCodes::TypeMismatch, str::stream() << "Invalid type " << epochPart.type() << " for version epoch part."}; version._epoch = epochPart.OID(); } return version; }
void ClientInfo::_addWriteBack( vector<WBInfo>& all, const BSONObj& gle, bool fromLastOperation ) { BSONElement w = gle["writeback"]; if ( w.type() != jstOID ) return; BSONElement cid = gle["connectionId"]; if ( cid.eoo() ) { error() << "getLastError writeback can't work because of version mismatch" << endl; return; } string ident = ""; if ( gle["instanceIdent"].type() == String ) ident = gle["instanceIdent"].String(); all.push_back( WBInfo( WriteBackListener::ConnectionIdent( ident , cid.numberLong() ), w.OID(), fromLastOperation ) ); }
void rollbackRefOp(BSONObj entry) { OID oid = entry["ref"].OID(); LOG(3) << "rollback ref " << entry << " oid " << oid << endl; long long seq = LLONG_MAX; while (1) { BSONObj currEntry; { LOCK_REASON(lockReason, "repl: rolling back entry from oplog.refs"); Client::ReadContext ctx(rsOplogRefs, lockReason); verify(rsOplogRefsDetails != NULL); shared_ptr<Cursor> c( Cursor::make( rsOplogRefsDetails, rsOplogRefsDetails->getPKIndex(), KeyPattern::toKeyFormat(BSON( "_id" << BSON("oid" << oid << "seq" << seq))), // right endpoint KeyPattern::toKeyFormat(BSON( "_id" << BSON("oid" << oid << "seq" << 0))), // left endpoint false, -1 // direction ) ); if (c->ok()) { currEntry = c->current().copy(); } else { break; } } BSONElement e = currEntry.getFieldDotted("_id.seq"); seq = e.Long(); BSONElement eOID = currEntry.getFieldDotted("_id.oid"); if (oid != eOID.OID()) { break; } LOG(3) << "apply " << currEntry << " seq=" << seq << endl; rollbackOps(currEntry["ops"].Array()); // decrement seq so next query gets the next value seq--; } }
// find all oplog entries for a given OID in the oplog.refs collection and apply them // TODO this should be a range query on oplog.refs where _id.oid == oid and applyOps to // each entry found. The locking of the query interleaved with the locking in the applyOps // did not work, so it a sequence of point queries. // TODO verify that the query plan is a indexed lookup. // TODO verify that the query plan does not fetch too many docs and then only process one of them. void applyRefOp(BSONObj entry) { OID oid = entry["ref"].OID(); LOG(3) << "apply ref " << entry << " oid " << oid << endl; long long seq = 0; // note that 0 is smaller than any of the seq numbers while (1) { BSONObj entry; { Client::ReadContext ctx(rsOplogRefs); // TODO: Should this be using rsOplogRefsDetails, verifying non-null? NamespaceDetails *d = nsdetails(rsOplogRefs); if (d == NULL || !d->findOne(BSON("_id" << BSON("$gt" << BSON("oid" << oid << "seq" << seq))), entry, true)) { break; } } BSONElement e = entry.getFieldDotted("_id.seq"); seq = e.Long(); BSONElement eOID = entry.getFieldDotted("_id.oid"); if (oid != eOID.OID()) { break; } LOG(3) << "apply " << entry << " seq=" << seq << endl; applyOps(entry["ops"].Array()); } }
// PD_TRACE_DECLARE_FUNCTION ( SDB__RTNCONTEXTLOB_OPEN, "_rtnContextLob::open" ) INT32 _rtnContextLob::open( const BSONObj &lob, BOOLEAN isLocal, _pmdEDUCB *cb, SDB_DPSCB *dpsCB ) { INT32 rc = SDB_OK ; PD_TRACE_ENTRY( SDB__RTNCONTEXTLOB_OPEN ) ; BSONElement mode ; BSONElement oid ; BSONElement fullName = lob.getField( FIELD_NAME_COLLECTION ) ; if ( String != fullName.type() ) { PD_LOG( PDERROR, "can not find collection name in lob[%s]", lob.toString( FALSE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } oid = lob.getField( FIELD_NAME_LOB_OID ) ; if ( jstOID != oid.type() ) { PD_LOG( PDERROR, "invalid oid in meta bsonobj:%s", lob.toString( FALSE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } mode = lob.getField( FIELD_NAME_LOB_OPEN_MODE ) ; if ( NumberInt != mode.type() ) { PD_LOG( PDERROR, "invalid mode in meta bsonobj:%s", lob.toString( FALSE, TRUE ).c_str() ) ; rc = SDB_INVALIDARG ; goto error ; } if ( isLocal ) { _stream = SDB_OSS_NEW _rtnLocalLobStream() ; } else { _stream = SDB_OSS_NEW _rtnCoordLobStream() ; } if ( NULL == _stream ) { PD_LOG( PDERROR, "failed to allocate mem." ) ; rc = SDB_OOM ; goto error ; } _stream->setDPSCB( dpsCB ) ; rc = _stream->open( fullName.valuestr(), oid.OID(), mode.Int(), cb ) ; if ( SDB_OK != rc ) { PD_LOG( PDERROR, "failed to open lob stream:%d", rc ) ; goto error ; } _isOpened = TRUE ; _hitEnd = FALSE ; done: PD_TRACE_EXITRC( SDB__RTNCONTEXTLOB_OPEN, rc ) ; return rc ; error: goto done ; }
Status IsMasterResponse::initialize(const BSONObj& doc) { Status status = bsonExtractBooleanField(doc, kIsMasterFieldName, &_isMaster); if (!status.isOK()) { return status; } _isMasterSet = true; status = bsonExtractBooleanField(doc, kSecondaryFieldName, &_secondary); if (!status.isOK()) { return status; } _isSecondarySet = true; if (doc.hasField(kInfoFieldName)) { if (_isMaster || _secondary || !doc.hasField(kIsReplicaSetFieldName) || !doc[kIsReplicaSetFieldName].booleanSafe()) { return Status(ErrorCodes::FailedToParse, str::stream() << "Expected presence of \"" << kInfoFieldName << "\" field to indicate no valid config loaded, but other " "fields weren't as we expected"); } _configSet = false; return Status::OK(); } else { if (doc.hasField(kIsReplicaSetFieldName)) { return Status(ErrorCodes::FailedToParse, str::stream() << "Found \"" << kIsReplicaSetFieldName << "\" field which should indicate that no valid config " "is loaded, but we didn't also have an \"" << kInfoFieldName << "\" field as we expected"); } } status = bsonExtractStringField(doc, kSetNameFieldName, &_setName); if (!status.isOK()) { return status; } _setNameSet = true; status = bsonExtractIntegerField(doc, kSetVersionFieldName, &_setVersion); if (!status.isOK()) { return status; } _setVersionSet = true; if (doc.hasField(kHostsFieldName)) { BSONElement hostsElement; status = bsonExtractTypedField(doc, kHostsFieldName, Array, &hostsElement); if (!status.isOK()) { return status; } for (BSONObjIterator it(hostsElement.Obj()); it.more();) { BSONElement hostElement = it.next(); if (hostElement.type() != String) { return Status(ErrorCodes::TypeMismatch, str::stream() << "Elements in \"" << kHostsFieldName << "\" array of isMaster response must be of type " << typeName(String) << " but found type " << typeName(hostElement.type())); } _hosts.push_back(HostAndPort(hostElement.String())); } _hostsSet = true; } if (doc.hasField(kPassivesFieldName)) { BSONElement passivesElement; status = bsonExtractTypedField(doc, kPassivesFieldName, Array, &passivesElement); if (!status.isOK()) { return status; } for (BSONObjIterator it(passivesElement.Obj()); it.more();) { BSONElement passiveElement = it.next(); if (passiveElement.type() != String) { return Status(ErrorCodes::TypeMismatch, str::stream() << "Elements in \"" << kPassivesFieldName << "\" array of isMaster response must be of type " << typeName(String) << " but found type " << typeName(passiveElement.type())); } _passives.push_back(HostAndPort(passiveElement.String())); } _passivesSet = true; } if (doc.hasField(kArbitersFieldName)) { BSONElement arbitersElement; status = bsonExtractTypedField(doc, kArbitersFieldName, Array, &arbitersElement); if (!status.isOK()) { return status; } for (BSONObjIterator it(arbitersElement.Obj()); it.more();) { BSONElement arbiterElement = it.next(); if (arbiterElement.type() != String) { return Status(ErrorCodes::TypeMismatch, str::stream() << "Elements in \"" << kArbitersFieldName << "\" array of isMaster response must be of type " << typeName(String) << " but found type " << typeName(arbiterElement.type())); } _arbiters.push_back(HostAndPort(arbiterElement.String())); } _arbitersSet = true; } if (doc.hasField(kPrimaryFieldName)) { std::string primaryString; status = bsonExtractStringField(doc, kPrimaryFieldName, &primaryString); if (!status.isOK()) { return status; } _primary = HostAndPort(primaryString); _primarySet = true; } if (doc.hasField(kArbiterOnlyFieldName)) { status = bsonExtractBooleanField(doc, kArbiterOnlyFieldName, &_arbiterOnly); if (!status.isOK()) { return status; } _arbiterOnlySet = true; } if (doc.hasField(kPassiveFieldName)) { status = bsonExtractBooleanField(doc, kPassiveFieldName, &_passive); if (!status.isOK()) { return status; } _passiveSet = true; } if (doc.hasField(kHiddenFieldName)) { status = bsonExtractBooleanField(doc, kHiddenFieldName, &_hidden); if (!status.isOK()) { return status; } _hiddenSet = true; } if (doc.hasField(kBuildIndexesFieldName)) { status = bsonExtractBooleanField(doc, kBuildIndexesFieldName, &_buildIndexes); if (!status.isOK()) { return status; } _buildIndexesSet = true; } if (doc.hasField(kSlaveDelayFieldName)) { long long slaveDelaySecs; status = bsonExtractIntegerField(doc, kSlaveDelayFieldName, &slaveDelaySecs); if (!status.isOK()) { return status; } _slaveDelaySet = true; _slaveDelay = Seconds(slaveDelaySecs); } if (doc.hasField(kTagsFieldName)) { BSONElement tagsElement; status = bsonExtractTypedField(doc, kTagsFieldName, Object, &tagsElement); if (!status.isOK()) { return status; } for (BSONObjIterator it(tagsElement.Obj()); it.more();) { BSONElement tagElement = it.next(); if (tagElement.type() != String) { return Status(ErrorCodes::TypeMismatch, str::stream() << "Elements in \"" << kTagsFieldName << "\" obj " "of isMaster response must be of type " << typeName(String) << " but found type " << typeName(tagsElement.type())); } _tags[tagElement.fieldNameStringData().toString()] = tagElement.String(); } _tagsSet = true; } if (doc.hasField(kElectionIdFieldName)) { BSONElement electionIdElem; status = bsonExtractTypedField(doc, kElectionIdFieldName, jstOID, &electionIdElem); if (!status.isOK()) { return status; } _electionId = electionIdElem.OID(); } std::string meString; status = bsonExtractStringField(doc, kMeFieldName, &meString); if (!status.isOK()) { return status; } _me = HostAndPort(meString); _meSet = true; return Status::OK(); }
StatusWith<MoveChunkRequest> MoveChunkRequest::createFromCommand(NamespaceString nss, const BSONObj& obj) { auto secondaryThrottleStatus = MigrationSecondaryThrottleOptions::createFromCommand(obj); if (!secondaryThrottleStatus.isOK()) { return secondaryThrottleStatus.getStatus(); } auto rangeStatus = ChunkRange::fromBSON(obj); if (!rangeStatus.isOK()) { return rangeStatus.getStatus(); } MoveChunkRequest request(std::move(nss), std::move(rangeStatus.getValue()), std::move(secondaryThrottleStatus.getValue())); { std::string shardStr; Status status = bsonExtractStringField(obj, kFromShardId, &shardStr); request._fromShardId = shardStr; if (!status.isOK()) { return status; } } { std::string shardStr; Status status = bsonExtractStringField(obj, kToShardId, &shardStr); request._toShardId = shardStr; if (!status.isOK()) { return status; } } { BSONElement epochElem; Status status = bsonExtractTypedField(obj, kEpoch, BSONType::jstOID, &epochElem); if (!status.isOK()) return status; request._versionEpoch = epochElem.OID(); } { Status status = bsonExtractBooleanFieldWithDefault(obj, kWaitForDelete, false, &request._waitForDelete); if (!status.isOK()) { return status; } } // Check for the deprecated name '_waitForDelete' if 'waitForDelete' was false. if (!request._waitForDelete) { Status status = bsonExtractBooleanFieldWithDefault( obj, kWaitForDeleteDeprecated, false, &request._waitForDelete); if (!status.isOK()) { return status; } } { long long maxChunkSizeBytes; Status status = bsonExtractIntegerField(obj, kMaxChunkSizeBytes, &maxChunkSizeBytes); if (!status.isOK()) { return status; } request._maxChunkSizeBytes = static_cast<int64_t>(maxChunkSizeBytes); } { // TODO: delete this block in 3.8 bool takeDistLock = false; Status status = bsonExtractBooleanField(obj, kTakeDistLock, &takeDistLock); if (status.isOK() && takeDistLock) { return Status{ErrorCodes::IncompatibleShardingConfigVersion, str::stream() << "Request received from an older, incompatible mongodb version"}; } } return request; }
StatusWith<ShardCollectionType> ShardCollectionType::fromBSON(const BSONObj& source) { NamespaceString uuid; { std::string ns; Status status = bsonExtractStringField(source, ShardCollectionType::uuid.name(), &ns); if (!status.isOK()) return status; uuid = NamespaceString{ns}; } NamespaceString nss; { std::string ns; Status status = bsonExtractStringField(source, ShardCollectionType::ns.name(), &ns); if (!status.isOK()) { return status; } nss = NamespaceString{ns}; } OID epoch; { BSONElement oidElem; Status status = bsonExtractTypedField( source, ShardCollectionType::epoch.name(), BSONType::jstOID, &oidElem); if (!status.isOK()) return status; epoch = oidElem.OID(); } BSONElement collKeyPattern; Status status = bsonExtractTypedField( source, ShardCollectionType::keyPattern.name(), Object, &collKeyPattern); if (!status.isOK()) { return status; } BSONObj obj = collKeyPattern.Obj(); if (obj.isEmpty()) { return Status(ErrorCodes::ShardKeyNotFound, str::stream() << "Empty shard key. Failed to parse: " << source.toString()); } KeyPattern pattern(obj.getOwned()); BSONObj collation; { BSONElement defaultCollation; Status status = bsonExtractTypedField( source, ShardCollectionType::defaultCollation.name(), Object, &defaultCollation); if (status.isOK()) { BSONObj obj = defaultCollation.Obj(); if (obj.isEmpty()) { return Status(ErrorCodes::BadValue, "empty defaultCollation"); } collation = obj.getOwned(); } else if (status != ErrorCodes::NoSuchKey) { return status; } } bool unique; { Status status = bsonExtractBooleanField(source, ShardCollectionType::unique.name(), &unique); if (!status.isOK()) { return status; } } ShardCollectionType shardCollectionType(uuid, nss, epoch, pattern, collation, unique); // Below are optional fields. bool refreshing; { Status status = bsonExtractBooleanField(source, ShardCollectionType::refreshing.name(), &refreshing); if (status.isOK()) { shardCollectionType.setRefreshing(refreshing); } else if (status == ErrorCodes::NoSuchKey) { // The field is not set yet, which is okay. } else { return status; } } long long refreshSequenceNumber; { Status status = bsonExtractIntegerField( source, ShardCollectionType::refreshSequenceNumber.name(), &refreshSequenceNumber); if (status.isOK()) { shardCollectionType.setRefreshSequenceNumber(refreshSequenceNumber); } else if (status == ErrorCodes::NoSuchKey) { // The field is not set yet, which is okay. } else { return status; } } return shardCollectionType; }
/* **************************************************************************** * * SubscriptionCache::insert - */ void SubscriptionCache::insert(const std::string& tenant, BSONObj bobj) { BSONElement idField = bobj.getField("_id"); if (idField.eoo() == true) { LM_E(("Database Error (error retrieving _id field in doc: '%s')", bobj.toString().c_str())); return; } // // 01. Extract values from database object 'bobj' // std::string subId = idField.OID().toString(); int64_t expiration = bobj.hasField(CSUB_EXPIRATION)? bobj.getField(CSUB_EXPIRATION).Long() : 0; std::string reference = bobj.getField(CSUB_REFERENCE).String(); int64_t throttling = bobj.hasField(CSUB_THROTTLING)? bobj.getField(CSUB_THROTTLING).Long() : -1; std::vector<BSONElement> eVec = bobj.getField(CSUB_ENTITIES).Array(); std::vector<BSONElement> attrVec = bobj.getField(CSUB_ATTRS).Array(); std::vector<BSONElement> condVec = bobj.getField(CSUB_CONDITIONS).Array(); std::string formatString = bobj.hasField(CSUB_FORMAT)? bobj.getField(CSUB_FORMAT).String() : "XML"; std::string servicePath = bobj.hasField(CSUB_SERVICE_PATH)? bobj.getField(CSUB_SERVICE_PATH).String() : "/"; Format format = stringToFormat(formatString); int lastNotification = bobj.hasField(CSUB_LASTNOTIFICATION)? bobj.getField(CSUB_LASTNOTIFICATION).Int() : 0; std::vector<EntityInfo*> eiV; std::vector<std::string> attrV; Restriction restriction; NotifyConditionVector notifyConditionVector; // // 02. Push Entity-data names to EntityInfo Vector (eiV) // for (unsigned int ix = 0; ix < eVec.size(); ++ix) { BSONObj entity = eVec[ix].embeddedObject(); if (!entity.hasField(CSUB_ENTITY_ID)) { LM_W(("Runtime Error (got a subscription without id)")); continue; } std::string id = entity.getStringField(ENT_ENTITY_ID); if (!entity.hasField(CSUB_ENTITY_ISPATTERN)) { continue; } std::string isPattern = entity.getStringField(CSUB_ENTITY_ISPATTERN); if (isPattern != "true") { continue; } std::string type = ""; if (entity.hasField(CSUB_ENTITY_TYPE)) { type = entity.getStringField(CSUB_ENTITY_TYPE); } EntityInfo* eiP = new EntityInfo(id, type); eiV.push_back(eiP); } if (eiV.size() == 0) { return; } // // 03. Push attribute names to Attribute Vector (attrV) // for (unsigned int ix = 0; ix < attrVec.size(); ++ix) { std::string attributeName = attrVec[ix].String(); attrV.push_back(attributeName); } // // 04. FIXME P4: Restriction not implemented // // // 05. Fill in notifyConditionVector from condVec // for (unsigned int ix = 0; ix < condVec.size(); ++ix) { BSONObj condition = condVec[ix].embeddedObject(); std::string condType; std::vector<BSONElement> valueVec; condType = condition.getStringField(CSUB_CONDITIONS_TYPE); if (condType != "ONCHANGE") { continue; } NotifyCondition* ncP = new NotifyCondition(); ncP->type = condType; valueVec = condition.getField(CSUB_CONDITIONS_VALUE).Array(); for (unsigned int vIx = 0; vIx < valueVec.size(); ++vIx) { std::string condValue; condValue = valueVec[vIx].String(); ncP->condValueList.push_back(condValue); } notifyConditionVector.push_back(ncP); } if (notifyConditionVector.size() == 0) { for (unsigned int ix = 0; ix < eiV.size(); ++ix) { delete(eiV[ix]); } eiV.clear(); restriction.release(); return; } // // 06. Create Subscription and add it to the subscription-cache // Subscription* subP = new Subscription(tenant, servicePath, subId, eiV, attrV, throttling, expiration, restriction, notifyConditionVector, reference, lastNotification, format); subCache->insert(subP); notifyConditionVector.release(); // Subscription constructor makes a copy }
int LocalMemoryGridFile::flush() { trace() << " -> LocalMemoryGridFile::flush {file: " << _filename << "}" << endl; if (!_dirty) { // Since, there are no dirty chunks, this does not need a flush info() << "buffers are not dirty.. need not flush {filename: " << _filename << "}" << endl; return 0; } size_t bufferLen = 0; boost::shared_array<char> buffer = createFlushBuffer(bufferLen); if (!buffer.get() && bufferLen > 0) { // Failed to create flush buffer return -ENOMEM; } // Get the existing gridfile from GridFS to get metadata and delete the // file from the system try { ScopedDbConnection dbc(globalFSOptions._connectString); GridFS gridFS(dbc.conn(), globalFSOptions._db, globalFSOptions._collPrefix); GridFile origGridFile = gridFS.findFile(BSON("filename" << _filename)); if (!origGridFile.exists()) { dbc.done(); warn() << "Requested file not found for flushing back data {file: " << _filename << "}" << endl; return -EBADF; } //TODO: Make checks for appropriate object correctness //i.e. do not update anything that is not a Regular File //Check what happens in case of a link gridFS.removeFile(_filename); trace() << "Removing the current file from GridFS {file: " << _filename << "}" << endl; //TODO: Check for remove status if that was successfull or not //TODO: Rather have an update along with active / passive flag for the //file try { GridFS gridFS(dbc.conn(), globalFSOptions._db, globalFSOptions._collPrefix); // Create an empty file to signify the file creation and open a local file for the same trace() << "Adding new file to GridFS {file: " << _filename << "}" << endl; BSONObj fileObj = gridFS.storeFile(buffer.get(), bufferLen, _filename); if (!fileObj.isValid()) { warn() << "Failed to save file object in data flush {file: " << _filename << "}" << std::endl; dbc.done(); return -EBADF; } // Update the last updated date for the document BSONObj metadata = origGridFile.getMetadata(); BSONElement fileObjId = fileObj.getField("_id"); dbc->update(globalFSOptions._filesNS, BSON("_id" << fileObjId.OID()), BSON("$set" << BSON( "uploadDate" << origGridFile.getUploadDate() << "metadata.type" << "file" << "metadata.filename" << mgridfs::getPathBasename(_filename) << "metadata.directory" << mgridfs::getPathDirname(_filename) << "metadata.lastUpdated" << jsTime() << "metadata.uid" << metadata["uid"] << "metadata.gid" << metadata["gid"] << "metadata.mode" << metadata["mode"] ) ) ); } catch (DBException& e) { error() << "Caught exception in saving remote file in flush {code: " << e.getCode() << ", what: " << e.what() << ", exception: " << e.toString() << "}" << endl; return -EIO; } dbc.done(); } catch (DBException& e) { // Something failed in getting the file from GridFS error() << "Caught exception in getting remote file for flush {code: " << e.getCode() << ", what: " << e.what() << ", exception: " << e.toString() << "}" << endl; return -EIO; } _dirty = false; debug() << "Completed flushing the file content to GridFS {file: " << _filename << "}" << endl; return 0; }
void WriteBackListener::run(){ OID lastID; lastID.clear(); int secsToSleep = 0; while ( ! inShutdown() && Shard::isMember( _addr ) ){ if ( lastID.isSet() ){ scoped_lock lk( _seenWritebacksLock ); _seenWritebacks.insert( lastID ); lastID.clear(); } try { ScopedDbConnection conn( _addr ); BSONObj result; { BSONObjBuilder cmd; cmd.appendOID( "writebacklisten" , &serverID ); // Command will block for data if ( ! conn->runCommand( "admin" , cmd.obj() , result ) ){ log() << "writebacklisten command failed! " << result << endl; conn.done(); continue; } } log(1) << "writebacklisten result: " << result << endl; BSONObj data = result.getObjectField( "data" ); if ( data.getBoolField( "writeBack" ) ){ string ns = data["ns"].valuestrsafe(); { BSONElement e = data["id"]; if ( e.type() == jstOID ) lastID = e.OID(); } int len; Message m( (void*)data["msg"].binData( len ) , false ); massert( 10427 , "invalid writeback message" , m.header()->valid() ); DBConfigPtr db = grid.getDBConfig( ns ); ShardChunkVersion needVersion( data["version"] ); log(1) << "writeback id: " << lastID << " needVersion : " << needVersion.toString() << " mine : " << db->getChunkManager( ns )->getVersion().toString() << endl;// TODO change to log(3) if ( logLevel ) log(1) << debugString( m ) << endl; if ( needVersion.isSet() && needVersion <= db->getChunkManager( ns )->getVersion() ){ // this means when the write went originally, the version was old // if we're here, it means we've already updated the config, so don't need to do again //db->getChunkManager( ns , true ); // SERVER-1349 } else { // we received a writeback object that was sent to a previous version of a shard // the actual shard may not have the object the writeback operation is for // we need to reload the chunk manager and get the new shard versions db->getChunkManager( ns , true ); } Request r( m , 0 ); r.init(); r.process(); } else if ( result["noop"].trueValue() ){ // no-op } else { log() << "unknown writeBack result: " << result << endl; } conn.done(); secsToSleep = 0; continue; } catch ( std::exception e ){ if ( inShutdown() ){ // we're shutting down, so just clean up return; } log() << "WriteBackListener exception : " << e.what() << endl; // It's possible this shard was removed Shard::reloadShardInfo(); } catch ( ... ){ log() << "WriteBackListener uncaught exception!" << endl; } secsToSleep++; sleepsecs(secsToSleep); if ( secsToSleep > 10 ) secsToSleep = 0; } log() << "WriteBackListener exiting : address no longer in cluster " << _addr; }
void addWriteBack( vector<OID>& all , const BSONObj& o ){ BSONElement e = o["writeback"]; if ( e.type() == jstOID ) all.push_back( e.OID() ); }
/* **************************************************************************** * * addTriggeredSubscriptions - */ static bool addTriggeredSubscriptions ( ContextRegistration cr, map<string, TriggeredSubscription*>& subs, std::string& err, std::string tenant ) { BSONArrayBuilder entitiesNoPatternA; std::vector<std::string> idJsV; std::vector<std::string> typeJsV; for (unsigned int ix = 0; ix < cr.entityIdVector.size(); ++ix) { // FIXME: take into account subscriptions with no type EntityId* enP = cr.entityIdVector[ix]; // The registration of isPattern=true entities is not supported, so we don't include them here if (enP->isPattern == "false") { entitiesNoPatternA.append(BSON(CASUB_ENTITY_ID << enP->id << CASUB_ENTITY_TYPE << enP->type << CASUB_ENTITY_ISPATTERN << "false")); idJsV.push_back(enP->id); typeJsV.push_back(enP->type); } } BSONArrayBuilder attrA; for (unsigned int ix = 0; ix < cr.contextRegistrationAttributeVector.size(); ++ix) { ContextRegistrationAttribute* craP = cr.contextRegistrationAttributeVector[ix]; attrA.append(craP->name); } BSONObjBuilder queryNoPattern; queryNoPattern.append(CASUB_ENTITIES, BSON("$in" << entitiesNoPatternA.arr())); if (attrA.arrSize() > 0) { // If we don't do this checking, the {$in: [] } in the attribute name part will // make the query fail // // queryB.append(CASUB_ATTRS, BSON("$in" << attrA.arr())); queryNoPattern.append("$or", BSON_ARRAY( BSON(CASUB_ATTRS << BSON("$in" << attrA.arr())) << BSON(CASUB_ATTRS << BSON("$size" << 0)))); } else { queryNoPattern.append(CASUB_ATTRS, BSON("$size" << 0)); } queryNoPattern.append(CASUB_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); // // This is JavaScript code that runs in MongoDB engine. As far as I know, this is the only // way to do a "reverse regex" query in MongoDB (see // http://stackoverflow.com/questions/15966991/mongodb-reverse-regex/15989520). // Note that although we are using a isPattern=true in the MongoDB query besides $where, we // also need to check that in the if statement in the JavaScript function given that a given // sub document could include both isPattern=true and isPattern=false documents // std::string idJsString = "[ "; for (unsigned int ix = 0; ix < idJsV.size(); ++ix) { if (ix != idJsV.size() - 1) { idJsString += "\""+idJsV[ix]+ "\" ,"; } else { idJsString += "\"" +idJsV[ix]+ "\""; } } idJsString += " ]"; std::string typeJsString = "[ "; for (unsigned int ix = 0; ix < typeJsV.size(); ++ix) { if (ix != typeJsV.size() - 1) { typeJsString += "\"" +typeJsV[ix] + "\" ,"; } else { typeJsString += "\"" + typeJsV[ix] + "\""; } } typeJsString += " ]"; std::string function = std::string("function()") + "{" + "enId = " + idJsString + ";" + "enType = " + typeJsString + ";" + "for (var i=0; i < this." + CASUB_ENTITIES + ".length; i++) {" + "if (this." + CASUB_ENTITIES + "[i]." + CASUB_ENTITY_ISPATTERN + " == \"true\") {" + "for (var j = 0; j < enId.length; j++) {" + "if (enId[j].match(this." + CASUB_ENTITIES + "[i]." + CASUB_ENTITY_ID + ") && this." + CASUB_ENTITIES + "[i]." + CASUB_ENTITY_TYPE + " == enType[j]) {" + "return true; " + "}" + "}" + "}" + "}" + "return false; " + "}"; LM_T(LmtMongo, ("JS function: %s", function.c_str())); std::string entPatternQ = CSUB_ENTITIES "." CSUB_ENTITY_ISPATTERN; BSONObjBuilder queryPattern; queryPattern.append(entPatternQ, "true"); queryPattern.append(CASUB_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); queryPattern.appendCode("$where", function); auto_ptr<DBClientCursor> cursor; BSONObj query = BSON("$or" << BSON_ARRAY(queryNoPattern.obj() << queryPattern.obj())); TIME_STAT_MONGO_READ_WAIT_START(); DBClientBase* connection = getMongoConnection(); if (!collectionQuery(connection, getSubscribeContextAvailabilityCollectionName(tenant), query, &cursor, &err)) { TIME_STAT_MONGO_READ_WAIT_STOP(); releaseMongoConnection(connection); return false; } TIME_STAT_MONGO_READ_WAIT_STOP(); /* For each one of the subscriptions found, add it to the map (if not already there) */ while (moreSafe(cursor)) { BSONObj sub; std::string err; if (!nextSafeOrErrorF(cursor, &sub, &err)) { LM_E(("Runtime Error (exception in nextSafe(): %s - query: %s)", err.c_str(), query.toString().c_str())); continue; } BSONElement idField = getFieldF(sub, "_id"); // // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub' // // Now, if 'getFieldF(sub, "_id")' is not found, if we continue, calling OID() on it, then we get // an exception and the broker crashes. // if (idField.eoo() == true) { std::string details = std::string("error retrieving _id field in doc: '") + sub.toString() + "'"; alarmMgr.dbError(details); continue; } alarmMgr.dbErrorReset(); std::string subIdStr = idField.OID().toString(); if (subs.count(subIdStr) == 0) { LM_T(LmtMongo, ("adding subscription: '%s'", sub.toString().c_str())); // // FIXME P4: Once ctx availability notification formats get defined for NGSIv2, // the first parameter for TriggeredSubscription will have "normalized" as default value // TriggeredSubscription* trigs = new TriggeredSubscription( sub.hasField(CASUB_FORMAT)? stringToRenderFormat(getStringFieldF(sub, CASUB_FORMAT)) : NGSI_V1_LEGACY, getStringFieldF(sub, CASUB_REFERENCE), subToAttributeList(sub)); subs.insert(std::pair<string, TriggeredSubscription*>(subIdStr, trigs)); } } releaseMongoConnection(connection); return true; }
/* **************************************************************************** * * addTriggeredSubscriptions * */ static bool addTriggeredSubscriptions(ContextRegistration cr, map<string, TriggeredSubscription*>& subs, std::string& err, std::string tenant) { DBClientBase* connection = NULL; BSONArrayBuilder entitiesNoPatternA; std::vector<std::string> idJsV; std::vector<std::string> typeJsV; for (unsigned int ix = 0; ix < cr.entityIdVector.size(); ++ix ) { //FIXME: take into account subscriptions with no type EntityId* enP = cr.entityIdVector.get(ix); /* The registration of isPattern=true entities is not supported, so we don't include them here */ if (enP->isPattern == "false") { entitiesNoPatternA.append(BSON(CASUB_ENTITY_ID << enP->id << CASUB_ENTITY_TYPE << enP->type << CASUB_ENTITY_ISPATTERN << "false")); idJsV.push_back(enP->id); typeJsV.push_back(enP->type); } } BSONArrayBuilder attrA; for (unsigned int ix = 0; ix < cr.contextRegistrationAttributeVector.size(); ++ix) { ContextRegistrationAttribute* craP = cr.contextRegistrationAttributeVector.get(ix); attrA.append(craP->name); } BSONObjBuilder queryNoPattern; queryNoPattern.append(CASUB_ENTITIES, BSON("$in" << entitiesNoPatternA.arr())); if (attrA.arrSize() > 0) { /* If we don't do this checking, the {$in: [] } in the attribute name part will * make the query fail*/ //queryB.append(CASUB_ATTRS, BSON("$in" << attrA.arr())); queryNoPattern.append("$or", BSON_ARRAY( BSON(CASUB_ATTRS << BSON("$in" << attrA.arr())) << BSON(CASUB_ATTRS << BSON("$size" << 0)) )); } else { queryNoPattern.append(CASUB_ATTRS, BSON("$size" << 0)); } queryNoPattern.append(CASUB_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); /* This is JavaScript code that runs in MongoDB engine. As far as I know, this is the only * way to do a "reverse regex" query in MongoDB (see * http://stackoverflow.com/questions/15966991/mongodb-reverse-regex/15989520). * Note that although we are using a isPattern=true in the MongoDB query besides $where, we * also need to check that in the if statement in the JavaScript function given that a given * sub document could include both isPattern=true and isPattern=false documents */ std::string idJsString = "[ "; for (unsigned int ix = 0; ix < idJsV.size(); ++ix ) { if (ix != idJsV.size()-1) { idJsString += "\""+idJsV[ix]+ "\" ,"; } else { idJsString += "\"" +idJsV[ix]+ "\""; } } idJsString += " ]"; std::string typeJsString = "[ "; for (unsigned int ix = 0; ix < typeJsV.size(); ++ix ) { if (ix != typeJsV.size()-1) { typeJsString += "\"" +typeJsV[ix] + "\" ,"; } else { typeJsString += "\"" + typeJsV[ix] + "\""; } } typeJsString += " ]"; std::string function = std::string("function()") + "{" + "enId = "+idJsString+ ";" + "enType = "+typeJsString+ ";" + "for (var i=0; i < this."+CASUB_ENTITIES+".length; i++) {" + "if (this."+CASUB_ENTITIES+"[i]."+CASUB_ENTITY_ISPATTERN+" == \"true\") {" + "for (var j=0; j < enId.length; j++) {" + "if (enId[j].match(this."+CASUB_ENTITIES+"[i]."+CASUB_ENTITY_ID+") && this."+CASUB_ENTITIES+"[i]."+CASUB_ENTITY_TYPE+" == enType[j]) {" + "return true; " + "}" + "}" + "}" + "}" + "return false; " + "}"; LM_T(LmtMongo, ("JS function: %s", function.c_str())); std::string entPatternQ = CSUB_ENTITIES "." CSUB_ENTITY_ISPATTERN; BSONObjBuilder queryPattern; queryPattern.append(entPatternQ, "true"); queryPattern.append(CASUB_EXPIRATION, BSON("$gt" << (long long) getCurrentTime())); queryPattern.appendCode("$where", function); BSONObj query = BSON("$or" << BSON_ARRAY(queryNoPattern.obj() << queryPattern.obj())); /* Do the query */ auto_ptr<DBClientCursor> cursor; LM_T(LmtMongo, ("query() in '%s' collection: '%s'", getSubscribeContextAvailabilityCollectionName(tenant).c_str(), query.toString().c_str())); try { connection = getMongoConnection(); cursor = connection->query(getSubscribeContextAvailabilityCollectionName(tenant).c_str(), query); /* * We have observed that in some cases of DB errors (e.g. the database daemon is down) instead of * raising an exception, the query() method sets the cursor to NULL. In this case, we raise the * exception ourselves */ if (cursor.get() == NULL) { throw DBException("Null cursor from mongo (details on this is found in the source code)", 0); } releaseMongoConnection(connection); LM_I(("Database Operation Successful (%s)", query.toString().c_str())); } catch (const DBException &e) { releaseMongoConnection(connection); err = std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - query(): " + query.toString() + " - exception: " + e.what(); LM_E(("Database Error (%s)", err.c_str())); return false; } catch (...) { releaseMongoConnection(connection); err = std::string("collection: ") + getSubscribeContextAvailabilityCollectionName(tenant).c_str() + " - query(): " + query.toString() + " - exception: " + "generic"; LM_E(("Database Error (%s)", err.c_str())); return false; } /* For each one of the subscriptions found, add it to the map (if not already there) */ while (cursor->more()) { BSONObj sub = cursor->next(); BSONElement idField = sub.getField("_id"); // // BSONElement::eoo returns true if 'not found', i.e. the field "_id" doesn't exist in 'sub' // // Now, if 'sub.getField("_id")' is not found, if we continue, calling OID() on it, then we get // an exception and the broker crashes. // if (idField.eoo() == true) { LM_E(("Database Error (error retrieving _id field in doc: %s)", sub.toString().c_str())); continue; } std::string subIdStr = idField.OID().toString(); if (subs.count(subIdStr) == 0) { LM_T(LmtMongo, ("adding subscription: '%s'", sub.toString().c_str())); TriggeredSubscription* trigs = new TriggeredSubscription(sub.hasField(CASUB_FORMAT) ? stringToFormat(STR_FIELD(sub, CASUB_FORMAT)) : XML, STR_FIELD(sub, CASUB_REFERENCE), subToAttributeList(sub)); subs.insert(std::pair<string, TriggeredSubscription*>(subIdStr, trigs)); } } return true; }