BSONObj DBClientCursor::next() { if (!_putBack.empty()) { BSONObj ret = _putBack.top(); _putBack.pop(); return ret; } uassert(13422, "DBClientCursor next() called but more() is false", batch.pos < batch.nReturned); uassert(ErrorCodes::InvalidBSON, "Got invalid BSON from external server while reading from cursor.", validateBSON(batch.data, batch.remainingBytes, enableBSON1_1 ? BSONVersion::kV1_1 : BSONVersion::kV1_0) .isOK()); BSONObj o(batch.data); batch.pos++; batch.data += o.objsize(); batch.remainingBytes -= o.objsize(); /* todo would be good to make data null at end of batch for safety */ return o; }
/** * data and len must be the arguments from RecordStore::insert() on an oplog collection. */ StatusWith<RecordId> extractKey(const char* data, int len) { DEV invariant(validateBSON(data, len).isOK()); const BSONObj obj(data); const BSONElement elem = obj["ts"]; if (elem.eoo()) return StatusWith<RecordId>(ErrorCodes::BadValue, "no ts field"); if (elem.type() != bsonTimestamp) return StatusWith<RecordId>(ErrorCodes::BadValue, "ts must be a Timestamp"); return keyForOptime(elem.timestamp()); }
/** * data and len must be the arguments from RecordStore::insert() on an oplog collection. */ StatusWith<RecordId> extractKey(const char* data, int len) { // Use the latest BSON validation version. Oplog entries are allowed to contain decimal data // even if decimal is disabled. DEV invariant(validateBSON(data, len, BSONVersion::kLatest).isOK()); const BSONObj obj(data); const BSONElement elem = obj["ts"]; if (elem.eoo()) return StatusWith<RecordId>(ErrorCodes::BadValue, "no ts field"); if (elem.type() != bsonTimestamp) return StatusWith<RecordId>(ErrorCodes::BadValue, "ts must be a Timestamp"); return keyForOptime(elem.timestamp()); }
Status RecordStoreValidateAdaptor::validate(const RecordId& recordId, const RecordData& record, size_t* dataSize) { BSONObj recordBson = record.toBson(); const Status status = validateBSON( recordBson.objdata(), recordBson.objsize(), Validator<BSONObj>::enabledBSONVersion()); if (status.isOK()) { *dataSize = recordBson.objsize(); } else { return status; } if (!_indexCatalog->haveAnyIndexes()) { return status; } IndexCatalog::IndexIterator i = _indexCatalog->getIndexIterator(_opCtx, false); while (i.more()) { const IndexDescriptor* descriptor = i.next(); const std::string indexNs = descriptor->indexNamespace(); int indexNumber = _indexConsistency->getIndexNumber(indexNs); ValidateResults curRecordResults; const IndexAccessMethod* iam = _indexCatalog->getIndex(descriptor); if (descriptor->isPartial()) { const IndexCatalogEntry* ice = _indexCatalog->getEntry(descriptor); if (!ice->getFilterExpression()->matchesBSON(recordBson)) { (*_indexNsResultsMap)[indexNs] = curRecordResults; continue; } } BSONObjSet documentKeySet = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); BSONObjSet multikeyMetadataKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); MultikeyPaths multikeyPaths; iam->getKeys(recordBson, IndexAccessMethod::GetKeysMode::kEnforceConstraints, &documentKeySet, &multikeyMetadataKeys, &multikeyPaths); if (!descriptor->isMultikey(_opCtx) && iam->shouldMarkIndexAsMultikey(documentKeySet, multikeyMetadataKeys, multikeyPaths)) { std::string msg = str::stream() << "Index " << descriptor->indexName() << " is not multi-key, but a multikey path " << " is present in document " << recordId; curRecordResults.errors.push_back(msg); curRecordResults.valid = false; } for (const auto& key : multikeyMetadataKeys) { _indexConsistency->addMultikeyMetadataPath(makeWildCardMultikeyMetadataKeyString(key), indexNumber); } const auto& pattern = descriptor->keyPattern(); const Ordering ord = Ordering::make(pattern); bool largeKeyDisallowed = isLargeKeyDisallowed(); for (const auto& key : documentKeySet) { if (largeKeyDisallowed && key.objsize() >= static_cast<int64_t>(KeyString::TypeBits::kMaxKeyBytes)) { // Index keys >= 1024 bytes are not indexed. _indexConsistency->addLongIndexKey(indexNumber); continue; } // We want to use the latest version of KeyString here. KeyString ks(KeyString::kLatestVersion, key, ord, recordId); _indexConsistency->addDocKey(ks, indexNumber); } (*_indexNsResultsMap)[indexNs] = curRecordResults; } return status; }
long long BSONTool::processFile( const boost::filesystem::path& root ) { std::string fileName = root.string(); unsigned long long fileLength = file_size( root ); if ( fileLength == 0 ) { toolInfoOutput() << "file " << fileName << " empty, skipping" << std::endl; return 0; } FILE* file = fopen( fileName.c_str() , "rb" ); if ( ! file ) { toolError() << "error opening file: " << fileName << " " << errnoWithDescription() << std::endl; return 0; } #ifdef POSIX_FADV_SEQUENTIAL posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL); #endif if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { toolInfoOutput() << "\t file size: " << fileLength << std::endl; } unsigned long long read = 0; unsigned long long num = 0; unsigned long long processed = 0; const int BUF_SIZE = BSONObjMaxUserSize + ( 1024 * 1024 ); boost::scoped_array<char> buf_holder(new char[BUF_SIZE]); char * buf = buf_holder.get(); ProgressMeter m(fileLength); if (!toolGlobalParams.quiet) { m.setUnits( "bytes" ); } while ( read < fileLength ) { size_t amt = fread(buf, 1, 4, file); verify( amt == 4 ); int size = ((int*)buf)[0]; uassert( 10264 , str::stream() << "invalid object size: " << size , size < BUF_SIZE ); amt = fread(buf+4, 1, size-4, file); verify( amt == (size_t)( size - 4 ) ); BSONObj o( buf ); if (bsonToolGlobalParams.objcheck) { const Status status = validateBSON(buf, size); if (!status.isOK()) { toolError() << "INVALID OBJECT - going to try and print out " << std::endl; toolError() << "size: " << size << std::endl; toolError() << "error: " << status.reason() << std::endl; StringBuilder sb; try { o.toString(sb); // using StringBuilder version to get as much as possible } catch (...) { toolError() << "object up to error: " << sb.str() << endl; throw; } toolError() << "complete object: " << sb.str() << endl; // NOTE: continuing with object even though we know it is invalid. } } if (!bsonToolGlobalParams.hasFilter || _matcher->matches(o)) { gotObject( o ); processed++; } read += o.objsize(); num++; if (!toolGlobalParams.quiet) { m.hit(o.objsize()); } } fclose( file ); uassert(10265, "counts don't match", read == fileLength); toolInfoOutput() << num << ((num == 1) ? " document" : " documents") << " found" << std::endl; if (bsonToolGlobalParams.hasFilter) { toolInfoOutput() << processed << ((processed == 1) ? " document" : " documents") << " processed" << std::endl; } return processed; }
void operator()( DBClientCursorBatchIterator &i ) { Lock::GlobalWrite lk; context.relocked(); bool createdCollection = false; Collection* collection = NULL; while( i.moreInCurrentBatch() ) { if ( numSeen % 128 == 127 /*yield some*/ ) { collection = NULL; time_t now = time(0); if( now - lastLog >= 60 ) { // report progress if( lastLog ) log() << "clone " << to_collection << ' ' << numSeen << endl; lastLog = now; } mayInterrupt( _mayBeInterrupted ); dbtempreleaseif t( _mayYield ); } if ( isindex == false && collection == NULL ) { collection = context.db()->getCollection( to_collection ); if ( !collection ) { massert( 17321, str::stream() << "collection dropped during clone [" << to_collection << "]", !createdCollection ); createdCollection = true; collection = context.db()->createCollection( txn, to_collection ); verify( collection ); } } BSONObj tmp = i.nextSafe(); /* assure object is valid. note this will slow us down a little. */ const Status status = validateBSON(tmp.objdata(), tmp.objsize()); if (!status.isOK()) { out() << "Cloner: skipping corrupt object from " << from_collection << ": " << status.reason(); continue; } ++numSeen; BSONObj js = tmp; if ( isindex ) { verify(nsToCollectionSubstring(from_collection) == "system.indexes"); js = fixindex(context.db()->name(), tmp); indexesToBuild->push_back( js.getOwned() ); continue; } verify(nsToCollectionSubstring(from_collection) != "system.indexes"); StatusWith<DiskLoc> loc = collection->insertDocument( txn, js, true ); if ( !loc.isOK() ) { error() << "error: exception cloning object in " << from_collection << ' ' << loc.toString() << " obj:" << js; } uassertStatusOK( loc.getStatus() ); if ( logForRepl ) logOp(txn, "i", to_collection, js); getDur().commitIfNeeded(); RARELY if ( time( 0 ) - saveLast > 60 ) { log() << numSeen << " objects cloned so far from collection " << from_collection; saveLast = time( 0 ); } } }
bool BSONObj::valid() const { return validateBSON( objdata(), objsize() ).isOK(); }
Status Validator<BSONObj>::validateLoad(const char* ptr, size_t length) { return validateBSON(ptr, length); }
Status DurableViewCatalogImpl::iterate(OperationContext* opCtx, Callback callback) { dassert(opCtx->lockState()->isDbLockedForMode(_db->name(), MODE_IS) || opCtx->lockState()->isDbLockedForMode(_db->name(), MODE_IX)); Collection* systemViews = _db->getCollection(opCtx, _db->getSystemViewsName()); if (!systemViews) return Status::OK(); Lock::CollectionLock lk(opCtx->lockState(), _db->getSystemViewsName(), MODE_IS); auto cursor = systemViews->getCursor(opCtx); while (auto record = cursor->next()) { RecordData& data = record->data; // Check the document is valid BSON, with only the expected fields. // Use the latest BSON validation version. Existing view definitions are allowed to contain // decimal data even if decimal is disabled. fassert(40224, validateBSON(data.data(), data.size(), BSONVersion::kLatest)); BSONObj viewDef = data.toBson(); // Check read definitions for correct structure, and refuse reading past invalid // definitions. Ignore any further view definitions. bool valid = true; for (const BSONElement& e : viewDef) { std::string name(e.fieldName()); valid &= name == "_id" || name == "viewOn" || name == "pipeline" || name == "collation"; } const auto viewName = viewDef["_id"].str(); const auto viewNameIsValid = NamespaceString::validCollectionComponent(viewName) && NamespaceString::validDBName(nsToDatabaseSubstring(viewName)); valid &= viewNameIsValid; // Only perform validation via NamespaceString if the collection name has been determined to // be valid. If not valid then the NamespaceString constructor will uassert. if (viewNameIsValid) { NamespaceString viewNss(viewName); valid &= viewNss.isValid() && viewNss.db() == _db->name(); } valid &= NamespaceString::validCollectionName(viewDef["viewOn"].str()); const bool hasPipeline = viewDef.hasField("pipeline"); valid &= hasPipeline; if (hasPipeline) { valid &= viewDef["pipeline"].type() == mongo::Array; } valid &= (!viewDef.hasField("collation") || viewDef["collation"].type() == BSONType::Object); if (!valid) { return {ErrorCodes::InvalidViewDefinition, str::stream() << "found invalid view definition " << viewDef["_id"] << " while reading '" << _db->getSystemViewsName() << "'"}; } Status callbackStatus = callback(viewDef); if (!callbackStatus.isOK()) { return callbackStatus; } } return Status::OK(); }
Status Validator<BSONObj>::validateLoad(const char* ptr, size_t length) { return serverGlobalParams.objcheck ? validateBSON(ptr, length) : Status::OK(); }