BSONObj DBClientCursor::next() {
if (!_putBack.empty()) {
BSONObj ret = _putBack.top();
_putBack.pop();
return ret;
}
uassert(13422, "DBClientCursor next() called but more() is false", batch.pos < batch.nReturned);
uassert(ErrorCodes::InvalidBSON,
"Got invalid BSON from external server while reading from cursor.",
validateBSON(batch.data,
batch.remainingBytes,
enableBSON1_1 ? BSONVersion::kV1_1 : BSONVersion::kV1_0)
.isOK());
BSONObj o(batch.data);
batch.pos++;
batch.data += o.objsize();
batch.remainingBytes -= o.objsize();
/* todo would be good to make data null at end of batch for safety */
return o;
}
/**
* data and len must be the arguments from RecordStore::insert() on an oplog collection.
*/
StatusWith<RecordId> extractKey(const char* data, int len) {
DEV invariant(validateBSON(data, len).isOK());
const BSONObj obj(data);
const BSONElement elem = obj["ts"];
if (elem.eoo())
return StatusWith<RecordId>(ErrorCodes::BadValue, "no ts field");
if (elem.type() != bsonTimestamp)
return StatusWith<RecordId>(ErrorCodes::BadValue, "ts must be a Timestamp");
return keyForOptime(elem.timestamp());
}
/**
* data and len must be the arguments from RecordStore::insert() on an oplog collection.
*/
StatusWith<RecordId> extractKey(const char* data, int len) {
// Use the latest BSON validation version. Oplog entries are allowed to contain decimal data
// even if decimal is disabled.
DEV invariant(validateBSON(data, len, BSONVersion::kLatest).isOK());
const BSONObj obj(data);
const BSONElement elem = obj["ts"];
if (elem.eoo())
return StatusWith<RecordId>(ErrorCodes::BadValue, "no ts field");
if (elem.type() != bsonTimestamp)
return StatusWith<RecordId>(ErrorCodes::BadValue, "ts must be a Timestamp");
return keyForOptime(elem.timestamp());
}
Status RecordStoreValidateAdaptor::validate(const RecordId& recordId,
const RecordData& record,
size_t* dataSize) {
BSONObj recordBson = record.toBson();
const Status status = validateBSON(
recordBson.objdata(), recordBson.objsize(), Validator<BSONObj>::enabledBSONVersion());
if (status.isOK()) {
*dataSize = recordBson.objsize();
} else {
return status;
}
if (!_indexCatalog->haveAnyIndexes()) {
return status;
}
IndexCatalog::IndexIterator i = _indexCatalog->getIndexIterator(_opCtx, false);
while (i.more()) {
const IndexDescriptor* descriptor = i.next();
const std::string indexNs = descriptor->indexNamespace();
int indexNumber = _indexConsistency->getIndexNumber(indexNs);
ValidateResults curRecordResults;
const IndexAccessMethod* iam = _indexCatalog->getIndex(descriptor);
if (descriptor->isPartial()) {
const IndexCatalogEntry* ice = _indexCatalog->getEntry(descriptor);
if (!ice->getFilterExpression()->matchesBSON(recordBson)) {
(*_indexNsResultsMap)[indexNs] = curRecordResults;
continue;
}
}
BSONObjSet documentKeySet = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
BSONObjSet multikeyMetadataKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
MultikeyPaths multikeyPaths;
iam->getKeys(recordBson,
IndexAccessMethod::GetKeysMode::kEnforceConstraints,
&documentKeySet,
&multikeyMetadataKeys,
&multikeyPaths);
if (!descriptor->isMultikey(_opCtx) &&
iam->shouldMarkIndexAsMultikey(documentKeySet, multikeyMetadataKeys, multikeyPaths)) {
std::string msg = str::stream() << "Index " << descriptor->indexName()
<< " is not multi-key, but a multikey path "
<< " is present in document " << recordId;
curRecordResults.errors.push_back(msg);
curRecordResults.valid = false;
}
for (const auto& key : multikeyMetadataKeys) {
_indexConsistency->addMultikeyMetadataPath(makeWildCardMultikeyMetadataKeyString(key),
indexNumber);
}
const auto& pattern = descriptor->keyPattern();
const Ordering ord = Ordering::make(pattern);
bool largeKeyDisallowed = isLargeKeyDisallowed();
for (const auto& key : documentKeySet) {
if (largeKeyDisallowed &&
key.objsize() >= static_cast<int64_t>(KeyString::TypeBits::kMaxKeyBytes)) {
// Index keys >= 1024 bytes are not indexed.
_indexConsistency->addLongIndexKey(indexNumber);
continue;
}
// We want to use the latest version of KeyString here.
KeyString ks(KeyString::kLatestVersion, key, ord, recordId);
_indexConsistency->addDocKey(ks, indexNumber);
}
(*_indexNsResultsMap)[indexNs] = curRecordResults;
}
return status;
}
long long BSONTool::processFile( const boost::filesystem::path& root ) {
std::string fileName = root.string();
unsigned long long fileLength = file_size( root );
if ( fileLength == 0 ) {
toolInfoOutput() << "file " << fileName << " empty, skipping" << std::endl;
return 0;
}
FILE* file = fopen( fileName.c_str() , "rb" );
if ( ! file ) {
toolError() << "error opening file: " << fileName << " " << errnoWithDescription()
<< std::endl;
return 0;
}
#ifdef POSIX_FADV_SEQUENTIAL
posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL);
#endif
if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) {
toolInfoOutput() << "\t file size: " << fileLength << std::endl;
}
unsigned long long read = 0;
unsigned long long num = 0;
unsigned long long processed = 0;
const int BUF_SIZE = BSONObjMaxUserSize + ( 1024 * 1024 );
boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
char * buf = buf_holder.get();
ProgressMeter m(fileLength);
if (!toolGlobalParams.quiet) {
m.setUnits( "bytes" );
}
while ( read < fileLength ) {
size_t amt = fread(buf, 1, 4, file);
verify( amt == 4 );
int size = ((int*)buf)[0];
uassert( 10264 , str::stream() << "invalid object size: " << size , size < BUF_SIZE );
amt = fread(buf+4, 1, size-4, file);
verify( amt == (size_t)( size - 4 ) );
BSONObj o( buf );
if (bsonToolGlobalParams.objcheck) {
const Status status = validateBSON(buf, size);
if (!status.isOK()) {
toolError() << "INVALID OBJECT - going to try and print out " << std::endl;
toolError() << "size: " << size << std::endl;
toolError() << "error: " << status.reason() << std::endl;
StringBuilder sb;
try {
o.toString(sb); // using StringBuilder version to get as much as possible
} catch (...) {
toolError() << "object up to error: " << sb.str() << endl;
throw;
}
toolError() << "complete object: " << sb.str() << endl;
// NOTE: continuing with object even though we know it is invalid.
}
}
if (!bsonToolGlobalParams.hasFilter || _matcher->matches(o)) {
gotObject( o );
processed++;
}
read += o.objsize();
num++;
if (!toolGlobalParams.quiet) {
m.hit(o.objsize());
}
}
fclose( file );
uassert(10265, "counts don't match", read == fileLength);
toolInfoOutput() << num << ((num == 1) ? " document" : " documents")
<< " found" << std::endl;
if (bsonToolGlobalParams.hasFilter) {
toolInfoOutput() << processed
<< ((processed == 1) ? " document" : " documents")
<< " processed" << std::endl;
}
return processed;
}
void operator()( DBClientCursorBatchIterator &i ) {
Lock::GlobalWrite lk;
context.relocked();
bool createdCollection = false;
Collection* collection = NULL;
while( i.moreInCurrentBatch() ) {
if ( numSeen % 128 == 127 /*yield some*/ ) {
collection = NULL;
time_t now = time(0);
if( now - lastLog >= 60 ) {
// report progress
if( lastLog )
log() << "clone " << to_collection << ' ' << numSeen << endl;
lastLog = now;
}
mayInterrupt( _mayBeInterrupted );
dbtempreleaseif t( _mayYield );
}
if ( isindex == false && collection == NULL ) {
collection = context.db()->getCollection( to_collection );
if ( !collection ) {
massert( 17321,
str::stream()
<< "collection dropped during clone ["
<< to_collection << "]",
!createdCollection );
createdCollection = true;
collection = context.db()->createCollection( txn, to_collection );
verify( collection );
}
}
BSONObj tmp = i.nextSafe();
/* assure object is valid. note this will slow us down a little. */
const Status status = validateBSON(tmp.objdata(), tmp.objsize());
if (!status.isOK()) {
out() << "Cloner: skipping corrupt object from " << from_collection
<< ": " << status.reason();
continue;
}
++numSeen;
BSONObj js = tmp;
if ( isindex ) {
verify(nsToCollectionSubstring(from_collection) == "system.indexes");
js = fixindex(context.db()->name(), tmp);
indexesToBuild->push_back( js.getOwned() );
continue;
}
verify(nsToCollectionSubstring(from_collection) != "system.indexes");
StatusWith<DiskLoc> loc = collection->insertDocument( txn, js, true );
if ( !loc.isOK() ) {
error() << "error: exception cloning object in " << from_collection
<< ' ' << loc.toString() << " obj:" << js;
}
uassertStatusOK( loc.getStatus() );
if ( logForRepl )
logOp(txn, "i", to_collection, js);
getDur().commitIfNeeded();
RARELY if ( time( 0 ) - saveLast > 60 ) {
log() << numSeen << " objects cloned so far from collection " << from_collection;
saveLast = time( 0 );
}
}
}
bool BSONObj::valid() const {
return validateBSON( objdata(), objsize() ).isOK();
}
Status Validator<BSONObj>::validateLoad(const char* ptr, size_t length) {
return validateBSON(ptr, length);
}
Status DurableViewCatalogImpl::iterate(OperationContext* opCtx, Callback callback) {
dassert(opCtx->lockState()->isDbLockedForMode(_db->name(), MODE_IS) ||
opCtx->lockState()->isDbLockedForMode(_db->name(), MODE_IX));
Collection* systemViews = _db->getCollection(opCtx, _db->getSystemViewsName());
if (!systemViews)
return Status::OK();
Lock::CollectionLock lk(opCtx->lockState(), _db->getSystemViewsName(), MODE_IS);
auto cursor = systemViews->getCursor(opCtx);
while (auto record = cursor->next()) {
RecordData& data = record->data;
// Check the document is valid BSON, with only the expected fields.
// Use the latest BSON validation version. Existing view definitions are allowed to contain
// decimal data even if decimal is disabled.
fassert(40224, validateBSON(data.data(), data.size(), BSONVersion::kLatest));
BSONObj viewDef = data.toBson();
// Check read definitions for correct structure, and refuse reading past invalid
// definitions. Ignore any further view definitions.
bool valid = true;
for (const BSONElement& e : viewDef) {
std::string name(e.fieldName());
valid &= name == "_id" || name == "viewOn" || name == "pipeline" || name == "collation";
}
const auto viewName = viewDef["_id"].str();
const auto viewNameIsValid = NamespaceString::validCollectionComponent(viewName) &&
NamespaceString::validDBName(nsToDatabaseSubstring(viewName));
valid &= viewNameIsValid;
// Only perform validation via NamespaceString if the collection name has been determined to
// be valid. If not valid then the NamespaceString constructor will uassert.
if (viewNameIsValid) {
NamespaceString viewNss(viewName);
valid &= viewNss.isValid() && viewNss.db() == _db->name();
}
valid &= NamespaceString::validCollectionName(viewDef["viewOn"].str());
const bool hasPipeline = viewDef.hasField("pipeline");
valid &= hasPipeline;
if (hasPipeline) {
valid &= viewDef["pipeline"].type() == mongo::Array;
}
valid &=
(!viewDef.hasField("collation") || viewDef["collation"].type() == BSONType::Object);
if (!valid) {
return {ErrorCodes::InvalidViewDefinition,
str::stream() << "found invalid view definition " << viewDef["_id"]
<< " while reading '"
<< _db->getSystemViewsName()
<< "'"};
}
Status callbackStatus = callback(viewDef);
if (!callbackStatus.isOK()) {
return callbackStatus;
}
}
return Status::OK();
}
Status Validator<BSONObj>::validateLoad(const char* ptr, size_t length) {
return serverGlobalParams.objcheck ? validateBSON(ptr, length) : Status::OK();
}
