void Projection::init( const BSONObj& o ) {
massert( 10371 , "can only add to Projection once", _source.isEmpty());
_source = o;
BSONObjIterator i( o );
int true_false = -1;
while ( i.more() ) {
BSONElement e = i.next();
if ( ! e.isNumber() )
_hasNonSimple = true;
if (e.type() == Object) {
BSONObj obj = e.embeddedObject();
BSONElement e2 = obj.firstElement();
if ( mongoutils::str::equals( e2.fieldName(), "$slice" ) ) {
if (e2.isNumber()) {
int i = e2.numberInt();
if (i < 0)
add(e.fieldName(), i, -i); // limit is now positive
else
add(e.fieldName(), 0, i);
}
else if (e2.type() == Array) {
BSONObj arr = e2.embeddedObject();
uassert(13099, "$slice array wrong size", arr.nFields() == 2 );
BSONObjIterator it(arr);
int skip = it.next().numberInt();
int limit = it.next().numberInt();
uassert(13100, "$slice limit must be positive", limit > 0 );
add(e.fieldName(), skip, limit);
}
else {
uassert(13098, "$slice only supports numbers and [skip, limit] arrays", false);
}
}
else if ( mongoutils::str::equals( e2.fieldName(), "$elemMatch" ) ) {
// validate $elemMatch arguments and dependencies
uassert( 16342, "elemMatch: invalid argument. object required.",
e2.type() == Object );
uassert( 16343, "Cannot specify positional operator and $elemMatch"
" (currently unsupported).",
_arrayOpType != ARRAY_OP_POSITIONAL );
uassert( 16344, "Cannot use $elemMatch projection on a nested field"
" (currently unsupported).",
! mongoutils::str::contains( e.fieldName(), '.' ) );
_arrayOpType = ARRAY_OP_ELEM_MATCH;
// initialize new Matcher object(s)
_matchers.insert( make_pair( mongoutils::str::before( e.fieldName(), '.' ),
boost::make_shared<Matcher>( e.wrap(), true ) ) );
add( e.fieldName(), true );
}
else {
uasserted(13097, string("Unsupported projection option: ") +
obj.firstElementFieldName() );
}
}
else if (!strcmp(e.fieldName(), "_id") && !e.trueValue()) {
_includeID = false;
}
else {
add( e.fieldName(), e.trueValue() );
// validate input
if (true_false == -1) {
true_false = e.trueValue();
_include = !e.trueValue();
}
else {
uassert( 10053 , "You cannot currently mix including and excluding fields. "
"Contact us if this is an issue." ,
(bool)true_false == e.trueValue() );
}
}
if ( mongoutils::str::contains( e.fieldName(), ".$" ) ) {
// positional op found; verify dependencies
uassert( 16345, "Cannot exclude array elements with the positional operator"
" (currently unsupported).", e.trueValue() );
uassert( 16346, "Cannot specify more than one positional array element per query"
" (currently unsupported).", _arrayOpType != ARRAY_OP_POSITIONAL );
uassert( 16347, "Cannot specify positional operator and $elemMatch"
" (currently unsupported).", _arrayOpType != ARRAY_OP_ELEM_MATCH );
_arrayOpType = ARRAY_OP_POSITIONAL;
}
}
}
UpdateResult _updateObjects( bool su,
const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
RemoveSaver* rs,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy,
bool forReplication ) {
DEBUGUPDATE( "update: " << ns
<< " update: " << updateobj
<< " query: " << patternOrig
<< " upsert: " << upsert << " multi: " << multi );
Client& client = cc();
debug.updateobj = updateobj;
// The idea with these here it to make them loop invariant for
// multi updates, and thus be a bit faster for that case. The
// pointers may be left invalid on a failed or terminal yield
// recovery.
NamespaceDetails* d = nsdetails(ns); // can be null if an upsert...
NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get(ns);
auto_ptr<ModSet> mods;
bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$';
int modsIsIndexed = false; // really the # of indexes
if ( isOperatorUpdate ) {
mods.reset( new ModSet(updateobj, nsdt->indexKeys(), forReplication) );
modsIsIndexed = mods->maxNumIndexUpdated();
}
if( planPolicy.permitOptimalIdPlan() && !multi && isSimpleIdQuery(patternOrig) && d &&
!modsIsIndexed ) {
int idxNo = d->findIdIndex();
if( idxNo >= 0 ) {
debug.idhack = true;
UpdateResult result = _updateById( isOperatorUpdate,
idxNo,
mods.get(),
d,
nsdt,
su,
ns,
updateobj,
patternOrig,
logop,
debug,
fromMigrate);
if ( result.existing || ! upsert ) {
return result;
}
else if ( upsert && ! isOperatorUpdate ) {
// this handles repl inserts
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, false, su);
if ( logop )
logOp( "i", ns, no, 0, 0, fromMigrate, &no );
return UpdateResult( 0 , 0 , 1 , no );
}
}
}
int numModded = 0;
debug.nscanned = 0;
shared_ptr<Cursor> c = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy );
d = nsdetails(ns);
nsdt = &NamespaceDetailsTransient::get(ns);
bool autoDedup = c->autoDedup();
if( c->ok() ) {
set<DiskLoc> seenObjects;
MatchDetails details;
auto_ptr<ClientCursor> cc;
do {
if ( cc.get() == 0 &&
client.allowedToThrowPageFaultException() &&
! c->currLoc().isNull() &&
! c->currLoc().rec()->likelyInPhysicalMemory() ) {
throw PageFaultException( c->currLoc().rec() );
}
bool atomic = c->matcher() && c->matcher()->docMatcher().atomic();
if ( ! atomic && debug.nscanned > 0 ) {
// we need to use a ClientCursor to yield
if ( cc.get() == 0 ) {
shared_ptr< Cursor > cPtr = c;
cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
}
bool didYield;
if ( ! cc->yieldSometimes( ClientCursor::WillNeed, &didYield ) ) {
cc.release();
break;
}
if ( !c->ok() ) {
break;
}
if ( didYield ) {
d = nsdetails(ns);
if ( ! d )
break;
nsdt = &NamespaceDetailsTransient::get(ns);
if ( mods.get() ) {
mods->setIndexedStatus( nsdt->indexKeys() );
modsIsIndexed = mods->maxNumIndexUpdated();
}
}
} // end yielding block
debug.nscanned++;
if ( mods.get() && mods->hasDynamicArray() ) {
details.requestElemMatchKey();
}
if ( !c->currentMatches( &details ) ) {
c->advance();
continue;
}
Record* r = c->_current();
DiskLoc loc = c->currLoc();
if ( c->getsetdup( loc ) && autoDedup ) {
c->advance();
continue;
}
BSONObj pattern = patternOrig;
if ( logop ) {
BSONObj js = BSONObj::make(r);
BSONObj idQuery = makeOplogEntryQuery(js, multi);
pattern = idQuery;
}
/* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some
regular ones at the moment. */
if ( isOperatorUpdate ) {
if ( multi ) {
// go to next record in case this one moves
c->advance();
// Update operations are deduped for cursors that implement their own
// deduplication. In particular, some geo cursors are excluded.
if ( autoDedup ) {
if ( seenObjects.count( loc ) ) {
continue;
}
// SERVER-5198 Advance past the document to be modified, provided
// deduplication is enabled, but see SERVER-5725.
while( c->ok() && loc == c->currLoc() ) {
c->advance();
}
}
}
const BSONObj& onDisk = loc.obj();
ModSet* useMods = mods.get();
auto_ptr<ModSet> mymodset;
if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) {
useMods = mods->fixDynamicArray( details.elemMatchKey() );
mymodset.reset( useMods );
}
auto_ptr<ModSetState> mss = useMods->prepare( onDisk,
false /* not an insertion */ );
bool willAdvanceCursor = multi && c->ok() && ( modsIsIndexed || ! mss->canApplyInPlace() );
if ( willAdvanceCursor ) {
if ( cc.get() ) {
cc->setDoingDeletes( true );
}
c->prepareToTouchEarlierIterate();
}
// If we've made it this far, "ns" must contain a valid collection name, and so
// is of the form "db.collection". Therefore, the following expression must
// always be valid. "system.users" updates must never be done in place, in
// order to ensure that they are validated inside DataFileMgr::updateRecord(.).
bool isSystemUsersMod = nsToCollectionSubstring(ns) == "system.users";
BSONObj newObj;
if ( !mss->isUpdateIndexed() && mss->canApplyInPlace() && !isSystemUsersMod ) {
mss->applyModsInPlace( true );// const_cast<BSONObj&>(onDisk) );
DEBUGUPDATE( "\t\t\t doing in place update" );
if ( !multi )
debug.fastmod = true;
if ( modsIsIndexed ) {
seenObjects.insert( loc );
}
newObj = loc.obj();
d->paddingFits();
}
else {
newObj = mss->createNewFromMods();
checkTooLarge(newObj);
DiskLoc newLoc = theDataFileMgr.updateRecord(ns,
d,
nsdt,
r,
loc,
newObj.objdata(),
newObj.objsize(),
debug);
if ( newLoc != loc || modsIsIndexed ){
// log() << "Moved obj " << newLoc.obj()["_id"] << " from " << loc << " to " << newLoc << endl;
// object moved, need to make sure we don' get again
seenObjects.insert( newLoc );
}
}
if ( logop ) {
DEV verify( mods->size() );
BSONObj logObj = mss->getOpLogRewrite();
DEBUGUPDATE( "\t rewrite update: " << logObj );
// It is possible that the entire mod set was a no-op over this
// document. We would have an empty log record in that case. If we
// call logOp, with an empty record, that would be replicated as "clear
// this record", which is not what we want. Therefore, to get a no-op
// in the replica, we simply don't log.
if ( logObj.nFields() ) {
logOp("u", ns, logObj , &pattern, 0, fromMigrate, &newObj );
}
}
numModded++;
if ( ! multi )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( willAdvanceCursor )
c->recoverFromTouchingEarlierIterate();
getDur().commitIfNeeded();
continue;
}
uassert( 10158 , "multi update only works with $ operators" , ! multi );
BSONElementManipulator::lookForTimestamps( updateobj );
checkNoMods( updateobj );
theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, su);
if ( logop ) {
DEV wassert( !su ); // super used doesn't get logged, this would be bad.
logOp("u", ns, updateobj, &pattern, 0, fromMigrate, &updateobj );
}
return UpdateResult( 1 , 0 , 1 , BSONObj() );
} while ( c->ok() );
} // endif
if ( numModded )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( upsert ) {
if ( updateobj.firstElementFieldName()[0] == '$' ) {
// upsert of an $operation. build a default object
BSONObj newObj = mods->createNewFromQuery( patternOrig );
checkNoMods( newObj );
debug.fastmodinsert = true;
theDataFileMgr.insertWithObjMod(ns, newObj, false, su);
if ( logop )
logOp( "i", ns, newObj, 0, 0, fromMigrate, &newObj );
return UpdateResult( 0 , 1 , 1 , newObj );
}
uassert( 10159 , "multi update only works with $ operators" , ! multi );
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, false, su);
if ( logop )
logOp( "i", ns, no, 0, 0, fromMigrate, &no );
return UpdateResult( 0 , 0 , 1 , no );
}
return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() );
}
UpdateResult _updateObjectsNEW( bool su,
const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
RemoveSaver* rs,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy,
bool forReplication ) {
// TODO
// + Separate UpdateParser from UpdateRunner (the latter should be "stage-y")
// + All the yield and deduplicate logic would move to the query stage
// portion of it
//
// + Replication related
// + fast path for update for query by _id
// + support for relaxing viable path constraint in replication
//
// + Field Management
// + Force all upsert to contain _id
// + Prevent changes to immutable fields (_id, and those mentioned by sharding)
//
// + Yiedling related
// + $atomic support (or better, support proper yielding if not)
// + page fault support
debug.updateobj = updateobj;
NamespaceDetails* d = nsdetails( ns );
NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get( ns );
// TODO: Put this logic someplace central and check based on constants (maybe using the
// list of actually excluded config collections, and not global for the config db).
NamespaceString nsStr( ns );
// Should the modifiers validdate their embedded docs via okForStorage
bool shouldValidate = true;
// Config db docs shouldn't get checked for valid field names since the shard key can have
// a dot (".") in it. Therefore we disable validation for storage.
if ( nsStr.db() == "config" ) {
LOG(0) << "disabling okForStorage on config db";
shouldValidate = false;
}
UpdateDriver::Options opts;
opts.multi = multi;
opts.upsert = upsert;
opts.logOp = logop;
opts.modOptions = ModifierInterface::Options( forReplication, shouldValidate );
UpdateDriver driver( opts );
// TODO: This copies the index keys, but we may not actually need to.
Status status = driver.parse( nsdt->indexKeys(), updateobj );
if ( !status.isOK() ) {
uasserted( 16840, status.reason() );
}
shared_ptr<Cursor> cursor = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy );
// If the update was marked with '$isolated' (a.k.a '$atomic'), we are not allowed to
// yield while evaluating the update loop below.
//
// TODO: Old code checks this repeatedly within the update loop. Is that necessary? It seems
// that once atomic should be always atomic.
const bool canYield =
cursor->ok() &&
cursor->matcher() &&
cursor->matcher()->docMatcher().atomic();
// The 'cursor' the optimizer gave us may contain query plans that generate duplicate
// diskloc's. We set up here the mechanims that will prevent us from processing those
// twice if we see them. We also set up a 'ClientCursor' so that we can support
// yielding.
//
// TODO: Is it valid to call this on a non-ok cursor?
const bool dedupHere = cursor->autoDedup();
//
// We'll start assuming we have one or more documents for this update. (Othwerwise,
// we'll fallback to upserting.)
//
// We record that this will not be an upsert, in case a mod doesn't want to be applied
// when in strict update mode.
driver.setContext( ModifierInterface::ExecInfo::UPDATE_CONTEXT );
// Let's fetch each of them and pipe them through the update expression, making sure to
// keep track of the necessary stats. Recall that we'll be pulling documents out of
// cursors and some of them do not deduplicate the entries they generate. We have
// deduping logic in here, too -- for now.
unordered_set<DiskLoc, DiskLoc::Hasher> seenLocs;
int numUpdated = 0;
debug.nscanned = 0;
Client& client = cc();
mutablebson::Document doc;
// If we are going to be yielding, we will need a ClientCursor scoped to this loop. We
// only loop as long as the underlying cursor is OK.
for ( auto_ptr<ClientCursor> clientCursor; cursor->ok(); ) {
// If we haven't constructed a ClientCursor, and if the client allows us to throw
// page faults, and if we are referring to a location that is likely not in
// physical memory, then throw a PageFaultException. The entire operation will be
// restarted.
if ( clientCursor.get() == NULL &&
client.allowedToThrowPageFaultException() &&
!cursor->currLoc().isNull() &&
!cursor->currLoc().rec()->likelyInPhysicalMemory() ) {
// We should never throw a PFE if we have already updated items.
dassert(numUpdated == 0);
throw PageFaultException( cursor->currLoc().rec() );
}
if ( !canYield && debug.nscanned != 0 ) {
// We are permitted to yield. To do so we need a ClientCursor, so create one
// now if we have not yet done so.
if ( !clientCursor.get() )
clientCursor.reset(
new ClientCursor( QueryOption_NoCursorTimeout, cursor, ns ) );
// Ask the client cursor to yield. We get two bits of state back: whether or not
// we yielded, and whether or not we correctly recovered from yielding.
bool yielded = false;
const bool recovered = clientCursor->yieldSometimes(
ClientCursor::WillNeed, &yielded );
// If we couldn't recover from the yield, or if the cursor died while we were
// yielded, get out of the update loop right away. We don't need to reset
// 'clientCursor' since we are leaving the scope.
if ( !recovered || !cursor->ok() )
break;
if ( yielded ) {
// Details about our namespace may have changed while we were yielded, so
// we re-acquire them here. If we can't do so, escape the update
// loop. Otherwise, refresh the driver so that it knows about what is
// currently indexed.
d = nsdetails( ns );
if ( !d )
break;
nsdt = &NamespaceDetailsTransient::get( ns );
// TODO: This copies the index keys, but it may not need to do so.
driver.refreshIndexKeys( nsdt->indexKeys() );
}
}
// Let's fetch the next candidate object for this update.
Record* r = cursor->_current();
DiskLoc loc = cursor->currLoc();
const BSONObj oldObj = loc.obj();
// We count how many documents we scanned even though we may skip those that are
// deemed duplicated. The final 'numUpdated' and 'nscanned' numbers may differ for
// that reason.
debug.nscanned++;
// Skips this document if it:
// a) doesn't match the query portion of the update
// b) was deemed duplicate by the underlying cursor machinery
//
// Now, if we are going to update the document,
// c) we don't want to do so while the cursor is at it, as that may invalidate
// the cursor. So, we advance to next document, before issuing the update.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
if ( !cursor->currentMatches( &matchDetails ) ) {
// a)
cursor->advance();
continue;
}
else if ( cursor->getsetdup( loc ) && dedupHere ) {
// b)
cursor->advance();
continue;
}
else if (driver.dollarModMode() && multi) {
// c)
cursor->advance();
if ( dedupHere ) {
if ( seenLocs.count( loc ) ) {
continue;
}
}
// There are certain kind of cursors that hold multiple pointers to data
// underneath. $or cursors is one example. In a $or cursor, it may be the case
// that when we did the last advance(), we finished consuming documents from
// one of $or child and started consuming the next one. In that case, it is
// possible that the last document of the previous child is the same as the
// first document of the next (see SERVER-5198 and jstests/orp.js).
//
// So we advance the cursor here until we see a new diskloc.
//
// Note that we won't be yielding, and we may not do so for a while if we find
// a particularly duplicated sequence of loc's. That is highly unlikely,
// though. (See SERVER-5725, if curious, but "stage" based $or will make that
// ticket moot).
while( cursor->ok() && loc == cursor->currLoc() ) {
cursor->advance();
}
}
// For some (unfortunate) historical reasons, not all cursors would be valid after
// a write simply because we advanced them to a document not affected by the write.
// To protect in those cases, not only we engaged in the advance() logic above, but
// we also tell the cursor we're about to write a document that we've just seen.
// prepareToTouchEarlierIterate() requires calling later
// recoverFromTouchingEarlierIterate(), so we make a note here to do so.
bool touchPreviousDoc = multi && cursor->ok();
if ( touchPreviousDoc ) {
if ( clientCursor.get() )
clientCursor->setDoingDeletes( true );
cursor->prepareToTouchEarlierIterate();
}
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new
// document is needed to accomodate the new bson layout of the resulting document.
doc.reset( oldObj, mutablebson::Document::kInPlaceEnabled );
BSONObj logObj;
StringData matchedField = matchDetails.hasElemMatchKey() ?
matchDetails.elemMatchKey():
StringData();
status = driver.update( matchedField, &doc, &logObj );
if ( !status.isOK() ) {
uasserted( 16837, status.reason() );
}
// If the driver applied the mods in place, we can ask the mutable for what
// changed. We call those changes "damages". :) We use the damages to inform the
// journal what was changed, and then apply them to the original document
// ourselves. If, however, the driver applied the mods out of place, we ask it to
// generate a new, modified document for us. In that case, the file manager will
// take care of the journaling details for us.
//
// This code flow is admittedly odd. But, right now, journaling is baked in the file
// manager. And if we aren't using the file manager, we have to do jounaling
// ourselves.
bool objectWasChanged = false;
BSONObj newObj;
const char* source = NULL;
mutablebson::DamageVector damages;
bool inPlace = doc.getInPlaceUpdates(&damages, &source);
if ( inPlace && !damages.empty() && !driver.modsAffectIndices() ) {
d->paddingFits();
// All updates were in place. Apply them via durability and writing pointer.
mutablebson::DamageVector::const_iterator where = damages.begin();
const mutablebson::DamageVector::const_iterator end = damages.end();
for( ; where != end; ++where ) {
const char* sourcePtr = source + where->sourceOffset;
void* targetPtr = getDur().writingPtr(
const_cast<char*>(oldObj.objdata()) + where->targetOffset,
where->size);
std::memcpy(targetPtr, sourcePtr, where->size);
}
newObj = oldObj;
debug.fastmod = true;
objectWasChanged = true;
}
else {
// The updates were not in place. Apply them through the file manager.
newObj = doc.getObject();
DiskLoc newLoc = theDataFileMgr.updateRecord(ns,
d,
nsdt,
r,
loc,
newObj.objdata(),
newObj.objsize(),
debug);
// If we've moved this object to a new location, make sure we don't apply
// that update again if our traversal picks the objecta again.
//
// We also take note that the diskloc if the updates are affecting indices.
// Chances are that we're traversing one of them and they may be multi key and
// therefore duplicate disklocs.
if ( newLoc != loc || driver.modsAffectIndices() ) {
seenLocs.insert( newLoc );
}
objectWasChanged = true;
}
// Log Obj
if ( logop ) {
if ( !logObj.isEmpty() ) {
BSONObj idQuery = driver.makeOplogEntryQuery(newObj, multi);
logOp("u", ns, logObj , &idQuery, 0, fromMigrate, &newObj);
}
}
// If we applied any in-place updates, or asked the DataFileMgr to write for us,
// then count this as an update.
if (objectWasChanged)
numUpdated++;
if (!multi) {
break;
}
// If we used the cursor mechanism that prepares an earlier seen document for a
// write we need to tell such mechanisms that the write is over.
if ( touchPreviousDoc ) {
cursor->recoverFromTouchingEarlierIterate();
}
getDur().commitIfNeeded();
}
if (numUpdated > 0) {
return UpdateResult( true /* updated existing object(s) */,
driver.dollarModMode() /* $mod or obj replacement */,
numUpdated /* # of docments update */,
BSONObj() );
}
else if (numUpdated == 0 && !upsert) {
return UpdateResult( false /* no object updated */,
driver.dollarModMode() /* $mod or obj replacement */,
0 /* no updates */,
BSONObj() );
}
//
// We haven't succeeded updating any existing document but upserts are allowed.
//
// If this is a $mod base update, we need to generate a document by examining the
// query and the mods. Otherwise, we can use the object replacement sent by the user
// update command that was parsed by the driver before.
BSONObj oldObj;
if ( *updateobj.firstElementFieldName() == '$' ) {
if ( !driver.createFromQuery( patternOrig, &oldObj ) ) {
uasserted( 16835, "cannot create object to update" );
}
debug.fastmodinsert = true;
}
else {
// Copy the _id
if (patternOrig.hasElement("_id")) {
oldObj = patternOrig.getField("_id").wrap();
}
debug.upsert = true;
}
// Since this is an upsert, we will be oplogging it as an insert. We don't
// need the driver's help to build the oplog record, then. We also set the
// context of the update driver to an "upsert". Some mods may only work in that
// context (e.g. $setOnInsert).
driver.setLogOp( false );
driver.setContext( ModifierInterface::ExecInfo::INSERT_CONTEXT );
doc.reset( oldObj, mutablebson::Document::kInPlaceDisabled );
status = driver.update( StringData(), &doc, NULL /* no oplog record */);
if ( !status.isOK() ) {
uasserted( 16836, status.reason() );
}
BSONObj newObj = doc.getObject();
theDataFileMgr.insertWithObjMod( ns, newObj, false, su );
if ( logop ) {
logOp( "i", ns, newObj, 0, 0, fromMigrate, &newObj );
}
return UpdateResult( false /* updated a non existing document */,
driver.dollarModMode() /* $mod or obj replacement? */,
1 /* count of updated documents */,
newObj /* object that was upserted */ );
}
Status UpdateDriver::parse(const BSONObj& updateExpr, const bool multi) {
clear();
// Check if the update expression is a full object replacement.
if (*updateExpr.firstElementFieldName() != '$') {
if (multi) {
return Status(ErrorCodes::FailedToParse, "multi update only works with $ operators");
}
// Modifiers expect BSONElements as input. But the input to object replace is, by
// definition, an object. We wrap the 'updateExpr' as the mod is expecting. Note
// that the wrapper is temporary so the object replace mod should make a copy of
// the object.
unique_ptr<ModifierObjectReplace> mod(new ModifierObjectReplace);
BSONObj wrapper = BSON("dummy" << updateExpr);
Status status = mod->init(wrapper.firstElement(), _modOptions);
if (!status.isOK()) {
return status;
}
_mods.push_back(mod.release());
// Register the fact that this driver will only do full object replacements.
_replacementMode = true;
return Status::OK();
}
// The update expression is made of mod operators, that is
// { <$mod>: {...}, <$mod>: {...}, ... }
BSONObjIterator outerIter(updateExpr);
while (outerIter.more()) {
BSONElement outerModElem = outerIter.next();
// Check whether this is a valid mod type.
modifiertable::ModifierType modType = modifiertable::getType(outerModElem.fieldName());
if (modType == modifiertable::MOD_UNKNOWN) {
return Status(ErrorCodes::FailedToParse,
str::stream() << "Unknown modifier: " << outerModElem.fieldName());
}
// Check whether there is indeed a list of mods under this modifier.
if (outerModElem.type() != Object) {
return Status(ErrorCodes::FailedToParse,
str::stream() << "Modifiers operate on fields but we found type "
<< typeName(outerModElem.type())
<< " instead. For example: {$mod: {<field>: ...}}"
<< " not {"
<< outerModElem.toString()
<< "}");
}
// Check whether there are indeed mods under this modifier.
if (outerModElem.embeddedObject().isEmpty()) {
return Status(ErrorCodes::FailedToParse,
str::stream() << "'" << outerModElem.fieldName()
<< "' is empty. You must specify a field like so: "
"{"
<< outerModElem.fieldName()
<< ": {<field>: ...}}");
}
BSONObjIterator innerIter(outerModElem.embeddedObject());
while (innerIter.more()) {
BSONElement innerModElem = innerIter.next();
Status status = addAndParse(modType, innerModElem);
if (!status.isOK()) {
return status;
}
}
}
// Register the fact that there will be only $mod's in this driver -- no object
// replacement.
_replacementMode = false;
return Status::OK();
}
void validateNS(const char *ns,
NamespaceDetails *d,
const BSONObj& cmdObj,
BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( d->isCapped() ){
result.append("capped", d->isCapped());
result.appendNumber("max", d->maxCappedDocs());
}
result.append("firstExtent", str::stream() << d->firstExtent.toString()
<< " ns:" << d->firstExtent.ext()->nsDiagnostic.toString());
result.append( "lastExtent", str::stream() << d->lastExtent.toString()
<< " ns:" << d->lastExtent.ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
int extentCount = 0;
try {
d->firstExtent.ext()->assertOk();
d->lastExtent.ext()->assertOk();
DiskLoc extentDiskLoc = d->firstExtent;
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = extentDiskLoc.ext();
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &errors)) {
valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& nextDiskLoc.ext()->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != d->lastExtent) {
StringBuilder sb;
sb << "'lastExtent' pointer " << d->lastExtent.toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
killCurrentOp.checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
errors << sb.str();
valid = false;
}
result.append("extentCount", extentCount);
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", d->stats.datasize);
result.appendNumber("nrecords", d->stats.nrecords);
result.appendNumber("lastExtentSize", d->lastExtentSize);
result.appendNumber("padding", d->paddingFactor());
try {
bool testingLastExtent = false;
try {
if (d->firstExtent.isNull()) {
errors << "'firstExtent' pointer is null";
valid=false;
}
else {
result.append("firstExtentDetails", d->firstExtent.ext()->dump());
if (!d->firstExtent.ext()->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << d->firstExtent.toString()
<< " is " << d->firstExtent.ext()->xprev.toString()
<< ", should be null";
errors << sb.str();
valid=false;
}
}
testingLastExtent = true;
if (d->lastExtent.isNull()) {
errors << "'lastExtent' pointer is null";
valid=false;
}
else {
if (d->firstExtent != d->lastExtent) {
result.append("lastExtentDetails", d->lastExtent.ext()->dump());
if (!d->lastExtent.ext()->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << d->lastExtent.toString()
<< " is " << d->lastExtent.ext()->xnext.toString()
<< ", should be null";
errors << sb.str();
valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
errors << sb.str();
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
int n = 0;
int nInvalid = 0;
long long len = 0;
long long nlen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
while ( c->ok() ) {
n++;
DiskLoc cl = c->currLoc();
if ( n < 1000000 )
recs.insert(cl);
if ( d->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = c->_current();
len += r->lengthWithHeaders();
nlen += r->netLength();
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
}
c->advance();
}
if ( d->isCapped() && !d->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << d->deletedList[i].isNull();
}
int ndel = 0;
long long delSize = 0;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = d->deletedList[i];
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( d->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
errors << err;
valid = false;
break;
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
result.append("nIndexes", d->nIndexes);
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
long long keys = id.idxInterface().fullValidate(id.head, id.keyPattern());
indexes.appendNumber(id.indexNamespace(), keys);
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt, requires repair");
}
}
void validateNS(const string& ns,
Collection* collection,
const BSONObj& cmdObj,
BSONObjBuilder& result) {
const bool full = cmdObj["full"].trueValue();
const bool scanData = full || cmdObj["scandata"].trueValue();
NamespaceDetails* nsd = collection->details();
bool valid = true;
BSONArrayBuilder errors; // explanation(s) for why valid = false
if ( collection->isCapped() ){
result.append("capped", nsd->isCapped());
result.appendNumber("max", nsd->maxCappedDocs());
}
if ( nsd->firstExtent().isNull() )
result.append( "firstExtent", "null" );
else
result.append( "firstExtent", str::stream() << nsd->firstExtent().toString()
<< " ns:" << nsd->firstExtent().ext()->nsDiagnostic.toString());
if ( nsd->lastExtent().isNull() )
result.append( "lastExtent", "null" );
else
result.append( "lastExtent", str::stream() << nsd->lastExtent().toString()
<< " ns:" << nsd->lastExtent().ext()->nsDiagnostic.toString());
BSONArrayBuilder extentData;
int extentCount = 0;
try {
if ( !nsd->firstExtent().isNull() ) {
nsd->firstExtent().ext()->assertOk();
nsd->lastExtent().ext()->assertOk();
}
DiskLoc extentDiskLoc = nsd->firstExtent();
while (!extentDiskLoc.isNull()) {
Extent* thisExtent = extentDiskLoc.ext();
if (full) {
extentData << thisExtent->dump();
}
if (!thisExtent->validates(extentDiskLoc, &errors)) {
valid = false;
}
DiskLoc nextDiskLoc = thisExtent->xnext;
if (extentCount > 0 && !nextDiskLoc.isNull()
&& nextDiskLoc.ext()->xprev != extentDiskLoc) {
StringBuilder sb;
sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString()
<< " in extent " << nextDiskLoc.toString()
<< " does not point to extent " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
if (nextDiskLoc.isNull() && extentDiskLoc != nsd->lastExtent()) {
StringBuilder sb;
sb << "'lastExtent' pointer " << nsd->lastExtent().toString()
<< " does not point to last extent in list " << extentDiskLoc.toString();
errors << sb.str();
valid = false;
}
extentDiskLoc = nextDiskLoc;
extentCount++;
killCurrentOp.checkForInterrupt();
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception validating extent " << extentCount
<< ": " << e.what();
errors << sb.str();
valid = false;
}
result.append("extentCount", extentCount);
if ( full )
result.appendArray( "extents" , extentData.arr() );
result.appendNumber("datasize", nsd->dataSize());
result.appendNumber("nrecords", nsd->numRecords());
result.appendNumber("lastExtentSize", nsd->lastExtentSize());
result.appendNumber("padding", nsd->paddingFactor());
try {
bool testingLastExtent = false;
try {
if (nsd->firstExtent().isNull()) {
// this is ok
}
else {
result.append("firstExtentDetails", nsd->firstExtent().ext()->dump());
if (!nsd->firstExtent().ext()->xprev.isNull()) {
StringBuilder sb;
sb << "'xprev' pointer in 'firstExtent' " << nsd->firstExtent().toString()
<< " is " << nsd->firstExtent().ext()->xprev.toString()
<< ", should be null";
errors << sb.str();
valid=false;
}
}
testingLastExtent = true;
if (nsd->lastExtent().isNull()) {
// this is ok
}
else {
if (nsd->firstExtent() != nsd->lastExtent()) {
result.append("lastExtentDetails", nsd->lastExtent().ext()->dump());
if (!nsd->lastExtent().ext()->xnext.isNull()) {
StringBuilder sb;
sb << "'xnext' pointer in 'lastExtent' " << nsd->lastExtent().toString()
<< " is " << nsd->lastExtent().ext()->xnext.toString()
<< ", should be null";
errors << sb.str();
valid = false;
}
}
}
}
catch (const DBException& e) {
StringBuilder sb;
sb << "exception processing '"
<< (testingLastExtent ? "lastExtent" : "firstExtent")
<< "': " << e.what();
errors << sb.str();
valid = false;
}
set<DiskLoc> recs;
if( scanData ) {
int n = 0;
int nInvalid = 0;
long long nQuantizedSize = 0;
long long nPowerOf2QuantizedSize = 0;
long long len = 0;
long long nlen = 0;
long long bsonLen = 0;
int outOfOrder = 0;
DiskLoc cl_last;
DiskLoc cl;
Runner::RunnerState state;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &cl))) {
n++;
if ( n < 1000000 )
recs.insert(cl);
if ( nsd->isCapped() ) {
if ( cl < cl_last )
outOfOrder++;
cl_last = cl;
}
Record *r = cl.rec();
len += r->lengthWithHeaders();
nlen += r->netLength();
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizeAllocationSpace
( r->lengthWithHeaders() ) ) {
// Count the number of records having a size consistent with
// the quantizeAllocationSpace quantization implementation.
++nQuantizedSize;
}
if ( r->lengthWithHeaders() ==
NamespaceDetails::quantizePowerOf2AllocationSpace
( r->lengthWithHeaders() - 1 ) ) {
// Count the number of records having a size consistent with the
// quantizePowerOf2AllocationSpace quantization implementation.
// Because of SERVER-8311, power of 2 quantization is not idempotent and
// r->lengthWithHeaders() - 1 must be checked instead of the record
// length itself.
++nPowerOf2QuantizedSize;
}
if (full){
BSONObj obj = BSONObj::make(r);
if (!obj.isValid() || !obj.valid()){ // both fast and deep checks
valid = false;
if (nInvalid == 0) // only log once;
errors << "invalid bson object detected (see logs for more info)";
nInvalid++;
if (strcmp("_id", obj.firstElementFieldName()) == 0){
try {
obj.firstElement().validate(); // throws on error
log() << "Invalid bson detected in " << ns << " with _id: " << obj.firstElement().toString(false) << endl;
}
catch(...){
log() << "Invalid bson detected in " << ns << " with corrupt _id" << endl;
}
}
else {
log() << "Invalid bson detected in " << ns << " and couldn't find _id" << endl;
}
}
else {
bsonLen += obj.objsize();
}
}
}
if (Runner::RUNNER_EOF != state) {
// TODO: more descriptive logging.
warning() << "Internal error while reading collection " << ns << endl;
}
if ( nsd->isCapped() && !nsd->capLooped() ) {
result.append("cappedOutOfOrder", outOfOrder);
if ( outOfOrder > 1 ) {
valid = false;
errors << "too many out of order records";
}
}
result.append("objectsFound", n);
if (full) {
result.append("invalidObjects", nInvalid);
}
result.appendNumber("nQuantizedSize", nQuantizedSize);
result.appendNumber("nPowerOf2QuantizedSize", nPowerOf2QuantizedSize);
result.appendNumber("bytesWithHeaders", len);
result.appendNumber("bytesWithoutHeaders", nlen);
if (full) {
result.appendNumber("bytesBson", bsonLen);
}
}
BSONArrayBuilder deletedListArray;
for ( int i = 0; i < Buckets; i++ ) {
deletedListArray << nsd->deletedListEntry(i).isNull();
}
int ndel = 0;
long long delSize = 0;
BSONArrayBuilder delBucketSizes;
int incorrect = 0;
for ( int i = 0; i < Buckets; i++ ) {
DiskLoc loc = nsd->deletedListEntry(i);
try {
int k = 0;
while ( !loc.isNull() ) {
if ( recs.count(loc) )
incorrect++;
ndel++;
if ( loc.questionable() ) {
if( nsd->isCapped() && !loc.isValid() && i == 1 ) {
/* the constructor for NamespaceDetails intentionally sets deletedList[1] to invalid
see comments in namespace.h
*/
break;
}
string err( str::stream() << "bad pointer in deleted record list: "
<< loc.toString()
<< " bucket: " << i
<< " k: " << k );
errors << err;
valid = false;
break;
}
DeletedRecord *d = loc.drec();
delSize += d->lengthWithHeaders();
loc = d->nextDeleted();
k++;
killCurrentOp.checkForInterrupt();
}
delBucketSizes << k;
}
catch (...) {
errors << ("exception in deleted chain for bucket " + BSONObjBuilder::numStr(i));
valid = false;
}
}
result.appendNumber("deletedCount", ndel);
result.appendNumber("deletedSize", delSize);
if ( full ) {
result << "delBucketSizes" << delBucketSizes.arr();
}
if ( incorrect ) {
errors << (BSONObjBuilder::numStr(incorrect) + " records from datafile are in deleted list");
valid = false;
}
int idxn = 0;
try {
IndexCatalog* indexCatalog = collection->getIndexCatalog();
result.append("nIndexes", nsd->getCompletedIndexCount());
BSONObjBuilder indexes; // not using subObjStart to be exception safe
NamespaceDetails::IndexIterator i = nsd->ii();
while( i.more() ) {
IndexDetails& id = i.next();
log() << "validating index " << idxn << ": " << id.indexNamespace() << endl;
IndexDescriptor* descriptor = indexCatalog->getDescriptor( idxn );
verify( descriptor );
IndexAccessMethod* iam = indexCatalog->getIndex( descriptor );
verify( iam );
int64_t keys;
iam->validate(&keys);
indexes.appendNumber(id.indexNamespace(), static_cast<long long>(keys));
idxn++;
}
result.append("keysPerIndex", indexes.done());
}
catch (...) {
errors << ("exception during index validate idxn " + BSONObjBuilder::numStr(idxn));
valid=false;
}
}
catch (AssertionException) {
errors << "exception during validate";
valid = false;
}
result.appendBool("valid", valid);
result.append("errors", errors.arr());
if ( !full ){
result.append("warning", "Some checks omitted for speed. use {full:true} option to do more thorough scan.");
}
if ( !valid ) {
result.append("advice", "ns corrupt. See http://dochub.mongodb.org/core/data-recovery");
}
}
Status updateShardChunks(OperationContext* opCtx,
const NamespaceString& nss,
const std::vector<ChunkType>& chunks,
const OID& currEpoch) {
invariant(!chunks.empty());
NamespaceString chunkMetadataNss(ChunkType::ShardNSPrefix + nss.ns());
try {
DBDirectClient client(opCtx);
/**
* Here are examples of the operations that can happen on the config server to update
* the config.chunks collection. 'chunks' only includes the chunks that result from the
* operations, which can be read from the config server, not any that were removed, so
* we must delete any chunks that overlap with the new 'chunks'.
*
* CollectionVersion = 10.3
*
* moveChunk
* {_id: 3, max: 5, version: 10.1} --> {_id: 3, max: 5, version: 11.0}
*
* splitChunk
* {_id: 3, max: 9, version 10.3} --> {_id: 3, max: 5, version 10.4}
* {_id: 5, max: 8, version 10.5}
* {_id: 8, max: 9, version 10.6}
*
* mergeChunk
* {_id: 10, max: 14, version 4.3} --> {_id: 10, max: 22, version 10.4}
* {_id: 14, max: 19, version 7.1}
* {_id: 19, max: 22, version 2.0}
*
*/
for (auto& chunk : chunks) {
// Check for a different epoch.
if (!chunk.getVersion().hasEqualEpoch(currEpoch)) {
return Status{ErrorCodes::ConflictingOperationInProgress,
str::stream() << "Invalid chunks found when reloading '"
<< nss.toString()
<< "'. Previous collection epoch was '"
<< currEpoch.toString()
<< "', but unexpectedly found a new epoch '"
<< chunk.getVersion().epoch().toString()
<< "'. Collection was dropped and recreated."};
}
// Delete any overlapping chunk ranges. Overlapping chunks will have a min value
// ("_id") between (chunk.min, chunk.max].
//
// query: { "_id" : {"$gte": chunk.min, "$lt": chunk.max}}
auto deleteDocs(stdx::make_unique<BatchedDeleteDocument>());
deleteDocs->setQuery(BSON(ChunkType::minShardID << BSON(
"$gte" << chunk.getMin() << "$lt" << chunk.getMax())));
deleteDocs->setLimit(0);
auto deleteRequest(stdx::make_unique<BatchedDeleteRequest>());
deleteRequest->addToDeletes(deleteDocs.release());
BatchedCommandRequest batchedDeleteRequest(deleteRequest.release());
batchedDeleteRequest.setNS(chunkMetadataNss);
const BSONObj deleteCmdObj = batchedDeleteRequest.toBSON();
rpc::UniqueReply deleteCommandResponse =
client.runCommandWithMetadata(chunkMetadataNss.db().toString(),
deleteCmdObj.firstElementFieldName(),
rpc::makeEmptyMetadata(),
deleteCmdObj);
auto deleteStatus =
getStatusFromCommandResult(deleteCommandResponse->getCommandReply());
if (!deleteStatus.isOK()) {
return deleteStatus;
}
// Now the document can be expected to cleanly insert without overlap.
auto insert(stdx::make_unique<BatchedInsertRequest>());
insert->addToDocuments(chunk.toShardBSON());
BatchedCommandRequest insertRequest(insert.release());
insertRequest.setNS(chunkMetadataNss);
const BSONObj insertCmdObj = insertRequest.toBSON();
rpc::UniqueReply commandResponse =
client.runCommandWithMetadata(chunkMetadataNss.db().toString(),
insertCmdObj.firstElementFieldName(),
rpc::makeEmptyMetadata(),
insertCmdObj);
auto insertStatus = getStatusFromCommandResult(commandResponse->getCommandReply());
if (!insertStatus.isOK()) {
return insertStatus;
}
}
return Status::OK();
} catch (const DBException& ex) {
return ex.toStatus();
}
}
bool processObj(const BSONObj &obj) {
if (obj.hasField("$err")) {
log() << "error getting oplog: " << obj << endl;
return false;
}
static const char *names[] = {"ts", "op", "ns", "o", "b"};
BSONElement fields[5];
obj.getFields(5, names, fields);
BSONElement &tsElt = fields[0];
if (!tsElt.ok()) {
log() << "oplog format error: " << obj << " missing 'ts' field." << endl;
return false;
}
if (tsElt.type() != Date && tsElt.type() != Timestamp) {
log() << "oplog format error: " << obj << " wrong 'ts' field type." << endl;
return false;
}
_thisTime = OpTime(tsElt.date());
BSONElement &opElt = fields[1];
if (!opElt.ok()) {
log() << "oplog format error: " << obj << " missing 'op' field." << endl;
return false;
}
StringData op = opElt.Stringdata();
// nop
if (op == "n") {
if (!_insertBuf.empty()) {
flushInserts();
}
_maxOpTimeSynced = _thisTime;
_thisTime = OpTime();
return true;
}
// "presence of a database"
if (op == "db") {
if (!_insertBuf.empty()) {
flushInserts();
}
_maxOpTimeSynced = _thisTime;
_thisTime = OpTime();
return true;
}
if (op != "c" && op != "i" && op != "u" && op != "d") {
log() << "oplog format error: " << obj << " has an invalid 'op' field of '" << op << "'." << endl;
return false;
}
if (op != "i" && !_insertBuf.empty()) {
flushInserts();
}
BSONElement &nsElt = fields[2];
if (!nsElt.ok()) {
log() << "oplog format error: " << obj << " missing 'ns' field." << endl;
return false;
}
StringData ns = nsElt.Stringdata();
size_t i = ns.find('.');
if (i == string::npos) {
log() << "oplog format error: invalid namespace '" << ns << "' in op " << obj << "." << endl;
return false;
}
StringData dbname = ns.substr(0, i);
StringData collname = ns.substr(i + 1);
BSONElement &oElt = fields[3];
if (!oElt.ok()) {
log() << "oplog format error: " << obj << " missing 'o' field." << endl;
return false;
}
BSONObj o = obj["o"].Obj();
if (op == "c") {
if (collname != "$cmd") {
log() << "oplog format error: invalid namespace '" << ns << "' for command in op " << obj << "." << endl;
return false;
}
BSONObj info;
bool ok = _conn.runCommand(dbname.toString(), o, info);
if (!ok) {
StringData fieldName = o.firstElementFieldName();
BSONElement errmsgElt = info["errmsg"];
StringData errmsg = errmsgElt.type() == String ? errmsgElt.Stringdata() : "";
bool isDropIndexes = (fieldName == "dropIndexes" || fieldName == "deleteIndexes");
if (((fieldName == "drop" || isDropIndexes) && errmsg == "ns not found") ||
(isDropIndexes && (errmsg == "index not found" || errmsg.find("can't find index with key:") == 0))) {
// This is actually ok. We don't mind dropping something that's not there.
LOG(1) << "Tried to replay " << o << ", got " << info << ", ignoring." << endl;
}
else {
log() << "replay of command " << o << " failed: " << info << endl;
return false;
}
}
}
else {
string nsstr = ns.toString();
if (op == "i") {
if (collname == "system.indexes") {
// Can't ensure multiple indexes in the same batch.
flushInserts();
// For now, we need to strip out any background fields from
// ensureIndex. Once we do hot indexing we can do something more
// like what vanilla applyOperation_inlock does.
if (o["background"].trueValue()) {
BSONObjBuilder builder;
BSONObjIterator it(o);
while (it.more()) {
BSONElement e = it.next();
if (strncmp(e.fieldName(), "background", sizeof("background")) != 0) {
builder.append(e);
}
}
o = builder.obj();
}
// We need to warn very carefully about dropDups.
if (o["dropDups"].trueValue()) {
BSONObjBuilder builder;
BSONObjIterator it(o);
while (it.more()) {
BSONElement e = it.next();
if (strncmp(e.fieldName(), "dropDups", sizeof("dropDups")) != 0) {
builder.append(e);
}
}
warning() << "Detected an ensureIndex with dropDups: true in " << o << "." << endl;
warning() << "This option is not supported in TokuMX, because it deletes arbitrary data." << endl;
warning() << "If it were replayed, it could result in a completely different data set than the source database." << endl;
warning() << "We will attempt to replay it without dropDups, but if that fails, you must restart your migration process." << endl;
_conn.insert(nsstr, o);
string err = _conn.getLastError(dbname.toString(), false, false);
if (!err.empty()) {
log() << "replay of operation " << obj << " failed: " << err << endl;
warning() << "You cannot continue processing this replication stream. You need to restart the migration process." << endl;
_running = false;
_logAtExit = false;
return true;
}
}
}
pushInsert(nsstr, o);
// Don't call GLE or update _maxOpTimeSynced yet.
_thisTime = OpTime();
return true;
}
else if (op == "u") {
BSONElement o2Elt = obj["o2"];
if (!o2Elt.ok()) {
log() << "oplog format error: " << obj << " missing 'o2' field." << endl;
return false;
}
BSONElement &bElt = fields[4];
bool upsert = bElt.booleanSafe();
BSONObj o2 = o2Elt.Obj();
_conn.update(nsstr, o2, o, upsert, false);
}
else if (op == "d") {
BSONElement &bElt = fields[4];
bool justOne = bElt.booleanSafe();
_conn.remove(nsstr, o, justOne);
}
string err = _conn.getLastError(dbname.toString(), false, false);
if (!err.empty()) {
log() << "replay of operation " << obj << " failed: " << err << endl;
return false;
}
}
// If we got here, we completed the operation successfully.
_maxOpTimeSynced = _thisTime;
_thisTime = OpTime();
return true;
}
void Projection::init( const BSONObj& o ) {
massert( 10371 , "can only add to Projection once", _source.isEmpty());
_source = o;
BSONObjIterator i( o );
int true_false = -1;
while ( i.more() ) {
BSONElement e = i.next();
if ( ! e.isNumber() )
_hasNonSimple = true;
if (e.type() == Object) {
BSONObj obj = e.embeddedObject();
BSONElement e2 = obj.firstElement();
if ( strcmp(e2.fieldName(), "$slice") == 0 ) {
if (e2.isNumber()) {
int i = e2.numberInt();
if (i < 0)
add(e.fieldName(), i, -i); // limit is now positive
else
add(e.fieldName(), 0, i);
}
else if (e2.type() == Array) {
BSONObj arr = e2.embeddedObject();
uassert(13099, "$slice array wrong size", arr.nFields() == 2 );
BSONObjIterator it(arr);
int skip = it.next().numberInt();
int limit = it.next().numberInt();
uassert(13100, "$slice limit must be positive", limit > 0 );
add(e.fieldName(), skip, limit);
}
else {
uassert(13098, "$slice only supports numbers and [skip, limit] arrays", false);
}
}
else {
uassert(13097, string("Unsupported projection option: ") + obj.firstElementFieldName(), false);
}
}
else if (!strcmp(e.fieldName(), "_id") && !e.trueValue()) {
_includeID = false;
}
else {
add (e.fieldName(), e.trueValue());
// validate input
if (true_false == -1) {
true_false = e.trueValue();
_include = !e.trueValue();
}
else {
uassert( 10053 , "You cannot currently mix including and excluding fields. Contact us if this is an issue." ,
(bool)true_false == e.trueValue() );
}
}
}
}
bool UpdateDriver::isDocReplacement(const BSONObj& updateExpr) {
return *updateExpr.firstElementFieldName() != '$';
}
UpdateResult _updateObjects( bool su,
const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
RemoveSaver* rs,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy ) {
DEBUGUPDATE( "update: " << ns
<< " update: " << updateobj
<< " query: " << patternOrig
<< " upsert: " << upsert << " multi: " << multi );
Client& client = cc();
int profile = client.database()->profile;
debug.updateobj = updateobj;
// The idea with these here it to make them loop invariant for
// multi updates, and thus be a bit faster for that case. The
// pointers may be left invalid on a failed or terminal yield
// recovery.
NamespaceDetails* d = nsdetails(ns); // can be null if an upsert...
NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get(ns);
auto_ptr<ModSet> mods;
bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$';
int modsIsIndexed = false; // really the # of indexes
if ( isOperatorUpdate ) {
if( d && d->indexBuildInProgress ) {
set<string> bgKeys;
d->inProgIdx().keyPattern().getFieldNames(bgKeys);
mods.reset( new ModSet(updateobj, nsdt->indexKeys(), &bgKeys) );
}
else {
mods.reset( new ModSet(updateobj, nsdt->indexKeys()) );
}
modsIsIndexed = mods->isIndexed();
}
if( planPolicy.permitOptimalIdPlan() && !multi && isSimpleIdQuery(patternOrig) && d &&
!modsIsIndexed ) {
int idxNo = d->findIdIndex();
if( idxNo >= 0 ) {
debug.idhack = true;
UpdateResult result = _updateById( isOperatorUpdate,
idxNo,
mods.get(),
profile,
d,
nsdt,
su,
ns,
updateobj,
patternOrig,
logop,
debug,
fromMigrate);
if ( result.existing || ! upsert ) {
return result;
}
else if ( upsert && ! isOperatorUpdate && ! logop) {
// this handles repl inserts
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, su);
return UpdateResult( 0 , 0 , 1 , no );
}
}
}
int numModded = 0;
debug.nscanned = 0;
shared_ptr<Cursor> c =
NamespaceDetailsTransient::getCursor( ns, patternOrig, BSONObj(), planPolicy );
d = nsdetails(ns);
nsdt = &NamespaceDetailsTransient::get(ns);
bool autoDedup = c->autoDedup();
if( c->ok() ) {
set<DiskLoc> seenObjects;
MatchDetails details;
auto_ptr<ClientCursor> cc;
do {
if ( cc.get() == 0 &&
client.allowedToThrowPageFaultException() &&
! c->currLoc().isNull() &&
! c->currLoc().rec()->likelyInPhysicalMemory() ) {
throw PageFaultException( c->currLoc().rec() );
}
bool atomic = c->matcher() && c->matcher()->docMatcher().atomic();
if ( ! atomic && debug.nscanned > 0 ) {
// we need to use a ClientCursor to yield
if ( cc.get() == 0 ) {
shared_ptr< Cursor > cPtr = c;
cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
}
bool didYield;
if ( ! cc->yieldSometimes( ClientCursor::WillNeed, &didYield ) ) {
cc.release();
break;
}
if ( !c->ok() ) {
break;
}
if ( didYield ) {
d = nsdetails(ns);
nsdt = &NamespaceDetailsTransient::get(ns);
}
} // end yielding block
debug.nscanned++;
if ( mods.get() && mods->hasDynamicArray() ) {
// The Cursor must have a Matcher to record an elemMatchKey. But currently
// a modifier on a dynamic array field may be applied even if there is no
// elemMatchKey, so a matcher cannot be required.
//verify( c->matcher() );
details.requestElemMatchKey();
}
if ( !c->currentMatches( &details ) ) {
c->advance();
if ( debug.nscanned % 256 == 0 && ! atomic ) {
if ( cc.get() == 0 ) {
shared_ptr< Cursor > cPtr = c;
cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
}
if ( ! cc->yield() ) {
cc.release();
// TODO should we assert or something?
break;
}
if ( !c->ok() ) {
break;
}
d = nsdetails(ns);
nsdt = &NamespaceDetailsTransient::get(ns);
}
continue;
}
Record* r = c->_current();
DiskLoc loc = c->currLoc();
if ( c->getsetdup( loc ) && autoDedup ) {
c->advance();
continue;
}
BSONObj js = BSONObj::make(r);
BSONObj pattern = patternOrig;
if ( logop ) {
BSONObjBuilder idPattern;
BSONElement id;
// NOTE: If the matching object lacks an id, we'll log
// with the original pattern. This isn't replay-safe.
// It might make sense to suppress the log instead
// if there's no id.
if ( js.getObjectID( id ) ) {
idPattern.append( id );
pattern = idPattern.obj();
}
else {
uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi );
}
}
/* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some
regular ones at the moment. */
if ( isOperatorUpdate ) {
if ( multi ) {
// go to next record in case this one moves
c->advance();
// Update operations are deduped for cursors that implement their own
// deduplication. In particular, some geo cursors are excluded.
if ( autoDedup ) {
if ( seenObjects.count( loc ) ) {
continue;
}
// SERVER-5198 Advance past the document to be modified, provided
// deduplication is enabled, but see SERVER-5725.
while( c->ok() && loc == c->currLoc() ) {
c->advance();
}
}
}
const BSONObj& onDisk = loc.obj();
ModSet* useMods = mods.get();
bool forceRewrite = false;
auto_ptr<ModSet> mymodset;
if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) {
useMods = mods->fixDynamicArray( details.elemMatchKey() );
mymodset.reset( useMods );
forceRewrite = true;
}
auto_ptr<ModSetState> mss = useMods->prepare( onDisk );
bool willAdvanceCursor = multi && c->ok() && ( modsIsIndexed || ! mss->canApplyInPlace() );
if ( willAdvanceCursor ) {
if ( cc.get() ) {
cc->setDoingDeletes( true );
}
c->prepareToTouchEarlierIterate();
}
if ( modsIsIndexed <= 0 && mss->canApplyInPlace() ) {
mss->applyModsInPlace( true );// const_cast<BSONObj&>(onDisk) );
DEBUGUPDATE( "\t\t\t doing in place update" );
if ( profile && !multi )
debug.fastmod = true;
if ( modsIsIndexed ) {
seenObjects.insert( loc );
}
d->paddingFits();
}
else {
if ( rs )
rs->goingToDelete( onDisk );
BSONObj newObj = mss->createNewFromMods();
checkTooLarge(newObj);
DiskLoc newLoc = theDataFileMgr.updateRecord(ns,
d,
nsdt,
r,
loc,
newObj.objdata(),
newObj.objsize(),
debug);
if ( newLoc != loc || modsIsIndexed ){
// log() << "Moved obj " << newLoc.obj()["_id"] << " from " << loc << " to " << newLoc << endl;
// object moved, need to make sure we don' get again
seenObjects.insert( newLoc );
}
}
if ( logop ) {
DEV verify( mods->size() );
if ( mss->haveArrayDepMod() ) {
BSONObjBuilder patternBuilder;
patternBuilder.appendElements( pattern );
mss->appendSizeSpecForArrayDepMods( patternBuilder );
pattern = patternBuilder.obj();
}
if ( forceRewrite || mss->needOpLogRewrite() ) {
DEBUGUPDATE( "\t rewrite update: " << mss->getOpLogRewrite() );
logOp("u", ns, mss->getOpLogRewrite() ,
&pattern, 0, fromMigrate );
}
else {
logOp("u", ns, updateobj, &pattern, 0, fromMigrate );
}
}
numModded++;
if ( ! multi )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( willAdvanceCursor )
c->recoverFromTouchingEarlierIterate();
if ( debug.nscanned % 64 == 0 && ! atomic ) {
if ( cc.get() == 0 ) {
shared_ptr< Cursor > cPtr = c;
cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
}
if ( ! cc->yield() ) {
cc.release();
break;
}
if ( !c->ok() ) {
break;
}
d = nsdetails(ns);
nsdt = &NamespaceDetailsTransient::get(ns);
}
getDur().commitIfNeeded();
continue;
}
uassert( 10158 , "multi update only works with $ operators" , ! multi );
BSONElementManipulator::lookForTimestamps( updateobj );
checkNoMods( updateobj );
theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, su);
if ( logop ) {
DEV wassert( !su ); // super used doesn't get logged, this would be bad.
logOp("u", ns, updateobj, &pattern, 0, fromMigrate );
}
return UpdateResult( 1 , 0 , 1 , BSONObj() );
} while ( c->ok() );
} // endif
if ( numModded )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( upsert ) {
if ( updateobj.firstElementFieldName()[0] == '$' ) {
// upsert of an $operation. build a default object
BSONObj newObj = mods->createNewFromQuery( patternOrig );
checkNoMods( newObj );
debug.fastmodinsert = true;
theDataFileMgr.insertWithObjMod(ns, newObj, su);
if ( logop )
logOp( "i", ns, newObj, 0, 0, fromMigrate );
return UpdateResult( 0 , 1 , 1 , newObj );
}
uassert( 10159 , "multi update only works with $ operators" , ! multi );
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, su);
if ( logop )
logOp( "i", ns, no, 0, 0, fromMigrate );
return UpdateResult( 0 , 0 , 1 , no );
}
return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() );
}
bool CmdExplain::run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj, int options,
string& errmsg,
BSONObjBuilder& result,
bool fromRepl) {
// Should never get explain commands issued from replication.
if (fromRepl) {
Status commandStat(ErrorCodes::IllegalOperation,
"explain command should not be from repl");
appendCommandStatus(result, commandStat);
return false;
}
// Get the verbosity. We use the executionStats verbosity by default.
Explain::Verbosity verbosity = Explain::EXEC_STATS;
if (!cmdObj["verbosity"].eoo()) {
const char* verbStr = cmdObj["verbosity"].valuestrsafe();
if (mongoutils::str::equals(verbStr, "queryPlanner")) {
verbosity = Explain::QUERY_PLANNER;
}
else if (mongoutils::str::equals(verbStr, "allPlansExecution")) {
verbosity = Explain::EXEC_ALL_PLANS;
}
else if (mongoutils::str::equals(verbStr, "full")) {
verbosity = Explain::FULL;
}
else if (!mongoutils::str::equals(verbStr, "executionStats")) {
Status commandStat(ErrorCodes::BadValue,
"verbosity string must be one of "
"{'queryPlanner', 'executionStats', 'allPlansExecution'}");
appendCommandStatus(result, commandStat);
return false;
}
}
if (Object != cmdObj.firstElement().type()) {
Status commandStat(ErrorCodes::BadValue,
"explain command requires a nested object");
appendCommandStatus(result, commandStat);
return false;
}
// This is the nested command which we are explaining.
BSONObj explainObj = cmdObj.firstElement().Obj();
Command* commToExplain = Command::findCommand(explainObj.firstElementFieldName());
if (NULL == commToExplain) {
mongoutils::str::stream ss;
ss << "unknown command: " << explainObj.firstElementFieldName();
Status explainStatus(ErrorCodes::CommandNotFound, ss);
return appendCommandStatus(result, explainStatus);
}
// Check whether the child command is allowed to run here. TODO: this logic is
// copied from Command::execCommand and should be abstracted. Until then, make
// sure to keep it up to date.
repl::ReplicationCoordinator* replCoord = repl::getGlobalReplicationCoordinator();
bool canRunHere =
replCoord->canAcceptWritesForDatabase(dbname) ||
commToExplain->slaveOk() ||
(commToExplain->slaveOverrideOk() && (options & QueryOption_SlaveOk));
if (!canRunHere) {
mongoutils::str::stream ss;
ss << "Explain's child command cannot run on this node. "
<< "Are you explaining a write command on a secondary?";
appendCommandStatus(result, false, ss);
return false;
}
// Actually call the nested command's explain(...) method.
Status explainStatus = commToExplain->explain(txn, dbname, explainObj, verbosity, &result);
if (!explainStatus.isOK()) {
return appendCommandStatus(result, explainStatus);
}
return true;
}
