bool QueryResponseBuilder::currentMatches() { MatchDetails details; if ( _cursor->currentMatches( &details ) ) { return true; } _explain->noteIterate( false, false, details.hasLoadedRecord(), false ); return false; }
void run() { Matcher matcher( BSON( "a.b" << 1 ) ); MatchDetails details; details.requestElemMatchKey(); ASSERT( !details.hasElemMatchKey() ); ASSERT( matcher.matches( fromjson( "{ a:[ { b:1 } ] }" ), &details ) ); // The '0' entry of the 'a' array is matched. ASSERT( details.hasElemMatchKey() ); ASSERT_EQUALS( string( "0" ), details.elemMatchKey() ); }
void run() { CollatorInterface* collator = nullptr; M matcher(BSON("a.b" << 1), ExtensionsCallbackDisallowExtensions(), collator); MatchDetails details; details.requestElemMatchKey(); ASSERT(!details.hasElemMatchKey()); ASSERT(matcher.matches(fromjson("{ a:[ { b:1 } ] }"), &details)); // The '0' entry of the 'a' array is matched. ASSERT(details.hasElemMatchKey()); ASSERT_EQUALS(string("0"), details.elemMatchKey()); }
void run() { boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); M matcher(BSON("a.b" << 1), expCtx); MatchDetails details; details.requestElemMatchKey(); ASSERT(!details.hasElemMatchKey()); ASSERT(matcher.matches(fromjson("{ a:[ { b:1 } ] }"), &details)); // The '0' entry of the 'a' array is matched. ASSERT(details.hasElemMatchKey()); ASSERT_EQUALS(string("0"), details.elemMatchKey()); }
TEST( ElemMatchValueMatchExpression, ElemMatchKey ) { BSONObj baseOperand = BSON( "$gt" << 6 ); auto_ptr<ComparisonMatchExpression> gt( new ComparisonMatchExpression() ); ASSERT( gt->init( "", ComparisonMatchExpression::GT, baseOperand[ "$gt" ] ).isOK() ); ElemMatchValueMatchExpression op; ASSERT( op.init( "a.b", gt.release() ).isOK() ); MatchDetails details; details.requestElemMatchKey(); ASSERT( !op.matches( BSONObj(), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( !op.matches( BSON( "a" << BSON( "b" << BSON_ARRAY( 2 ) ) ), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( op.matches( BSON( "a" << BSON( "b" << BSON_ARRAY( 3 << 7 ) ) ), &details ) ); ASSERT( details.hasElemMatchKey() ); // The entry within the $elemMatch array is reported. ASSERT_EQUALS( "1", details.elemMatchKey() ); ASSERT( op.matches( BSON( "a" << BSON_ARRAY( 1 << 2 << BSON( "b" << BSON_ARRAY( 3 << 7 ) ) ) ), &details ) ); ASSERT( details.hasElemMatchKey() ); // The entry within a parent of the $elemMatch array is reported. ASSERT_EQUALS( "2", details.elemMatchKey() ); }
TEST( AllMatchExpression, ElemMatchKey ) { BSONObj operand = BSON_ARRAY( 5 ); AllMatchExpression all; ASSERT( all.init( "a" ).isOK() ); all.getArrayFilterEntries()->addEquality( operand[0] ); MatchDetails details; details.requestElemMatchKey(); ASSERT( !all.matches( BSON( "a" << 4 ), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( all.matches( BSON( "a" << 5 ), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( all.matches( BSON( "a" << BSON_ARRAY( 1 << 2 << 5 ) ), &details ) ); // The elemMatchKey feature is not implemented for $all. ASSERT( !details.hasElemMatchKey() ); }
TEST(NotMatchExpression, ElemMatchKey) { BSONObj baseOperand = BSON("$lt" << 5); unique_ptr<ComparisonMatchExpression> lt(new LTMatchExpression()); ASSERT(lt->init("a", baseOperand["$lt"]).isOK()); NotMatchExpression notOp; ASSERT(notOp.init(lt.release()).isOK()); MatchDetails details; details.requestElemMatchKey(); ASSERT(!notOp.matchesBSON(BSON("a" << BSON_ARRAY(1)), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(notOp.matchesBSON(BSON("a" << 6), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(notOp.matchesBSON(BSON("a" << BSON_ARRAY(6)), &details)); // elemMatchKey is not implemented for negative match operators. ASSERT(!details.hasElemMatchKey()); }
void run() { client().insert( ns(), fromjson( "{ a:[ {}, { b:1 } ] }" ) ); Client::ReadContext context( ns() ); CoveredIndexMatcher matcher( BSON( "a.b" << 1 ), BSON( "$natural" << 1 ) ); MatchDetails details; details.requestElemMatchKey(); boost::shared_ptr<Cursor> cursor = NamespaceDetailsTransient::getCursor( ns(), BSONObj() ); // Verify that the cursor is unindexed. ASSERT_EQUALS( "BasicCursor", cursor->toString() ); ASSERT( matcher.matchesCurrent( cursor.get(), &details ) ); // The '1' entry of the 'a' array is matched. ASSERT( details.hasElemMatchKey() ); ASSERT_EQUALS( string( "1" ), details.elemMatchKey() ); }
Status ProjectionExec::transform(const BSONObj& in, BSONObj* out) const { // If it's a positional projection we need a MatchDetails. MatchDetails matchDetails; if (transformRequiresDetails()) { matchDetails.requestElemMatchKey(); verify(NULL != _queryExpression); verify(_queryExpression->matchesBSON(in, &matchDetails)); } BSONObjBuilder bob; Status s = transform(in, &bob, &matchDetails); if (!s.isOK()) { return s; } *out = bob.obj(); return Status::OK(); }
void Projection::transform( const BSONObj& in , BSONObjBuilder& b, const MatchDetails* details ) const { const ArrayOpType& arrayOpType = getArrayOpType(); BSONObjIterator i(in); while ( i.more() ) { BSONElement e = i.next(); if ( mongoutils::str::equals( "_id" , e.fieldName() ) ) { if ( _includeID ) b.append( e ); } else { Matchers::const_iterator matcher = _matchers.find( e.fieldName() ); if ( matcher == _matchers.end() ) { // no array projection matchers for this field append( b, e, details, arrayOpType ); } else { // field has array projection with $elemMatch specified. massert( 16348, "matchers are only supported for $elemMatch", arrayOpType == ARRAY_OP_ELEM_MATCH ); MatchDetails arrayDetails; arrayDetails.requestElemMatchKey(); if ( matcher->second->matches( in, &arrayDetails ) ) { LOG(4) << "Matched array on field: " << matcher->first << endl << " from array: " << in.getField( matcher->first ) << endl << " in object: " << in << endl << " at position: " << arrayDetails.elemMatchKey() << endl; FieldMap::const_iterator field = _fields.find( e.fieldName() ); massert( 16349, "$elemMatch specified, but projection field not found.", field != _fields.end() ); BSONArrayBuilder a; BSONObjBuilder o; massert( 16350, "$elemMatch called on document element with eoo", ! in.getField( e.fieldName() ).eoo() ); massert( 16351, "$elemMatch called on array element with eoo", ! in.getField( e.fieldName() ).Obj().getField( arrayDetails.elemMatchKey() ).eoo() ); a.append( in.getField( e.fieldName() ).Obj() .getField( arrayDetails.elemMatchKey() ) ); o.appendArray( matcher->first, a.arr() ); append( b, o.done().firstElement(), details, arrayOpType ); } } } } }
void run() { client().insert( ns(), fromjson( "{ a:[ {}, { b:1 } ] }" ) ); Client::Transaction transaction(DB_SERIALIZABLE); Client::ReadContext context( ns(), mongo::unittest::EMPTY_STRING ); CoveredIndexMatcher matcher( BSON( "a.b" << 1 ), BSON( "$natural" << 1 ) ); MatchDetails details; details.requestElemMatchKey(); boost::shared_ptr<Cursor> cursor = getOptimizedCursor( ns(), BSONObj() ); // Verify that the cursor is unindexed. ASSERT_EQUALS( "BasicCursor", cursor->toString() ); ASSERT( matcher.matchesCurrent( cursor.get(), &details ) ); // The '1' entry of the 'a' array is matched. ASSERT( details.hasElemMatchKey() ); ASSERT_EQUALS( string( "1" ), details.elemMatchKey() ); transaction.commit(); }
void run() { client().ensureIndex( ns(), BSON( "a.b" << 1 ) ); client().insert( ns(), fromjson( "{ a:[ {}, { b:9 }, { b:1 } ] }" ) ); Client::ReadContext context( ns() ); BSONObj query = BSON( "a.b" << 1 ); CoveredIndexMatcher matcher( query, BSON( "a.b" << 1 ) ); MatchDetails details; details.requestElemMatchKey(); boost::shared_ptr<Cursor> cursor = NamespaceDetailsTransient::getCursor( ns(), query ); // Verify that the cursor is indexed. ASSERT_EQUALS( "BtreeCursor a.b_1", cursor->toString() ); ASSERT( matcher.matchesCurrent( cursor.get(), &details ) ); // The '2' entry of the 'a' array is matched. ASSERT( details.hasElemMatchKey() ); ASSERT_EQUALS( string( "2" ), details.elemMatchKey() ); }
TEST(AndOp, ElemMatchKey) { BSONObj baseOperand1 = BSON("a" << 1); BSONObj baseOperand2 = BSON("b" << 2); unique_ptr<ComparisonMatchExpression> sub1(new EqualityMatchExpression()); ASSERT(sub1->init("a", baseOperand1["a"]).isOK()); unique_ptr<ComparisonMatchExpression> sub2(new EqualityMatchExpression()); ASSERT(sub2->init("b", baseOperand2["b"]).isOK()); AndMatchExpression andOp; andOp.add(sub1.release()); andOp.add(sub2.release()); MatchDetails details; details.requestElemMatchKey(); ASSERT(!andOp.matchesBSON(BSON("a" << BSON_ARRAY(1)), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(!andOp.matchesBSON(BSON("b" << BSON_ARRAY(2)), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(andOp.matchesBSON(BSON("a" << BSON_ARRAY(1) << "b" << BSON_ARRAY(1 << 2)), &details)); ASSERT(details.hasElemMatchKey()); // The elem match key for the second $and clause is recorded. ASSERT_EQUALS("1", details.elemMatchKey()); }
/* * Takes a cursor and updates the partial score for said cursor in _scores map * @param cursor, btree cursor pointing to the current document to be scored */ void FTSSearch::_process( BtreeCursor* cursor ) { _keysLookedAt++; BSONObj key = cursor->currKey(); BSONObjIterator i( key ); for ( unsigned j = 0; j < _ftsSpec.numExtraBefore(); j++) i.next(); i.next(); // move past indexToken BSONElement scoreElement = i.next(); double score = scoreElement.number(); double& cur = _scores[(cursor->currLoc()).rec()]; if ( cur < 0 ) { // already been rejected return; } if ( cur == 0 && _matcher.get() ) { // we haven't seen this before and we have a matcher MatchDetails d; if ( !_matcher->matchesCurrent( cursor, &d ) ) { cur = -1; } if ( d.hasLoadedRecord() ) _objectsLookedAt++; if ( cur == -1 ) return; } if ( cur ) cur += score * (1 + 1 / score); else cur += score; }
TEST(NorOp, ElemMatchKey) { BSONObj baseOperand1 = BSON("a" << 1); BSONObj baseOperand2 = BSON("b" << 2); unique_ptr<ComparisonMatchExpression> sub1(new EqualityMatchExpression()); ASSERT(sub1->init("a", baseOperand1["a"]).isOK()); unique_ptr<ComparisonMatchExpression> sub2(new EqualityMatchExpression()); ASSERT(sub2->init("b", baseOperand2["b"]).isOK()); NorMatchExpression norOp; norOp.add(sub1.release()); norOp.add(sub2.release()); MatchDetails details; details.requestElemMatchKey(); ASSERT(!norOp.matchesBSON(BSON("a" << 1), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(!norOp.matchesBSON(BSON("a" << BSON_ARRAY(1) << "b" << BSON_ARRAY(10)), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(norOp.matchesBSON(BSON("a" << BSON_ARRAY(3) << "b" << BSON_ARRAY(4)), &details)); // The elem match key feature is not implemented for $nor. ASSERT(!details.hasElemMatchKey()); }
void run() { client().ensureIndex( ns(), BSON( "a.b" << 1 ) ); client().insert( ns(), fromjson( "{ a:[ { b:1 } ] }" ) ); Client::Transaction transaction(DB_SERIALIZABLE); Client::ReadContext context( ns(), mongo::unittest::EMPTY_STRING ); BSONObj query = BSON( "a.b" << 1 ); CoveredIndexMatcher matcher( query, BSON( "a.b" << 1 ) ); MatchDetails details; details.requestElemMatchKey(); boost::shared_ptr<Cursor> cursor = getOptimizedCursor( ns(), query ); // Verify that the cursor is indexed. ASSERT_EQUALS( "IndexCursor a.b_1", cursor->toString() ); // Verify that the cursor is not multikey. ASSERT( !cursor->isMultiKey() ); ASSERT( matcher.matchesCurrent( cursor.get(), &details ) ); // The '0' entry of the 'a' array is matched. ASSERT( details.hasElemMatchKey() ); ASSERT_EQUALS( string( "0" ), details.elemMatchKey() ); transaction.commit(); }
TEST(OrOp, ElemMatchKey) { BSONObj baseOperand1 = BSON("a" << 1); BSONObj baseOperand2 = BSON("b" << 2); const CollatorInterface* collator = nullptr; unique_ptr<ComparisonMatchExpression> sub1(new EqualityMatchExpression(collator)); ASSERT(sub1->init("a", baseOperand1["a"]).isOK()); unique_ptr<ComparisonMatchExpression> sub2(new EqualityMatchExpression(collator)); ASSERT(sub2->init("b", baseOperand2["b"]).isOK()); OrMatchExpression orOp; orOp.add(sub1.release()); orOp.add(sub2.release()); MatchDetails details; details.requestElemMatchKey(); ASSERT(!orOp.matchesBSON(BSONObj(), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(!orOp.matchesBSON(BSON("a" << BSON_ARRAY(10) << "b" << BSON_ARRAY(10)), &details)); ASSERT(!details.hasElemMatchKey()); ASSERT(orOp.matchesBSON(BSON("a" << BSON_ARRAY(1) << "b" << BSON_ARRAY(1 << 2)), &details)); // The elem match key feature is not implemented for $or. ASSERT(!details.hasElemMatchKey()); }
TEST( OrOp, ElemMatchKey ) { BSONObj baseOperand1 = BSON( "a" << 1 ); BSONObj baseOperand2 = BSON( "b" << 2 ); auto_ptr<ComparisonMatchExpression> sub1( new ComparisonMatchExpression() ); ASSERT( sub1->init( "a", ComparisonMatchExpression::EQ, baseOperand1[ "a" ] ).isOK() ); auto_ptr<ComparisonMatchExpression> sub2( new ComparisonMatchExpression() ); ASSERT( sub2->init( "b", ComparisonMatchExpression::EQ, baseOperand2[ "b" ] ).isOK() ); OrMatchExpression orOp; orOp.add( sub1.release() ); orOp.add( sub2.release() ); MatchDetails details; details.requestElemMatchKey(); ASSERT( !orOp.matches( BSONObj(), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( !orOp.matches( BSON( "a" << BSON_ARRAY( 10 ) << "b" << BSON_ARRAY( 10 ) ), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( orOp.matches( BSON( "a" << BSON_ARRAY( 1 ) << "b" << BSON_ARRAY( 1 << 2 ) ), &details ) ); // The elem match key feature is not implemented for $or. ASSERT( !details.hasElemMatchKey() ); }
TEST( SizeMatchExpression, ElemMatchKey ) { SizeMatchExpression size; ASSERT( size.init( "a.b", 3 ).isOK() ); MatchDetails details; details.requestElemMatchKey(); ASSERT( !size.matches( BSON( "a" << 1 ), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( size.matches( BSON( "a" << BSON( "b" << BSON_ARRAY( 1 << 2 << 3 ) ) ), &details ) ); ASSERT( !details.hasElemMatchKey() ); ASSERT( size.matches( BSON( "a" << BSON_ARRAY( 2 << BSON( "b" << BSON_ARRAY( 1 << 2 << 3 ) ) ) ), &details ) ); ASSERT( details.hasElemMatchKey() ); ASSERT_EQUALS( "1", details.elemMatchKey() ); }
void UpdateStage::transformAndUpdate(BSONObj& oldObj, DiskLoc& loc) { const UpdateRequest* request = _params.request; UpdateDriver* driver = _params.driver; CanonicalQuery* cq = _params.canonicalQuery; UpdateLifecycle* lifecycle = request->getLifecycle(); // Ask the driver to apply the mods. It may be that the driver can apply those "in // place", that is, some values of the old document just get adjusted without any // change to the binary layout on the bson layer. It may be that a whole new // document is needed to accomodate the new bson layout of the resulting document. _doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled); BSONObj logObj; FieldRefSet updatedFields; Status status = Status::OK(); if (!driver->needMatchDetails()) { // If we don't need match details, avoid doing the rematch status = driver->update(StringData(), &_doc, &logObj, &updatedFields); } else { // If there was a matched field, obtain it. MatchDetails matchDetails; matchDetails.requestElemMatchKey(); dassert(cq); verify(cq->root()->matchesBSON(oldObj, &matchDetails)); string matchedField; if (matchDetails.hasElemMatchKey()) matchedField = matchDetails.elemMatchKey(); // TODO: Right now, each mod checks in 'prepare' that if it needs positional // data, that a non-empty StringData() was provided. In principle, we could do // that check here in an else clause to the above conditional and remove the // checks from the mods. status = driver->update(matchedField, &_doc, &logObj, &updatedFields); } if (!status.isOK()) { uasserted(16837, status.reason()); } // Ensure _id exists and is first uassertStatusOK(ensureIdAndFirst(_doc)); // If the driver applied the mods in place, we can ask the mutable for what // changed. We call those changes "damages". :) We use the damages to inform the // journal what was changed, and then apply them to the original document // ourselves. If, however, the driver applied the mods out of place, we ask it to // generate a new, modified document for us. In that case, the file manager will // take care of the journaling details for us. // // This code flow is admittedly odd. But, right now, journaling is baked in the file // manager. And if we aren't using the file manager, we have to do jounaling // ourselves. bool docWasModified = false; BSONObj newObj; const char* source = NULL; bool inPlace = _doc.getInPlaceUpdates(&_damages, &source); // If something changed in the document, verify that no immutable fields were changed // and data is valid for storage. if ((!inPlace || !_damages.empty()) ) { if (!(request->isFromReplication() || request->isFromMigration())) { const std::vector<FieldRef*>* immutableFields = NULL; if (lifecycle) immutableFields = lifecycle->getImmutableFields(); uassertStatusOK(validate(oldObj, updatedFields, _doc, immutableFields, driver->modOptions()) ); } } // Save state before making changes saveState(); { WriteUnitOfWork wunit(request->getOpCtx()); if (inPlace && !driver->modsAffectIndices()) { // If a set of modifiers were all no-ops, we are still 'in place', but there // is no work to do, in which case we want to consider the object unchanged. if (!_damages.empty() ) { // Don't actually do the write if this is an explain. if (!request->isExplain()) { invariant(_collection); const RecordData oldRec(oldObj.objdata(), oldObj.objsize()); _collection->updateDocumentWithDamages(request->getOpCtx(), loc, oldRec, source, _damages); } docWasModified = true; _specificStats.fastmod = true; } newObj = oldObj; } else { // The updates were not in place. Apply them through the file manager. newObj = _doc.getObject(); uassert(17419, str::stream() << "Resulting document after update is larger than " << BSONObjMaxUserSize, newObj.objsize() <= BSONObjMaxUserSize); docWasModified = true; // Don't actually do the write if this is an explain. if (!request->isExplain()) { invariant(_collection); StatusWith<DiskLoc> res = _collection->updateDocument(request->getOpCtx(), loc, newObj, true, _params.opDebug); uassertStatusOK(res.getStatus()); DiskLoc newLoc = res.getValue(); // If the document moved, we might see it again in a collection scan (maybe it's // a document after our current document). // // If the document is indexed and the mod changes an indexed value, we might see // it again. For an example, see the comment above near declaration of // updatedLocs. if (_updatedLocs && (newLoc != loc || driver->modsAffectIndices())) { _updatedLocs->insert(newLoc); } } } // Call logOp if requested, and we're not an explain. if (request->shouldCallLogOp() && !logObj.isEmpty() && !request->isExplain()) { BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request->isMulti()); repl::logOp(request->getOpCtx(), "u", request->getNamespaceString().ns().c_str(), logObj, &idQuery, NULL, request->isFromMigration()); } wunit.commit(); } // Restore state after modification // As restoreState may restore (recreate) cursors, make sure to restore the // state outside of the WritUnitOfWork. restoreState(request->getOpCtx()); // Only record doc modifications if they wrote (exclude no-ops). Explains get // recorded as if they wrote. if (docWasModified) { _specificStats.nModified++; } }
UpdateResult update( OperationContext* txn, Database* db, const UpdateRequest& request, OpDebug* opDebug, UpdateDriver* driver, CanonicalQuery* cq) { LOG(3) << "processing update : " << request; std::auto_ptr<CanonicalQuery> cqHolder(cq); const NamespaceString& nsString = request.getNamespaceString(); UpdateLifecycle* lifecycle = request.getLifecycle(); Collection* collection = db->getCollection(txn, nsString.ns()); validateUpdate(nsString.ns().c_str(), request.getUpdates(), request.getQuery()); // TODO: This seems a bit circuitious. opDebug->updateobj = request.getUpdates(); if (lifecycle) { lifecycle->setCollection(collection); driver->refreshIndexKeys(lifecycle->getIndexKeys()); } Runner* rawRunner; Status status = cq ? getRunner(collection, cqHolder.release(), &rawRunner) : getRunner(collection, nsString.ns(), request.getQuery(), &rawRunner, &cq); uassert(17243, "could not get runner " + request.getQuery().toString() + "; " + causedBy(status), status.isOK()); // Create the runner and setup all deps. auto_ptr<Runner> runner(rawRunner); // Register Runner with ClientCursor const ScopedRunnerRegistration safety(runner.get()); // // We'll start assuming we have one or more documents for this update. (Otherwise, // we'll fall-back to insert case (if upsert is true).) // // We are an update until we fall into the insert case below. driver->setContext(ModifierInterface::ExecInfo::UPDATE_CONTEXT); int numMatched = 0; // If the update was in-place, we may see it again. This only matters if we're doing // a multi-update; if we're not doing a multi-update we stop after one update and we // won't see any more docs. // // For example: If we're scanning an index {x:1} and performing {$inc:{x:5}}, we'll keep // moving the document forward and it will continue to reappear in our index scan. // Unless the index is multikey, the underlying query machinery won't de-dup. // // If the update wasn't in-place we may see it again. Our query may return the new // document and we wouldn't want to update that. // // So, no matter what, we keep track of where the doc wound up. typedef unordered_set<DiskLoc, DiskLoc::Hasher> DiskLocSet; const scoped_ptr<DiskLocSet> updatedLocs(request.isMulti() ? new DiskLocSet : NULL); // Reset these counters on each call. We might re-enter this function to retry this // update if we throw a page fault exception below, and we rely on these counters // reflecting only the actions taken locally. In particlar, we must have the no-op // counter reset so that we can meaningfully comapre it with numMatched above. opDebug->nscanned = 0; opDebug->nscannedObjects = 0; opDebug->nModified = 0; // Get the cached document from the update driver. mutablebson::Document& doc = driver->getDocument(); mutablebson::DamageVector damages; // Used during iteration of docs BSONObj oldObj; // Get first doc, and location Runner::RunnerState state = Runner::RUNNER_ADVANCED; uassert(ErrorCodes::NotMaster, mongoutils::str::stream() << "Not primary while updating " << nsString.ns(), !request.shouldCallLogOp() || repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase( nsString.db())); while (true) { // Get next doc, and location DiskLoc loc; state = runner->getNext(&oldObj, &loc); if (state != Runner::RUNNER_ADVANCED) { if (state == Runner::RUNNER_EOF) { // We have reached the logical end of the loop, so do yielding recovery break; } else { uassertStatusOK(Status(ErrorCodes::InternalError, str::stream() << " Update query failed -- " << Runner::statestr(state))); } } // We fill this with the new locs of moved doc so we don't double-update. if (updatedLocs && updatedLocs->count(loc) > 0) { continue; } // We count how many documents we scanned even though we may skip those that are // deemed duplicated. The final 'numMatched' and 'nscanned' numbers may differ for // that reason. // TODO: Do we want to pull this out of the underlying query plan? opDebug->nscanned++; // Found a matching document opDebug->nscannedObjects++; numMatched++; // Ask the driver to apply the mods. It may be that the driver can apply those "in // place", that is, some values of the old document just get adjusted without any // change to the binary layout on the bson layer. It may be that a whole new // document is needed to accomodate the new bson layout of the resulting document. doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled); BSONObj logObj; FieldRefSet updatedFields; Status status = Status::OK(); if (!driver->needMatchDetails()) { // If we don't need match details, avoid doing the rematch status = driver->update(StringData(), &doc, &logObj, &updatedFields); } else { // If there was a matched field, obtain it. MatchDetails matchDetails; matchDetails.requestElemMatchKey(); dassert(cq); verify(cq->root()->matchesBSON(oldObj, &matchDetails)); string matchedField; if (matchDetails.hasElemMatchKey()) matchedField = matchDetails.elemMatchKey(); // TODO: Right now, each mod checks in 'prepare' that if it needs positional // data, that a non-empty StringData() was provided. In principle, we could do // that check here in an else clause to the above conditional and remove the // checks from the mods. status = driver->update(matchedField, &doc, &logObj, &updatedFields); } if (!status.isOK()) { uasserted(16837, status.reason()); } // Ensure _id exists and is first uassertStatusOK(ensureIdAndFirst(doc)); // If the driver applied the mods in place, we can ask the mutable for what // changed. We call those changes "damages". :) We use the damages to inform the // journal what was changed, and then apply them to the original document // ourselves. If, however, the driver applied the mods out of place, we ask it to // generate a new, modified document for us. In that case, the file manager will // take care of the journaling details for us. // // This code flow is admittedly odd. But, right now, journaling is baked in the file // manager. And if we aren't using the file manager, we have to do jounaling // ourselves. bool docWasModified = false; BSONObj newObj; const char* source = NULL; bool inPlace = doc.getInPlaceUpdates(&damages, &source); // If something changed in the document, verify that no immutable fields were changed // and data is valid for storage. if ((!inPlace || !damages.empty()) ) { if (!(request.isFromReplication() || request.isFromMigration())) { const std::vector<FieldRef*>* immutableFields = NULL; if (lifecycle) immutableFields = lifecycle->getImmutableFields(); uassertStatusOK(validate(oldObj, updatedFields, doc, immutableFields, driver->modOptions()) ); } } // Save state before making changes runner->saveState(); if (inPlace && !driver->modsAffectIndices()) { // If a set of modifiers were all no-ops, we are still 'in place', but there is // no work to do, in which case we want to consider the object unchanged. if (!damages.empty() ) { collection->updateDocumentWithDamages( txn, loc, source, damages ); docWasModified = true; opDebug->fastmod = true; } newObj = oldObj; } else { // The updates were not in place. Apply them through the file manager. // XXX: With experimental document-level locking, we do not hold the sufficient // locks, so this would cause corruption. fassert(18516, !useExperimentalDocLocking); newObj = doc.getObject(); uassert(17419, str::stream() << "Resulting document after update is larger than " << BSONObjMaxUserSize, newObj.objsize() <= BSONObjMaxUserSize); StatusWith<DiskLoc> res = collection->updateDocument(txn, loc, newObj, true, opDebug); uassertStatusOK(res.getStatus()); DiskLoc newLoc = res.getValue(); docWasModified = true; // If the document moved, we might see it again in a collection scan (maybe it's // a document after our current document). // // If the document is indexed and the mod changes an indexed value, we might see it // again. For an example, see the comment above near declaration of updatedLocs. if (updatedLocs && (newLoc != loc || driver->modsAffectIndices())) { updatedLocs->insert(newLoc); } } // Restore state after modification uassert(17278, "Update could not restore runner state after updating a document.", runner->restoreState(txn)); // Call logOp if requested. if (request.shouldCallLogOp() && !logObj.isEmpty()) { BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request.isMulti()); repl::logOp(txn, "u", nsString.ns().c_str(), logObj , &idQuery, NULL, request.isFromMigration()); } // Only record doc modifications if they wrote (exclude no-ops) if (docWasModified) opDebug->nModified++; if (!request.isMulti()) { break; } // Opportunity for journaling to write during the update. txn->recoveryUnit()->commitIfNeeded(); } // TODO: Can this be simplified? if ((numMatched > 0) || (numMatched == 0 && !request.isUpsert()) ) { opDebug->nMatched = numMatched; return UpdateResult(numMatched > 0 /* updated existing object(s) */, !driver->isDocReplacement() /* $mod or obj replacement */, opDebug->nModified /* number of modified docs, no no-ops */, numMatched /* # of docs matched/updated, even no-ops */, BSONObj()); } // // We haven't found any existing document so an insert is done // (upsert is true). // opDebug->upsert = true; // Since this is an insert (no docs found and upsert:true), we will be logging it // as an insert in the oplog. We don't need the driver's help to build the // oplog record, then. We also set the context of the update driver to the INSERT_CONTEXT. // Some mods may only work in that context (e.g. $setOnInsert). driver->setLogOp(false); driver->setContext(ModifierInterface::ExecInfo::INSERT_CONTEXT); // Reset the document we will be writing to doc.reset(); // This remains the empty object in the case of an object replacement, but in the case // of an upsert where we are creating a base object from the query and applying mods, // we capture the query as the original so that we can detect immutable field mutations. BSONObj original = BSONObj(); // Calling createFromQuery will populate the 'doc' with fields from the query which // creates the base of the update for the inserterd doc (because upsert was true) if (cq) { uassertStatusOK(driver->populateDocumentWithQueryFields(cq, doc)); // Validate the base doc, as taken from the query -- no fields means validate all. FieldRefSet noFields; uassertStatusOK(validate(BSONObj(), noFields, doc, NULL, driver->modOptions())); if (!driver->isDocReplacement()) { opDebug->fastmodinsert = true; // We need all the fields from the query to compare against for validation below. original = doc.getObject(); } else { original = request.getQuery(); } } else { fassert(17354, CanonicalQuery::isSimpleIdQuery(request.getQuery())); BSONElement idElt = request.getQuery()["_id"]; original = idElt.wrap(); fassert(17352, doc.root().appendElement(idElt)); } // Apply the update modifications and then log the update as an insert manually. FieldRefSet updatedFields; status = driver->update(StringData(), &doc, NULL, &updatedFields); if (!status.isOK()) { uasserted(16836, status.reason()); } // Ensure _id exists and is first uassertStatusOK(ensureIdAndFirst(doc)); // Validate that the object replacement or modifiers resulted in a document // that contains all the immutable keys and can be stored. if (!(request.isFromReplication() || request.isFromMigration())){ const std::vector<FieldRef*>* immutableFields = NULL; if (lifecycle) immutableFields = lifecycle->getImmutableFields(); // This will only validate the modified fields if not a replacement. uassertStatusOK(validate(original, updatedFields, doc, immutableFields, driver->modOptions()) ); } // Only create the collection if the doc will be inserted. if (!collection) { collection = db->getCollection(txn, request.getNamespaceString().ns()); if (!collection) { collection = db->createCollection(txn, request.getNamespaceString().ns()); } } // Insert the doc BSONObj newObj = doc.getObject(); uassert(17420, str::stream() << "Document to upsert is larger than " << BSONObjMaxUserSize, newObj.objsize() <= BSONObjMaxUserSize); StatusWith<DiskLoc> newLoc = collection->insertDocument(txn, newObj, !request.isGod() /*enforceQuota*/); uassertStatusOK(newLoc.getStatus()); if (request.shouldCallLogOp()) { repl::logOp(txn, "i", nsString.ns().c_str(), newObj, NULL, NULL, request.isFromMigration()); } opDebug->nMatched = 1; return UpdateResult(false /* updated a non existing document */, !driver->isDocReplacement() /* $mod or obj replacement? */, 1 /* docs written*/, 1 /* count of updated documents */, newObj /* object that was upserted */ ); }
bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { Timer t; string ns = dbname + '.' + cmdObj.firstElement().valuestr(); string key = cmdObj["key"].valuestrsafe(); BSONObj keyPattern = BSON( key << 1 ); BSONObj query = getQuery( cmdObj ); int bufSize = BSONObjMaxUserSize - 4096; BufBuilder bb( bufSize ); char * start = bb.buf(); BSONArrayBuilder arr( bb ); BSONElementSet values; long long nscanned = 0; // locations looked at long long nscannedObjects = 0; // full objects looked at long long n = 0; // matches MatchDetails md; Collection *cl = getCollection( ns ); if ( ! cl ) { result.appendArray( "values" , BSONObj() ); result.append( "stats" , BSON( "n" << 0 << "nscanned" << 0 << "nscannedObjects" << 0 ) ); return true; } shared_ptr<Cursor> cursor; if ( ! query.isEmpty() ) { cursor = getOptimizedCursor(ns.c_str() , query , BSONObj() ); } else { // query is empty, so lets see if we can find an index // with the key so we don't have to hit the raw data for (int i = 0; i < cl->nIndexes(); i++) { IndexDetails &idx = cl->idx(i); if (cl->isMultikey(i)) { continue; } if ( idx.inKeyPattern( key ) ) { cursor = getBestGuessCursor( ns.c_str() , BSONObj() , idx.keyPattern() ); if( cursor.get() ) break; } } if ( ! cursor.get() ) { cursor = getOptimizedCursor(ns.c_str() , query , BSONObj() ); } } verify( cursor ); string cursorName = cursor->toString(); auto_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns)); for ( ; cursor->ok(); cursor->advance() ) { nscanned++; bool loadedRecord = false; if ( cursor->currentMatches( &md ) && !cursor->getsetdup( cursor->currPK() ) ) { n++; BSONObj holder; BSONElementSet temp; loadedRecord = ! cc->getFieldsDotted( key , temp, holder ); for ( BSONElementSet::iterator i=temp.begin(); i!=temp.end(); ++i ) { BSONElement e = *i; if ( values.count( e ) ) continue; int now = bb.len(); uassert(10044, "distinct too big, 16mb cap", ( now + e.size() + 1024 ) < bufSize ); arr.append( e ); BSONElement x( start + now ); values.insert( x ); } } if ( loadedRecord || md.hasLoadedRecord() ) nscannedObjects++; RARELY killCurrentOp.checkForInterrupt(); } verify( start == bb.buf() ); result.appendArray( "values" , arr.done() ); { BSONObjBuilder b; b.appendNumber( "n" , n ); b.appendNumber( "nscanned" , nscanned ); b.appendNumber( "nscannedObjects" , nscannedObjects ); b.appendNumber( "timems" , t.millis() ); b.append( "cursor" , cursorName ); result.append( "stats" , b.obj() ); } return true; }
BSONObj UpdateStage::transformAndUpdate(const Snapshotted<BSONObj>& oldObj, RecordId& recordId) { const UpdateRequest* request = _params.request; UpdateDriver* driver = _params.driver; CanonicalQuery* cq = _params.canonicalQuery; UpdateLifecycle* lifecycle = request->getLifecycle(); // If asked to return new doc, default to the oldObj, in case nothing changes. BSONObj newObj = oldObj.value(); // Ask the driver to apply the mods. It may be that the driver can apply those "in // place", that is, some values of the old document just get adjusted without any // change to the binary layout on the bson layer. It may be that a whole new document // is needed to accomodate the new bson layout of the resulting document. In any event, // only enable in-place mutations if the underlying storage engine offers support for // writing damage events. _doc.reset(oldObj.value(), (_collection->updateWithDamagesSupported() ? mutablebson::Document::kInPlaceEnabled : mutablebson::Document::kInPlaceDisabled)); BSONObj logObj; bool docWasModified = false; Status status = Status::OK(); const bool validateForStorage = getOpCtx()->writesAreReplicated() && _enforceOkForStorage; FieldRefSet immutablePaths; if (getOpCtx()->writesAreReplicated() && !request->isFromMigration()) { if (lifecycle) { auto immutablePathsVector = getImmutableFields(getOpCtx(), request->getNamespaceString()); if (immutablePathsVector) { immutablePaths.fillFrom( transitional_tools_do_not_use::unspool_vector(*immutablePathsVector)); } } immutablePaths.keepShortest(&idFieldRef); } if (!driver->needMatchDetails()) { // If we don't need match details, avoid doing the rematch status = driver->update( StringData(), &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified); } else { // If there was a matched field, obtain it. MatchDetails matchDetails; matchDetails.requestElemMatchKey(); dassert(cq); verify(cq->root()->matchesBSON(oldObj.value(), &matchDetails)); string matchedField; if (matchDetails.hasElemMatchKey()) matchedField = matchDetails.elemMatchKey(); status = driver->update( matchedField, &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified); } if (!status.isOK()) { uasserted(16837, status.reason()); } // Skip adding _id field if the collection is capped (since capped collection documents can // neither grow nor shrink). const auto createIdField = !_collection->isCapped(); // Ensure if _id exists it is first status = ensureIdFieldIsFirst(&_doc); if (status.code() == ErrorCodes::InvalidIdField) { // Create ObjectId _id field if we are doing that if (createIdField) { addObjectIDIdField(&_doc); } } else { uassertStatusOK(status); } // See if the changes were applied in place const char* source = NULL; const bool inPlace = _doc.getInPlaceUpdates(&_damages, &source); if (inPlace && _damages.empty()) { // An interesting edge case. A modifier didn't notice that it was really a no-op // during its 'prepare' phase. That represents a missed optimization, but we still // shouldn't do any real work. Toggle 'docWasModified' to 'false'. // // Currently, an example of this is '{ $push : { x : {$each: [], $sort: 1} } }' when the 'x' // array exists and is already sorted. docWasModified = false; } if (docWasModified) { // Prepare to write back the modified document WriteUnitOfWork wunit(getOpCtx()); RecordId newRecordId; OplogUpdateEntryArgs args; if (!request->isExplain()) { invariant(_collection); auto* css = CollectionShardingState::get(getOpCtx(), _collection->ns()); args.nss = _collection->ns(); args.uuid = _collection->uuid(); args.stmtId = request->getStmtId(); args.update = logObj; args.criteria = css->getMetadata().extractDocumentKey(newObj); uassert(16980, "Multi-update operations require all documents to have an '_id' field", !request->isMulti() || args.criteria.hasField("_id"_sd)); args.fromMigrate = request->isFromMigration(); args.storeDocOption = getStoreDocMode(*request); if (args.storeDocOption == OplogUpdateEntryArgs::StoreDocOption::PreImage) { args.preImageDoc = oldObj.value().getOwned(); } } if (inPlace) { if (!request->isExplain()) { newObj = oldObj.value(); const RecordData oldRec(oldObj.value().objdata(), oldObj.value().objsize()); Snapshotted<RecordData> snap(oldObj.snapshotId(), oldRec); StatusWith<RecordData> newRecStatus = _collection->updateDocumentWithDamages( getOpCtx(), recordId, std::move(snap), source, _damages, &args); newObj = uassertStatusOK(std::move(newRecStatus)).releaseToBson(); } newRecordId = recordId; } else { // The updates were not in place. Apply them through the file manager. newObj = _doc.getObject(); uassert(17419, str::stream() << "Resulting document after update is larger than " << BSONObjMaxUserSize, newObj.objsize() <= BSONObjMaxUserSize); if (!request->isExplain()) { newRecordId = _collection->updateDocument(getOpCtx(), recordId, oldObj, newObj, true, driver->modsAffectIndices(), _params.opDebug, &args); } } invariant(oldObj.snapshotId() == getOpCtx()->recoveryUnit()->getSnapshotId()); wunit.commit(); // If the document moved, we might see it again in a collection scan (maybe it's // a document after our current document). // // If the document is indexed and the mod changes an indexed value, we might see // it again. For an example, see the comment above near declaration of // updatedRecordIds. // // This must be done after the wunit commits so we are sure we won't be rolling back. if (_updatedRecordIds && (newRecordId != recordId || driver->modsAffectIndices())) { _updatedRecordIds->insert(newRecordId); } } // Only record doc modifications if they wrote (exclude no-ops). Explains get // recorded as if they wrote. if (docWasModified || request->isExplain()) { _specificStats.nModified++; } return newObj; }
UpdateResult _updateObjectsNEW( bool su, const char* ns, const BSONObj& updateobj, const BSONObj& patternOrig, bool upsert, bool multi, bool logop , OpDebug& debug, RemoveSaver* rs, bool fromMigrate, const QueryPlanSelectionPolicy& planPolicy, bool forReplication ) { // TODO // + Separate UpdateParser from UpdateRunner (the latter should be "stage-y") // + All the yield and deduplicate logic would move to the query stage // portion of it // // + Replication related // + fast path for update for query by _id // + support for relaxing viable path constraint in replication // // + Field Management // + Force all upsert to contain _id // + Prevent changes to immutable fields (_id, and those mentioned by sharding) // // + Yiedling related // + $atomic support (or better, support proper yielding if not) // + page fault support debug.updateobj = updateobj; NamespaceDetails* d = nsdetails( ns ); NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get( ns ); UpdateDriver::Options opts; opts.multi = multi; opts.upsert = upsert; opts.logOp = logop; UpdateDriver driver( opts ); Status status = driver.parse( nsdt->indexKeys(), updateobj ); if ( !status.isOK() ) { uasserted( 16840, status.reason() ); } shared_ptr<Cursor> cursor = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy ); // The 'cursor' the optimizer gave us may contain query plans that generate duplicate // diskloc's. We set up here the mechanims that will prevent us from processing those // twice if we see them. We also set up a 'ClientCursor' so that we can support // yielding. const bool dedupHere = cursor->autoDedup(); shared_ptr<Cursor> cPtr = cursor; auto_ptr<ClientCursor> clientCursor( new ClientCursor( QueryOption_NoCursorTimeout, cPtr, ns ) ); // // Upsert Logic // // We may or may not have documents for this update. If we don't, then try to upsert, // if allowed. if ( !cursor->ok() && upsert ) { // If this is a $mod base update, we need to generate a document by examining the // query and the mods. Otherwise, we can use the object replacement sent by the user // update command that was parsed by the driver before. BSONObj oldObj; if ( *updateobj.firstElementFieldName() == '$' ) { if ( !driver.createFromQuery( patternOrig, &oldObj ) ) { uasserted( 16835, "cannot create object to update" ); } debug.fastmodinsert = true; } else { debug.upsert = true; } // Since this is an upsert, we will be oplogging it as an insert. We don't // need the driver's help to build the oplog record, then. We also set the // context of the update driver to an "upsert". Some mods may only work in that // context (e.g. $setOnInsert). driver.setLogOp( false ); driver.setContext( ModifierInterface::ExecInfo::INSERT_CONTEXT ); mutablebson::Document doc( oldObj, mutablebson::Document::kInPlaceDisabled ); status = driver.update( StringData(), &doc, NULL /* no oplog record */); if ( !status.isOK() ) { uasserted( 16836, status.reason() ); } BSONObj newObj = doc.getObject(); theDataFileMgr.insertWithObjMod( ns, newObj, false, su ); if ( logop ) { logOp( "i", ns, newObj, 0, 0, fromMigrate, &newObj ); } return UpdateResult( false /* updated a non existing document */, driver.dollarModMode() /* $mod or obj replacement? */, 1 /* count of updated documents */, newObj /* object that was upserted */ ); } // // We have one or more documents for this update. // // We record that this will not be an upsert, in case a mod doesn't want to be applied // when in strict update mode. driver.setContext( ModifierInterface::ExecInfo::UPDATE_CONTEXT ); // Let's fetch each of them and pipe them through the update expression, making sure to // keep track of the necessary stats. Recall that we'll be pulling documents out of // cursors and some of them do not deduplicate the entries they generate. We have // deduping logic in here, too -- for now. unordered_set<DiskLoc, DiskLoc::Hasher> seenLocs; int numUpdated = 0; debug.nscanned = 0; while ( cursor->ok() ) { // Let's fetch the next candidate object for this update. Record* r = cursor->_current(); DiskLoc loc = cursor->currLoc(); const BSONObj oldObj = loc.obj(); // We count how many documents we scanned even though we may skip those that are // deemed duplicated. The final 'numUpdated' and 'nscanned' numbers may differ for // that reason. debug.nscanned++; // Skips this document if it: // a) doesn't match the query portion of the update // b) was deemed duplicate by the underlying cursor machinery // // Now, if we are going to update the document, // c) we don't want to do so while the cursor is at it, as that may invalidate // the cursor. So, we advance to next document, before issuing the update. MatchDetails matchDetails; matchDetails.requestElemMatchKey(); if ( !cursor->currentMatches( &matchDetails ) ) { // a) cursor->advance(); continue; } else if ( cursor->getsetdup( loc ) && dedupHere ) { // b) cursor->advance(); continue; } else if (driver.dollarModMode() && multi) { // c) cursor->advance(); if ( dedupHere ) { if ( seenLocs.count( loc ) ) { continue; } } // There are certain kind of cursors that hold multiple pointers to data // underneath. $or cursors is one example. In a $or cursor, it may be the case // that when we did the last advance(), we finished consuming documents from // one of $or child and started consuming the next one. In that case, it is // possible that the last document of the previous child is the same as the // first document of the next (see SERVER-5198 and jstests/orp.js). // // So we advance the cursor here until we see a new diskloc. // // Note that we won't be yielding, and we may not do so for a while if we find // a particularly duplicated sequence of loc's. That is highly unlikely, // though. (See SERVER-5725, if curious, but "stage" based $or will make that // ticket moot). while( cursor->ok() && loc == cursor->currLoc() ) { cursor->advance(); } } // For some (unfortunate) historical reasons, not all cursors would be valid after // a write simply because we advanced them to a document not affected by the write. // To protect in those cases, not only we engaged in the advance() logic above, but // we also tell the cursor we're about to write a document that we've just seen. // prepareToTouchEarlierIterate() requires calling later // recoverFromTouchingEarlierIterate(), so we make a note here to do so. bool touchPreviousDoc = multi && cursor->ok(); if ( touchPreviousDoc ) { clientCursor->setDoingDeletes( true ); cursor->prepareToTouchEarlierIterate(); } // Ask the driver to apply the mods. It may be that the driver can apply those "in // place", that is, some values of the old document just get adjusted without any // change to the binary layout on the bson layer. It may be that a whole new // document is needed to accomodate the new bson layout of the resulting document. mutablebson::Document doc( oldObj, mutablebson::Document::kInPlaceEnabled ); BSONObj logObj; StringData matchedField = matchDetails.hasElemMatchKey() ? matchDetails.elemMatchKey(): StringData(); status = driver.update( matchedField, &doc, &logObj ); if ( !status.isOK() ) { uasserted( 16837, status.reason() ); } // If the driver applied the mods in place, we can ask the mutable for what // changed. We call those changes "damages". :) We use the damages to inform the // journal what was changed, and then apply them to the original document // ourselves. If, however, the driver applied the mods out of place, we ask it to // generate a new, modified document for us. In that case, the file manager will // take care of the journaling details for us. // // This code flow is admittedly odd. But, right now, journaling is baked in the file // manager. And if we aren't using the file manager, we have to do jounaling // ourselves. BSONObj newObj; const char* source = NULL; mutablebson::DamageVector damages; bool inPlace = doc.getInPlaceUpdates(&damages, &source); if ( inPlace && !driver.modsAffectIndices() ) { // All updates were in place. Apply them via durability and writing pointer. mutablebson::DamageVector::const_iterator where = damages.begin(); const mutablebson::DamageVector::const_iterator end = damages.end(); for( ; where != end; ++where ) { const char* sourcePtr = source + where->sourceOffset; void* targetPtr = getDur().writingPtr( const_cast<char*>(oldObj.objdata()) + where->targetOffset, where->size); std::memcpy(targetPtr, sourcePtr, where->size); } newObj = oldObj; debug.fastmod = true; } else { // The updates were not in place. Apply them through the file manager. newObj = doc.getObject(); DiskLoc newLoc = theDataFileMgr.updateRecord(ns, d, nsdt, r, loc, newObj.objdata(), newObj.objsize(), debug); // If we've moved this object to a new location, make sure we don't apply // that update again if our traversal picks the objecta again. // // We also take note that the diskloc if the updates are affecting indices. // Chances are that we're traversing one of them and they may be multi key and // therefore duplicate disklocs. if ( newLoc != loc || driver.modsAffectIndices() ) { seenLocs.insert( newLoc ); } } // Log Obj if ( logop ) { if ( !logObj.isEmpty() ) { BSONObj pattern = patternOrig; logOp("u", ns, logObj , &pattern, 0, fromMigrate, &newObj ); } } // One more document updated. numUpdated++; if (!multi) { break; } // If we used the cursor mechanism that prepares an earlier seen document for a // write we need to tell such mechanisms that the write is over. if ( touchPreviousDoc ) { cursor->recoverFromTouchingEarlierIterate(); } getDur().commitIfNeeded(); } return UpdateResult( true /* updated existing object(s) */, driver.dollarModMode() /* $mod or obj replacement */, numUpdated /* # of docments update */, BSONObj() ); }
UpdateResult _updateObjects( const char* ns, const BSONObj& updateobj, const BSONObj& patternOrig, bool upsert, bool multi, bool logop , OpDebug& debug, bool fromMigrate, const QueryPlanSelectionPolicy& planPolicy ) { TOKULOG(2) << "update: " << ns << " update: " << updateobj << " query: " << patternOrig << " upsert: " << upsert << " multi: " << multi << endl; debug.updateobj = updateobj; NamespaceDetails *d = getAndMaybeCreateNS(ns, logop); auto_ptr<ModSet> mods; const bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$'; bool modsAreIndexed = false; if ( isOperatorUpdate ) { if ( d->indexBuildInProgress() ) { set<string> bgKeys; d->inProgIdx().keyPattern().getFieldNames(bgKeys); mods.reset( new ModSet(updateobj, d->indexKeys(), &bgKeys) ); } else { mods.reset( new ModSet(updateobj, d->indexKeys()) ); } modsAreIndexed = mods->isIndexed(); } int idIdxNo = -1; if ( planPolicy.permitOptimalIdPlan() && !multi && !modsAreIndexed && (idIdxNo = d->findIdIndex()) >= 0 && mayUpdateById(d, patternOrig) ) { debug.idhack = true; IndexDetails &idx = d->idx(idIdxNo); BSONObj pk = idx.getKeyFromQuery(patternOrig); TOKULOG(3) << "_updateObjects using simple _id query, pattern " << patternOrig << ", pk " << pk << endl; UpdateResult result = _updateById( pk, isOperatorUpdate, mods.get(), d, ns, updateobj, patternOrig, logop, debug, fromMigrate); if ( result.existing || ! upsert ) { return result; } else if ( upsert && ! isOperatorUpdate && ! logop) { debug.upsert = true; BSONObj objModified = updateobj; insertAndLog( ns, d, objModified, logop, fromMigrate ); return UpdateResult( 0 , 0 , 1 , updateobj ); } } int numModded = 0; debug.nscanned = 0; shared_ptr<Cursor> c = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy ); if( c->ok() ) { set<BSONObj> seenObjects; MatchDetails details; auto_ptr<ClientCursor> cc; do { debug.nscanned++; if ( mods.get() && mods->hasDynamicArray() ) { // The Cursor must have a Matcher to record an elemMatchKey. But currently // a modifier on a dynamic array field may be applied even if there is no // elemMatchKey, so a matcher cannot be required. //verify( c->matcher() ); details.requestElemMatchKey(); } if ( !c->currentMatches( &details ) ) { c->advance(); continue; } BSONObj currPK = c->currPK(); if ( c->getsetdup( currPK ) ) { c->advance(); continue; } BSONObj currentObj = c->current(); BSONObj pattern = patternOrig; if ( logop ) { BSONObjBuilder idPattern; BSONElement id; // NOTE: If the matching object lacks an id, we'll log // with the original pattern. This isn't replay-safe. // It might make sense to suppress the log instead // if there's no id. if ( currentObj.getObjectID( id ) ) { idPattern.append( id ); pattern = idPattern.obj(); } else { uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi ); } } /* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some regular ones at the moment. */ struct LogOpUpdateDetails loud; loud.logop = logop; loud.ns = ns; loud.fromMigrate = fromMigrate; if ( isOperatorUpdate ) { if ( multi ) { // Make our own copies of the currPK and currentObj before we invalidate // them by advancing the cursor. currPK = currPK.copy(); currentObj = currentObj.copy(); // Advance past the document to be modified. This used to be because of SERVER-5198, // but TokuMX does it because we want to avoid needing to do manual deduplication // of this PK on the next iteration if the current update modifies the next // entry in the index. For example, an index scan over a:1 with mod {$inc: {a:1}} // would cause every other key read to be a duplicate if we didn't advance here. while ( c->ok() && currPK == c->currPK() ) { c->advance(); } // Multi updates need to do their own deduplication because updates may modify the // keys the cursor is in the process of scanning over. if ( seenObjects.count( currPK ) ) { continue; } else { seenObjects.insert( currPK ); } } ModSet* useMods = mods.get(); auto_ptr<ModSet> mymodset; if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) { useMods = mods->fixDynamicArray( details.elemMatchKey() ); mymodset.reset( useMods ); } auto_ptr<ModSetState> mss = useMods->prepare( currentObj ); updateUsingMods( d, currPK, currentObj, *mss, &loud ); numModded++; if ( ! multi ) return UpdateResult( 1 , 1 , numModded , BSONObj() ); continue; } // end if operator is update uassert( 10158 , "multi update only works with $ operators" , ! multi ); updateNoMods( d, currPK, currentObj, updateobj, &loud ); return UpdateResult( 1 , 0 , 1 , BSONObj() ); } while ( c->ok() ); } // endif if ( numModded ) return UpdateResult( 1 , 1 , numModded , BSONObj() ); if ( upsert ) { BSONObj newObj = updateobj; if ( updateobj.firstElementFieldName()[0] == '$' ) { // upsert of an $operation. build a default object BSONObj newObj = mods->createNewFromQuery( patternOrig ); debug.fastmodinsert = true; insertAndLog( ns, d, newObj, logop, fromMigrate ); return UpdateResult( 0 , 1 , 1 , newObj ); } uassert( 10159 , "multi update only works with $ operators" , ! multi ); debug.upsert = true; insertAndLog( ns, d, newObj, logop, fromMigrate ); return UpdateResult( 0 , 0 , 1 , newObj ); } return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() ); }
QueryResult* processGetMore(const char* ns, int ntoreturn, long long cursorid, CurOp& curop, int pass, bool& exhaust, bool* isCursorAuthorized ) { bool hasRunner = false; // Scoped to kill the pin after seeing if the runner's there. { // See if there's a runner. We do this until agg. is behind a Runner instead of a CC. ClientCursorPin p(cursorid); ClientCursor *cc = p.c(); if (NULL != cc && NULL != cc->getRunner()) { hasRunner = true; } } if (hasRunner) { return newGetMore(ns, ntoreturn, cursorid, curop, pass, exhaust, isCursorAuthorized); } exhaust = false; int bufSize = 512 + sizeof( QueryResult ) + MaxBytesToReturnToClientAtOnce; BufBuilder b( bufSize ); b.skip(sizeof(QueryResult)); int resultFlags = ResultFlag_AwaitCapable; int start = 0; int n = 0; scoped_ptr<Client::ReadContext> ctx(new Client::ReadContext(ns)); // call this readlocked so state can't change replVerifyReadsOk(); ClientCursorPin p(cursorid); ClientCursor *cc = p.c(); if ( unlikely(!cc) ) { LOGSOME << "getMore: cursorid not found " << ns << " " << cursorid << endl; cursorid = 0; resultFlags = ResultFlag_CursorNotFound; } else { // Some internal users create a ClientCursor with a Runner. Don't crash if this // happens. Instead, hand them off to the new framework. if (NULL != cc->getRunner()) { p.release(); return newGetMore(ns, ntoreturn, cursorid, curop, pass, exhaust, isCursorAuthorized); } // check for spoofing of the ns such that it does not match the one originally there for the cursor uassert(14833, "auth error", str::equals(ns, cc->ns().c_str())); *isCursorAuthorized = true; // This must be done after auth check to ensure proper cleanup. uassert(16951, "failing getmore due to set failpoint", !MONGO_FAIL_POINT(getMoreError)); // If the operation that spawned this cursor had a time limit set, apply leftover // time to this getmore. curop.setMaxTimeMicros( cc->getLeftoverMaxTimeMicros() ); killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. if ( pass == 0 ) cc->updateSlaveLocation( curop ); int queryOptions = cc->queryOptions(); curop.debug().query = cc->query(); curop.setQuery( cc->query() ); start = cc->pos(); Cursor *c = cc->c(); if (!c->requiresLock()) { // make sure it won't be destroyed under us fassert(16952, !c->shouldDestroyOnNSDeletion()); fassert(16953, !c->supportYields()); ctx.reset(); // unlocks } c->recoverFromYield(); DiskLoc last; // This metadata may be stale, but it's the state of chunking when the cursor was // created. CollectionMetadataPtr metadata = cc->getCollMetadata(); KeyPattern keyPattern( metadata ? metadata->getKeyPattern() : BSONObj() ); while ( 1 ) { if ( !c->ok() ) { if ( c->tailable() ) { // when a tailable cursor hits "EOF", ok() goes false, and current() is // null. however advance() can still be retries as a reactivation attempt. // when there is new data, it will return true. that's what we are doing // here. if ( c->advance() ) continue; if( n == 0 && (queryOptions & QueryOption_AwaitData) && pass < 1000 ) { return 0; } break; } p.release(); bool ok = ClientCursor::erase(cursorid); verify(ok); cursorid = 0; cc = 0; break; } MatchDetails details; if ( cc->fields && cc->fields->getArrayOpType() == Projection::ARRAY_OP_POSITIONAL ) { // field projection specified, and contains an array operator details.requestElemMatchKey(); } // in some cases (clone collection) there won't be a matcher if ( !c->currentMatches( &details ) ) { } else if ( metadata && !metadata->keyBelongsToMe( extractKey(c, keyPattern ) ) ) { LOG(2) << "cursor skipping document in un-owned chunk: " << c->current() << endl; } else { if( c->getsetdup(c->currLoc()) ) { //out() << " but it's a dup \n"; } else { last = c->currLoc(); n++; // Fill out the fields requested by the query. const Projection::KeyOnly *keyFieldsOnly = c->keyFieldsOnly(); if ( keyFieldsOnly ) { fillQueryResultFromObj( b, 0, keyFieldsOnly->hydrate( c->currKey() ), &details ); } else { DiskLoc loc = c->currLoc(); fillQueryResultFromObj( b, cc->fields.get(), c->current(), &details, ( ( cc->pq.get() && cc->pq->showDiskLoc() ) ? &loc : 0 ) ); } if ( ( ntoreturn && n >= ntoreturn ) || b.len() > MaxBytesToReturnToClientAtOnce ) { c->advance(); cc->incPos( n ); break; } } } c->advance(); if ( ! cc->yieldSometimes( ( c->ok() && c->keyFieldsOnly() ) ? ClientCursor::DontNeed : ClientCursor::WillNeed ) ) { ClientCursor::erase(cursorid); cursorid = 0; cc = 0; break; } } if ( cc ) { if ( c->supportYields() ) { ClientCursor::YieldData data; verify( cc->prepareToYield( data ) ); } else { cc->c()->noteLocation(); } cc->storeOpForSlave( last ); exhaust = cc->queryOptions() & QueryOption_Exhaust; // If the getmore had a time limit, remaining time is "rolled over" back to the // cursor (for use by future getmore ops). cc->setLeftoverMaxTimeMicros( curop.getRemainingMaxTimeMicros() ); } } QueryResult *qr = (QueryResult *) b.buf(); qr->len = b.len(); qr->setOperation(opReply); qr->_resultFlags() = resultFlags; qr->cursorId = cursorid; qr->startingFrom = start; qr->nReturned = n; b.decouple(); return qr; }
Status ProjectionExec::transform(const BSONObj& in, BSONObjBuilder* bob, const MatchDetails* details) const { const ArrayOpType& arrayOpType = _arrayOpType; BSONObjIterator it(in); while (it.more()) { BSONElement elt = it.next(); // Case 1: _id if (mongoutils::str::equals("_id", elt.fieldName())) { if (_includeID) { bob->append(elt); } continue; } // Case 2: no array projection for this field. Matchers::const_iterator matcher = _matchers.find(elt.fieldName()); if (_matchers.end() == matcher) { Status s = append(bob, elt, details, arrayOpType); if (!s.isOK()) { return s; } continue; } // Case 3: field has array projection with $elemMatch specified. if (ARRAY_OP_ELEM_MATCH != arrayOpType) { return Status(ErrorCodes::BadValue, "Matchers are only supported for $elemMatch"); } MatchDetails arrayDetails; arrayDetails.requestElemMatchKey(); if (matcher->second->matchesBSON(in, &arrayDetails)) { FieldMap::const_iterator fieldIt = _fields.find(elt.fieldName()); if (_fields.end() == fieldIt) { return Status(ErrorCodes::BadValue, "$elemMatch specified, but projection field not found."); } BSONArrayBuilder arrBuilder; BSONObjBuilder subBob; if (in.getField(elt.fieldName()).eoo()) { return Status(ErrorCodes::InternalError, "$elemMatch called on document element with eoo"); } if (in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey()).eoo()) { return Status(ErrorCodes::InternalError, "$elemMatch called on array element with eoo"); } arrBuilder.append( in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey())); subBob.appendArray(matcher->first, arrBuilder.arr()); Status status = append(bob, subBob.done().firstElement(), details, arrayOpType); if (!status.isOK()) { return status; } } } return Status::OK(); }
UpdateResult update(const UpdateRequest& request, OpDebug* opDebug, UpdateDriver* driver) { LOG(3) << "processing update : " << request; const NamespaceString& nsString = request.getNamespaceString(); validateUpdate( nsString.ns().c_str(), request.getUpdates(), request.getQuery() ); NamespaceDetails* nsDetails = nsdetails( nsString.ns() ); NamespaceDetailsTransient* nsDetailsTransient = &NamespaceDetailsTransient::get( nsString.ns().c_str() ); // TODO: This seems a bit circuitious. opDebug->updateobj = request.getUpdates(); driver->refreshIndexKeys( nsDetailsTransient->indexKeys() ); shared_ptr<Cursor> cursor = getOptimizedCursor( nsString.ns(), request.getQuery(), BSONObj(), request.getQueryPlanSelectionPolicy() ); // If the update was marked with '$isolated' (a.k.a '$atomic'), we are not allowed to // yield while evaluating the update loop below. // // TODO: Old code checks this repeatedly within the update loop. Is that necessary? It seems // that once atomic should be always atomic. const bool isolated = cursor->ok() && cursor->matcher() && cursor->matcher()->docMatcher().atomic(); // The 'cursor' the optimizer gave us may contain query plans that generate duplicate // diskloc's. We set up here the mechanims that will prevent us from processing those // twice if we see them. We also set up a 'ClientCursor' so that we can support // yielding. // // TODO: Is it valid to call this on a non-ok cursor? const bool dedupHere = cursor->autoDedup(); // // We'll start assuming we have one or more documents for this update. (Othwerwise, // we'll fallback to upserting.) // // We record that this will not be an upsert, in case a mod doesn't want to be applied // when in strict update mode. driver->setContext( ModifierInterface::ExecInfo::UPDATE_CONTEXT ); // Let's fetch each of them and pipe them through the update expression, making sure to // keep track of the necessary stats. Recall that we'll be pulling documents out of // cursors and some of them do not deduplicate the entries they generate. We have // deduping logic in here, too -- for now. unordered_set<DiskLoc, DiskLoc::Hasher> seenLocs; int numMatched = 0; // Reset these counters on each call. We might re-enter this function to retry this // update if we throw a page fault exception below, and we rely on these counters // reflecting only the actions taken locally. In particlar, we must have the no-op // counter reset so that we can meaningfully comapre it with numMatched above. opDebug->nscanned = 0; opDebug->nupdateNoops = 0; Client& client = cc(); mutablebson::Document doc; mutablebson::DamageVector damages; // If we are going to be yielding, we will need a ClientCursor scoped to this loop. We // only loop as long as the underlying cursor is OK. for ( auto_ptr<ClientCursor> clientCursor; cursor->ok(); ) { // If we haven't constructed a ClientCursor, and if the client allows us to throw // page faults, and if we are referring to a location that is likely not in // physical memory, then throw a PageFaultException. The entire operation will be // restarted. if ( clientCursor.get() == NULL && client.allowedToThrowPageFaultException() && !cursor->currLoc().isNull() && !cursor->currLoc().rec()->likelyInPhysicalMemory() ) { // We should never throw a PFE if we have already updated items. The numMatched // variable includes no-ops, which do not prevent us from raising a PFE, so if // numMatched is non-zero, we are still OK to throw as long all matched items // resulted in a no-op. dassert((numMatched == 0) || (numMatched == opDebug->nupdateNoops)); throw PageFaultException( cursor->currLoc().rec() ); } if ( !isolated && opDebug->nscanned != 0 ) { // We are permitted to yield. To do so we need a ClientCursor, so create one // now if we have not yet done so. if ( !clientCursor.get() ) clientCursor.reset( new ClientCursor( QueryOption_NoCursorTimeout, cursor, nsString.ns() ) ); // Ask the client cursor to yield. We get two bits of state back: whether or not // we yielded, and whether or not we correctly recovered from yielding. bool yielded = false; const bool recovered = clientCursor->yieldSometimes( ClientCursor::WillNeed, &yielded ); if ( !recovered ) { // If we failed to recover from the yield, then the ClientCursor is already // gone. Release it so we don't destroy it a second time. clientCursor.release(); break; } if ( !cursor->ok() ) { // If the cursor died while we were yielded, just get out of the update loop. break; } if ( yielded ) { // We yielded and recovered OK, and our cursor is still good. Details about // our namespace may have changed while we were yielded, so we re-acquire // them here. If we can't do so, escape the update loop. Otherwise, refresh // the driver so that it knows about what is currently indexed. nsDetails = nsdetails( nsString.ns() ); if ( !nsDetails ) break; nsDetailsTransient = &NamespaceDetailsTransient::get( nsString.ns().c_str() ); // TODO: This copies the index keys, but it may not need to do so. driver->refreshIndexKeys( nsDetailsTransient->indexKeys() ); } } // Let's fetch the next candidate object for this update. Record* record = cursor->_current(); DiskLoc loc = cursor->currLoc(); const BSONObj oldObj = loc.obj(); // We count how many documents we scanned even though we may skip those that are // deemed duplicated. The final 'numUpdated' and 'nscanned' numbers may differ for // that reason. opDebug->nscanned++; // Skips this document if it: // a) doesn't match the query portion of the update // b) was deemed duplicate by the underlying cursor machinery // // Now, if we are going to update the document, // c) we don't want to do so while the cursor is at it, as that may invalidate // the cursor. So, we advance to next document, before issuing the update. MatchDetails matchDetails; matchDetails.requestElemMatchKey(); if ( !cursor->currentMatches( &matchDetails ) ) { // a) cursor->advance(); continue; } else if ( cursor->getsetdup( loc ) && dedupHere ) { // b) cursor->advance(); continue; } else if (!driver->isDocReplacement() && request.isMulti()) { // c) cursor->advance(); if ( dedupHere ) { if ( seenLocs.count( loc ) ) { continue; } } // There are certain kind of cursors that hold multiple pointers to data // underneath. $or cursors is one example. In a $or cursor, it may be the case // that when we did the last advance(), we finished consuming documents from // one of $or child and started consuming the next one. In that case, it is // possible that the last document of the previous child is the same as the // first document of the next (see SERVER-5198 and jstests/orp.js). // // So we advance the cursor here until we see a new diskloc. // // Note that we won't be yielding, and we may not do so for a while if we find // a particularly duplicated sequence of loc's. That is highly unlikely, // though. (See SERVER-5725, if curious, but "stage" based $or will make that // ticket moot). while( cursor->ok() && loc == cursor->currLoc() ) { cursor->advance(); } } // For some (unfortunate) historical reasons, not all cursors would be valid after // a write simply because we advanced them to a document not affected by the write. // To protect in those cases, not only we engaged in the advance() logic above, but // we also tell the cursor we're about to write a document that we've just seen. // prepareToTouchEarlierIterate() requires calling later // recoverFromTouchingEarlierIterate(), so we make a note here to do so. bool touchPreviousDoc = request.isMulti() && cursor->ok(); if ( touchPreviousDoc ) { if ( clientCursor.get() ) clientCursor->setDoingDeletes( true ); cursor->prepareToTouchEarlierIterate(); } // Found a matching document numMatched++; // Ask the driver to apply the mods. It may be that the driver can apply those "in // place", that is, some values of the old document just get adjusted without any // change to the binary layout on the bson layer. It may be that a whole new // document is needed to accomodate the new bson layout of the resulting document. doc.reset( oldObj, mutablebson::Document::kInPlaceEnabled ); BSONObj logObj; // If there was a matched field, obtain it. string matchedField; if (matchDetails.hasElemMatchKey()) matchedField = matchDetails.elemMatchKey(); Status status = driver->update( matchedField, &doc, &logObj ); if ( !status.isOK() ) { uasserted( 16837, status.reason() ); } // If the driver applied the mods in place, we can ask the mutable for what // changed. We call those changes "damages". :) We use the damages to inform the // journal what was changed, and then apply them to the original document // ourselves. If, however, the driver applied the mods out of place, we ask it to // generate a new, modified document for us. In that case, the file manager will // take care of the journaling details for us. // // This code flow is admittedly odd. But, right now, journaling is baked in the file // manager. And if we aren't using the file manager, we have to do jounaling // ourselves. bool objectWasChanged = false; BSONObj newObj; const char* source = NULL; bool inPlace = doc.getInPlaceUpdates(&damages, &source); if ( inPlace && !driver->modsAffectIndices() ) { // If a set of modifiers were all no-ops, we are still 'in place', but there is // no work to do, in which case we want to consider the object unchanged. if (!damages.empty() ) { nsDetails->paddingFits(); // All updates were in place. Apply them via durability and writing pointer. mutablebson::DamageVector::const_iterator where = damages.begin(); const mutablebson::DamageVector::const_iterator end = damages.end(); for( ; where != end; ++where ) { const char* sourcePtr = source + where->sourceOffset; void* targetPtr = getDur().writingPtr( const_cast<char*>(oldObj.objdata()) + where->targetOffset, where->size); std::memcpy(targetPtr, sourcePtr, where->size); } objectWasChanged = true; opDebug->fastmod = true; } newObj = oldObj; } else { // The updates were not in place. Apply them through the file manager. newObj = doc.getObject(); DiskLoc newLoc = theDataFileMgr.updateRecord(nsString.ns().c_str(), nsDetails, nsDetailsTransient, record, loc, newObj.objdata(), newObj.objsize(), *opDebug); // If we've moved this object to a new location, make sure we don't apply // that update again if our traversal picks the objecta again. // // We also take note that the diskloc if the updates are affecting indices. // Chances are that we're traversing one of them and they may be multi key and // therefore duplicate disklocs. if ( newLoc != loc || driver->modsAffectIndices() ) { seenLocs.insert( newLoc ); } objectWasChanged = true; } // Log Obj if ( request.shouldUpdateOpLog() ) { if ( driver->isDocReplacement() || !logObj.isEmpty() ) { BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request.isMulti()); logOp("u", nsString.ns().c_str(), logObj , &idQuery, NULL, request.isFromMigration(), &newObj); } } // If it was noop since the document didn't change, record that. if (!objectWasChanged) opDebug->nupdateNoops++; if (!request.isMulti()) { break; } // If we used the cursor mechanism that prepares an earlier seen document for a // write we need to tell such mechanisms that the write is over. if ( touchPreviousDoc ) { cursor->recoverFromTouchingEarlierIterate(); } getDur().commitIfNeeded(); } // TODO: Can this be simplified? if ((numMatched > 0) || (numMatched == 0 && !request.isUpsert()) ) { opDebug->nupdated = numMatched; return UpdateResult( numMatched > 0 /* updated existing object(s) */, !driver->isDocReplacement() /* $mod or obj replacement */, numMatched /* # of docments update, even no-ops */, BSONObj() ); } // // We haven't found any existing document so an insert is done // (upsert is true). // opDebug->upsert = true; // Since this is an insert (no docs found and upsert:true), we will be logging it // as an insert in the oplog. We don't need the driver's help to build the // oplog record, then. We also set the context of the update driver to the INSERT_CONTEXT. // Some mods may only work in that context (e.g. $setOnInsert). driver->setLogOp( false ); driver->setContext( ModifierInterface::ExecInfo::INSERT_CONTEXT ); BSONObj baseObj; // Reset the document we will be writing to doc.reset( baseObj, mutablebson::Document::kInPlaceDisabled ); if ( request.getQuery().hasElement("_id") ) { uassertStatusOK(doc.root().appendElement(request.getQuery().getField("_id"))); } // If this is a $mod base update, we need to generate a document by examining the // query and the mods. Otherwise, we can use the object replacement sent by the user // update command that was parsed by the driver before. // In the following block we handle the query part, and then do the regular mods after. if ( *request.getUpdates().firstElementFieldName() == '$' ) { uassertStatusOK(UpdateDriver::createFromQuery(request.getQuery(), doc)); opDebug->fastmodinsert = true; } // Apply the update modifications and then log the update as an insert manually. Status status = driver->update( StringData(), &doc, NULL /* no oplog record */); if ( !status.isOK() ) { uasserted( 16836, status.reason() ); } BSONObj newObj = doc.getObject(); theDataFileMgr.insertWithObjMod( nsString.ns().c_str(), newObj, false, request.isGod() ); if ( request.shouldUpdateOpLog() ) { logOp( "i", nsString.ns().c_str(), newObj, NULL, NULL, request.isFromMigration(), &newObj ); } opDebug->nupdated = 1; return UpdateResult( false /* updated a non existing document */, !driver->isDocReplacement() /* $mod or obj replacement? */, 1 /* count of updated documents */, newObj /* object that was upserted */ ); }
Status ProjectionExec::transform(WorkingSetMember* member) const { if (_hasReturnKey) { BSONObj keyObj; if (member->hasComputed(WSM_INDEX_KEY)) { const IndexKeyComputedData* key = static_cast<const IndexKeyComputedData*>(member->getComputed(WSM_INDEX_KEY)); keyObj = key->getKey(); } member->obj = Snapshotted<BSONObj>(SnapshotId(), keyObj.getOwned()); member->keyData.clear(); member->loc = RecordId(); member->transitionToOwnedObj(); return Status::OK(); } BSONObjBuilder bob; if (member->hasObj()) { MatchDetails matchDetails; // If it's a positional projection we need a MatchDetails. if (transformRequiresDetails()) { matchDetails.requestElemMatchKey(); verify(NULL != _queryExpression); verify(_queryExpression->matchesBSON(member->obj.value(), &matchDetails)); } Status projStatus = transform(member->obj.value(), &bob, &matchDetails); if (!projStatus.isOK()) { return projStatus; } } else { verify(!requiresDocument()); // Go field by field. if (_includeID) { BSONElement elt; // Sometimes the _id field doesn't exist... if (member->getFieldDotted("_id", &elt) && !elt.eoo()) { bob.appendAs(elt, "_id"); } } BSONObjIterator it(_source); while (it.more()) { BSONElement specElt = it.next(); if (mongoutils::str::equals("_id", specElt.fieldName())) { continue; } BSONElement keyElt; // We can project a field that doesn't exist. We just ignore it. if (member->getFieldDotted(specElt.fieldName(), &keyElt) && !keyElt.eoo()) { bob.appendAs(keyElt, specElt.fieldName()); } } } for (MetaMap::const_iterator it = _meta.begin(); it != _meta.end(); ++it) { if (META_GEONEAR_DIST == it->second) { if (member->hasComputed(WSM_COMPUTED_GEO_DISTANCE)) { const GeoDistanceComputedData* dist = static_cast<const GeoDistanceComputedData*>( member->getComputed(WSM_COMPUTED_GEO_DISTANCE)); bob.append(it->first, dist->getDist()); } else { return Status(ErrorCodes::InternalError, "near loc dist requested but no data available"); } } else if (META_GEONEAR_POINT == it->second) { if (member->hasComputed(WSM_GEO_NEAR_POINT)) { const GeoNearPointComputedData* point = static_cast<const GeoNearPointComputedData*>( member->getComputed(WSM_GEO_NEAR_POINT)); BSONObj ptObj = point->getPoint(); if (ptObj.couldBeArray()) { bob.appendArray(it->first, ptObj); } else { bob.append(it->first, ptObj); } } else { return Status(ErrorCodes::InternalError, "near loc proj requested but no data available"); } } else if (META_TEXT_SCORE == it->second) { if (member->hasComputed(WSM_COMPUTED_TEXT_SCORE)) { const TextScoreComputedData* score = static_cast<const TextScoreComputedData*>( member->getComputed(WSM_COMPUTED_TEXT_SCORE)); bob.append(it->first, score->getScore()); } else { bob.append(it->first, 0.0); } } else if (META_RECORDID == it->second) { bob.append(it->first, static_cast<long long>(member->loc.repr())); } } BSONObj newObj = bob.obj(); member->obj = Snapshotted<BSONObj>(SnapshotId(), newObj); member->keyData.clear(); member->loc = RecordId(); member->transitionToOwnedObj(); return Status::OK(); }
UpdateResult _updateObjects( bool su, const char* ns, const BSONObj& updateobj, const BSONObj& patternOrig, bool upsert, bool multi, bool logop , OpDebug& debug, RemoveSaver* rs, bool fromMigrate, const QueryPlanSelectionPolicy& planPolicy, bool forReplication ) { DEBUGUPDATE( "update: " << ns << " update: " << updateobj << " query: " << patternOrig << " upsert: " << upsert << " multi: " << multi ); Client& client = cc(); debug.updateobj = updateobj; // The idea with these here it to make them loop invariant for // multi updates, and thus be a bit faster for that case. The // pointers may be left invalid on a failed or terminal yield // recovery. NamespaceDetails* d = nsdetails(ns); // can be null if an upsert... NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get(ns); auto_ptr<ModSet> mods; bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$'; int modsIsIndexed = false; // really the # of indexes if ( isOperatorUpdate ) { mods.reset( new ModSet(updateobj, nsdt->indexKeys(), forReplication) ); modsIsIndexed = mods->maxNumIndexUpdated(); } if( planPolicy.permitOptimalIdPlan() && !multi && isSimpleIdQuery(patternOrig) && d && !modsIsIndexed ) { int idxNo = d->findIdIndex(); if( idxNo >= 0 ) { debug.idhack = true; UpdateResult result = _updateById( isOperatorUpdate, idxNo, mods.get(), d, nsdt, su, ns, updateobj, patternOrig, logop, debug, fromMigrate); if ( result.existing || ! upsert ) { return result; } else if ( upsert && ! isOperatorUpdate ) { // this handles repl inserts checkNoMods( updateobj ); debug.upsert = true; BSONObj no = updateobj; theDataFileMgr.insertWithObjMod(ns, no, false, su); if ( logop ) logOp( "i", ns, no, 0, 0, fromMigrate, &no ); return UpdateResult( 0 , 0 , 1 , no ); } } } int numModded = 0; debug.nscanned = 0; shared_ptr<Cursor> c = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy ); d = nsdetails(ns); nsdt = &NamespaceDetailsTransient::get(ns); bool autoDedup = c->autoDedup(); if( c->ok() ) { set<DiskLoc> seenObjects; MatchDetails details; auto_ptr<ClientCursor> cc; do { if ( cc.get() == 0 && client.allowedToThrowPageFaultException() && ! c->currLoc().isNull() && ! c->currLoc().rec()->likelyInPhysicalMemory() ) { throw PageFaultException( c->currLoc().rec() ); } bool atomic = c->matcher() && c->matcher()->docMatcher().atomic(); if ( ! atomic && debug.nscanned > 0 ) { // we need to use a ClientCursor to yield if ( cc.get() == 0 ) { shared_ptr< Cursor > cPtr = c; cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) ); } bool didYield; if ( ! cc->yieldSometimes( ClientCursor::WillNeed, &didYield ) ) { cc.release(); break; } if ( !c->ok() ) { break; } if ( didYield ) { d = nsdetails(ns); if ( ! d ) break; nsdt = &NamespaceDetailsTransient::get(ns); if ( mods.get() ) { mods->setIndexedStatus( nsdt->indexKeys() ); modsIsIndexed = mods->maxNumIndexUpdated(); } } } // end yielding block debug.nscanned++; if ( mods.get() && mods->hasDynamicArray() ) { details.requestElemMatchKey(); } if ( !c->currentMatches( &details ) ) { c->advance(); continue; } Record* r = c->_current(); DiskLoc loc = c->currLoc(); if ( c->getsetdup( loc ) && autoDedup ) { c->advance(); continue; } BSONObj js = BSONObj::make(r); BSONObj pattern = patternOrig; if ( logop ) { BSONObjBuilder idPattern; BSONElement id; // NOTE: If the matching object lacks an id, we'll log // with the original pattern. This isn't replay-safe. // It might make sense to suppress the log instead // if there's no id. if ( js.getObjectID( id ) ) { idPattern.append( id ); pattern = idPattern.obj(); } else { uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi ); } } /* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some regular ones at the moment. */ if ( isOperatorUpdate ) { if ( multi ) { // go to next record in case this one moves c->advance(); // Update operations are deduped for cursors that implement their own // deduplication. In particular, some geo cursors are excluded. if ( autoDedup ) { if ( seenObjects.count( loc ) ) { continue; } // SERVER-5198 Advance past the document to be modified, provided // deduplication is enabled, but see SERVER-5725. while( c->ok() && loc == c->currLoc() ) { c->advance(); } } } const BSONObj& onDisk = loc.obj(); ModSet* useMods = mods.get(); auto_ptr<ModSet> mymodset; if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) { useMods = mods->fixDynamicArray( details.elemMatchKey() ); mymodset.reset( useMods ); } auto_ptr<ModSetState> mss = useMods->prepare( onDisk, false /* not an insertion */ ); bool willAdvanceCursor = multi && c->ok() && ( modsIsIndexed || ! mss->canApplyInPlace() ); if ( willAdvanceCursor ) { if ( cc.get() ) { cc->setDoingDeletes( true ); } c->prepareToTouchEarlierIterate(); } // If we've made it this far, "ns" must contain a valid collection name, and so // is of the form "db.collection". Therefore, the following expression must // always be valid. "system.users" updates must never be done in place, in // order to ensure that they are validated inside DataFileMgr::updateRecord(.). bool isSystemUsersMod = (NamespaceString(ns).coll == "system.users"); BSONObj newObj; if ( !mss->isUpdateIndexed() && mss->canApplyInPlace() && !isSystemUsersMod ) { mss->applyModsInPlace( true );// const_cast<BSONObj&>(onDisk) ); DEBUGUPDATE( "\t\t\t doing in place update" ); if ( !multi ) debug.fastmod = true; if ( modsIsIndexed ) { seenObjects.insert( loc ); } newObj = loc.obj(); d->paddingFits(); } else { newObj = mss->createNewFromMods(); checkTooLarge(newObj); DiskLoc newLoc = theDataFileMgr.updateRecord(ns, d, nsdt, r, loc, newObj.objdata(), newObj.objsize(), debug); if ( newLoc != loc || modsIsIndexed ){ // log() << "Moved obj " << newLoc.obj()["_id"] << " from " << loc << " to " << newLoc << endl; // object moved, need to make sure we don' get again seenObjects.insert( newLoc ); } } if ( logop ) { DEV verify( mods->size() ); BSONObj logObj = mss->getOpLogRewrite(); DEBUGUPDATE( "\t rewrite update: " << logObj ); // It is possible that the entire mod set was a no-op over this // document. We would have an empty log record in that case. If we // call logOp, with an empty record, that would be replicated as "clear // this record", which is not what we want. Therefore, to get a no-op // in the replica, we simply don't log. if ( logObj.nFields() ) { logOp("u", ns, logObj , &pattern, 0, fromMigrate, &newObj ); } } numModded++; if ( ! multi ) return UpdateResult( 1 , 1 , numModded , BSONObj() ); if ( willAdvanceCursor ) c->recoverFromTouchingEarlierIterate(); getDur().commitIfNeeded(); continue; } uassert( 10158 , "multi update only works with $ operators" , ! multi ); BSONElementManipulator::lookForTimestamps( updateobj ); checkNoMods( updateobj ); theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, su); if ( logop ) { DEV wassert( !su ); // super used doesn't get logged, this would be bad. logOp("u", ns, updateobj, &pattern, 0, fromMigrate, &updateobj ); } return UpdateResult( 1 , 0 , 1 , BSONObj() ); } while ( c->ok() ); } // endif if ( numModded ) return UpdateResult( 1 , 1 , numModded , BSONObj() ); if ( upsert ) { if ( updateobj.firstElementFieldName()[0] == '$' ) { // upsert of an $operation. build a default object BSONObj newObj = mods->createNewFromQuery( patternOrig ); checkNoMods( newObj ); debug.fastmodinsert = true; theDataFileMgr.insertWithObjMod(ns, newObj, false, su); if ( logop ) logOp( "i", ns, newObj, 0, 0, fromMigrate, &newObj ); return UpdateResult( 0 , 1 , 1 , newObj ); } uassert( 10159 , "multi update only works with $ operators" , ! multi ); checkNoMods( updateobj ); debug.upsert = true; BSONObj no = updateobj; theDataFileMgr.insertWithObjMod(ns, no, false, su); if ( logop ) logOp( "i", ns, no, 0, 0, fromMigrate, &no ); return UpdateResult( 0 , 0 , 1 , no ); } return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() ); }