bool QueryResponseBuilder::currentMatches() {
MatchDetails details;
if ( _cursor->currentMatches( &details ) ) {
return true;
}
_explain->noteIterate( false, false, details.hasLoadedRecord(), false );
return false;
}
void run() {
Matcher matcher( BSON( "a.b" << 1 ) );
MatchDetails details;
details.requestElemMatchKey();
ASSERT( !details.hasElemMatchKey() );
ASSERT( matcher.matches( fromjson( "{ a:[ { b:1 } ] }" ), &details ) );
// The '0' entry of the 'a' array is matched.
ASSERT( details.hasElemMatchKey() );
ASSERT_EQUALS( string( "0" ), details.elemMatchKey() );
}
void run() {
CollatorInterface* collator = nullptr;
M matcher(BSON("a.b" << 1), ExtensionsCallbackDisallowExtensions(), collator);
MatchDetails details;
details.requestElemMatchKey();
ASSERT(!details.hasElemMatchKey());
ASSERT(matcher.matches(fromjson("{ a:[ { b:1 } ] }"), &details));
// The '0' entry of the 'a' array is matched.
ASSERT(details.hasElemMatchKey());
ASSERT_EQUALS(string("0"), details.elemMatchKey());
}
void run() {
boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest());
M matcher(BSON("a.b" << 1), expCtx);
MatchDetails details;
details.requestElemMatchKey();
ASSERT(!details.hasElemMatchKey());
ASSERT(matcher.matches(fromjson("{ a:[ { b:1 } ] }"), &details));
// The '0' entry of the 'a' array is matched.
ASSERT(details.hasElemMatchKey());
ASSERT_EQUALS(string("0"), details.elemMatchKey());
}
TEST( ElemMatchValueMatchExpression, ElemMatchKey ) {
BSONObj baseOperand = BSON( "$gt" << 6 );
auto_ptr<ComparisonMatchExpression> gt( new ComparisonMatchExpression() );
ASSERT( gt->init( "", ComparisonMatchExpression::GT, baseOperand[ "$gt" ] ).isOK() );
ElemMatchValueMatchExpression op;
ASSERT( op.init( "a.b", gt.release() ).isOK() );
MatchDetails details;
details.requestElemMatchKey();
ASSERT( !op.matches( BSONObj(), &details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( !op.matches( BSON( "a" << BSON( "b" << BSON_ARRAY( 2 ) ) ),
&details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( op.matches( BSON( "a" << BSON( "b" << BSON_ARRAY( 3 << 7 ) ) ),
&details ) );
ASSERT( details.hasElemMatchKey() );
// The entry within the $elemMatch array is reported.
ASSERT_EQUALS( "1", details.elemMatchKey() );
ASSERT( op.matches( BSON( "a" <<
BSON_ARRAY( 1 << 2 <<
BSON( "b" << BSON_ARRAY( 3 << 7 ) ) ) ),
&details ) );
ASSERT( details.hasElemMatchKey() );
// The entry within a parent of the $elemMatch array is reported.
ASSERT_EQUALS( "2", details.elemMatchKey() );
}
TEST( AllMatchExpression, ElemMatchKey ) {
BSONObj operand = BSON_ARRAY( 5 );
AllMatchExpression all;
ASSERT( all.init( "a" ).isOK() );
all.getArrayFilterEntries()->addEquality( operand[0] );
MatchDetails details;
details.requestElemMatchKey();
ASSERT( !all.matches( BSON( "a" << 4 ), &details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( all.matches( BSON( "a" << 5 ), &details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( all.matches( BSON( "a" << BSON_ARRAY( 1 << 2 << 5 ) ), &details ) );
// The elemMatchKey feature is not implemented for $all.
ASSERT( !details.hasElemMatchKey() );
}
TEST(NotMatchExpression, ElemMatchKey) {
BSONObj baseOperand = BSON("$lt" << 5);
unique_ptr<ComparisonMatchExpression> lt(new LTMatchExpression());
ASSERT(lt->init("a", baseOperand["$lt"]).isOK());
NotMatchExpression notOp;
ASSERT(notOp.init(lt.release()).isOK());
MatchDetails details;
details.requestElemMatchKey();
ASSERT(!notOp.matchesBSON(BSON("a" << BSON_ARRAY(1)), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(notOp.matchesBSON(BSON("a" << 6), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(notOp.matchesBSON(BSON("a" << BSON_ARRAY(6)), &details));
// elemMatchKey is not implemented for negative match operators.
ASSERT(!details.hasElemMatchKey());
}
void run() {
client().insert( ns(), fromjson( "{ a:[ {}, { b:1 } ] }" ) );
Client::ReadContext context( ns() );
CoveredIndexMatcher matcher( BSON( "a.b" << 1 ), BSON( "$natural" << 1 ) );
MatchDetails details;
details.requestElemMatchKey();
boost::shared_ptr<Cursor> cursor = NamespaceDetailsTransient::getCursor( ns(), BSONObj() );
// Verify that the cursor is unindexed.
ASSERT_EQUALS( "BasicCursor", cursor->toString() );
ASSERT( matcher.matchesCurrent( cursor.get(), &details ) );
// The '1' entry of the 'a' array is matched.
ASSERT( details.hasElemMatchKey() );
ASSERT_EQUALS( string( "1" ), details.elemMatchKey() );
}
Status ProjectionExec::transform(const BSONObj& in, BSONObj* out) const {
// If it's a positional projection we need a MatchDetails.
MatchDetails matchDetails;
if (transformRequiresDetails()) {
matchDetails.requestElemMatchKey();
verify(NULL != _queryExpression);
verify(_queryExpression->matchesBSON(in, &matchDetails));
}
BSONObjBuilder bob;
Status s = transform(in, &bob, &matchDetails);
if (!s.isOK()) {
return s;
}
*out = bob.obj();
return Status::OK();
}
void Projection::transform( const BSONObj& in , BSONObjBuilder& b, const MatchDetails* details ) const {
const ArrayOpType& arrayOpType = getArrayOpType();
BSONObjIterator i(in);
while ( i.more() ) {
BSONElement e = i.next();
if ( mongoutils::str::equals( "_id" , e.fieldName() ) ) {
if ( _includeID )
b.append( e );
}
else {
Matchers::const_iterator matcher = _matchers.find( e.fieldName() );
if ( matcher == _matchers.end() ) {
// no array projection matchers for this field
append( b, e, details, arrayOpType );
} else {
// field has array projection with $elemMatch specified.
massert( 16348, "matchers are only supported for $elemMatch",
arrayOpType == ARRAY_OP_ELEM_MATCH );
MatchDetails arrayDetails;
arrayDetails.requestElemMatchKey();
if ( matcher->second->matches( in, &arrayDetails ) ) {
LOG(4) << "Matched array on field: " << matcher->first << endl
<< " from array: " << in.getField( matcher->first ) << endl
<< " in object: " << in << endl
<< " at position: " << arrayDetails.elemMatchKey() << endl;
FieldMap::const_iterator field = _fields.find( e.fieldName() );
massert( 16349, "$elemMatch specified, but projection field not found.",
field != _fields.end() );
BSONArrayBuilder a;
BSONObjBuilder o;
massert( 16350, "$elemMatch called on document element with eoo",
! in.getField( e.fieldName() ).eoo() );
massert( 16351, "$elemMatch called on array element with eoo",
! in.getField( e.fieldName() ).Obj().getField(
arrayDetails.elemMatchKey() ).eoo() );
a.append( in.getField( e.fieldName() ).Obj()
.getField( arrayDetails.elemMatchKey() ) );
o.appendArray( matcher->first, a.arr() );
append( b, o.done().firstElement(), details, arrayOpType );
}
}
}
}
}
void run() {
client().insert( ns(), fromjson( "{ a:[ {}, { b:1 } ] }" ) );
Client::Transaction transaction(DB_SERIALIZABLE);
Client::ReadContext context( ns(), mongo::unittest::EMPTY_STRING );
CoveredIndexMatcher matcher( BSON( "a.b" << 1 ), BSON( "$natural" << 1 ) );
MatchDetails details;
details.requestElemMatchKey();
boost::shared_ptr<Cursor> cursor = getOptimizedCursor( ns(), BSONObj() );
// Verify that the cursor is unindexed.
ASSERT_EQUALS( "BasicCursor", cursor->toString() );
ASSERT( matcher.matchesCurrent( cursor.get(), &details ) );
// The '1' entry of the 'a' array is matched.
ASSERT( details.hasElemMatchKey() );
ASSERT_EQUALS( string( "1" ), details.elemMatchKey() );
transaction.commit();
}
void run() {
client().ensureIndex( ns(), BSON( "a.b" << 1 ) );
client().insert( ns(), fromjson( "{ a:[ {}, { b:9 }, { b:1 } ] }" ) );
Client::ReadContext context( ns() );
BSONObj query = BSON( "a.b" << 1 );
CoveredIndexMatcher matcher( query, BSON( "a.b" << 1 ) );
MatchDetails details;
details.requestElemMatchKey();
boost::shared_ptr<Cursor> cursor = NamespaceDetailsTransient::getCursor( ns(), query );
// Verify that the cursor is indexed.
ASSERT_EQUALS( "BtreeCursor a.b_1", cursor->toString() );
ASSERT( matcher.matchesCurrent( cursor.get(), &details ) );
// The '2' entry of the 'a' array is matched.
ASSERT( details.hasElemMatchKey() );
ASSERT_EQUALS( string( "2" ), details.elemMatchKey() );
}
TEST(AndOp, ElemMatchKey) {
BSONObj baseOperand1 = BSON("a" << 1);
BSONObj baseOperand2 = BSON("b" << 2);
unique_ptr<ComparisonMatchExpression> sub1(new EqualityMatchExpression());
ASSERT(sub1->init("a", baseOperand1["a"]).isOK());
unique_ptr<ComparisonMatchExpression> sub2(new EqualityMatchExpression());
ASSERT(sub2->init("b", baseOperand2["b"]).isOK());
AndMatchExpression andOp;
andOp.add(sub1.release());
andOp.add(sub2.release());
MatchDetails details;
details.requestElemMatchKey();
ASSERT(!andOp.matchesBSON(BSON("a" << BSON_ARRAY(1)), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(!andOp.matchesBSON(BSON("b" << BSON_ARRAY(2)), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(andOp.matchesBSON(BSON("a" << BSON_ARRAY(1) << "b" << BSON_ARRAY(1 << 2)), &details));
ASSERT(details.hasElemMatchKey());
// The elem match key for the second $and clause is recorded.
ASSERT_EQUALS("1", details.elemMatchKey());
}
/*
* Takes a cursor and updates the partial score for said cursor in _scores map
* @param cursor, btree cursor pointing to the current document to be scored
*/
void FTSSearch::_process( BtreeCursor* cursor ) {
_keysLookedAt++;
BSONObj key = cursor->currKey();
BSONObjIterator i( key );
for ( unsigned j = 0; j < _ftsSpec.numExtraBefore(); j++)
i.next();
i.next(); // move past indexToken
BSONElement scoreElement = i.next();
double score = scoreElement.number();
double& cur = _scores[(cursor->currLoc()).rec()];
if ( cur < 0 ) {
// already been rejected
return;
}
if ( cur == 0 && _matcher.get() ) {
// we haven't seen this before and we have a matcher
MatchDetails d;
if ( !_matcher->matchesCurrent( cursor, &d ) ) {
cur = -1;
}
if ( d.hasLoadedRecord() )
_objectsLookedAt++;
if ( cur == -1 )
return;
}
if ( cur )
cur += score * (1 + 1 / score);
else
cur += score;
}
TEST(NorOp, ElemMatchKey) {
BSONObj baseOperand1 = BSON("a" << 1);
BSONObj baseOperand2 = BSON("b" << 2);
unique_ptr<ComparisonMatchExpression> sub1(new EqualityMatchExpression());
ASSERT(sub1->init("a", baseOperand1["a"]).isOK());
unique_ptr<ComparisonMatchExpression> sub2(new EqualityMatchExpression());
ASSERT(sub2->init("b", baseOperand2["b"]).isOK());
NorMatchExpression norOp;
norOp.add(sub1.release());
norOp.add(sub2.release());
MatchDetails details;
details.requestElemMatchKey();
ASSERT(!norOp.matchesBSON(BSON("a" << 1), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(!norOp.matchesBSON(BSON("a" << BSON_ARRAY(1) << "b" << BSON_ARRAY(10)), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(norOp.matchesBSON(BSON("a" << BSON_ARRAY(3) << "b" << BSON_ARRAY(4)), &details));
// The elem match key feature is not implemented for $nor.
ASSERT(!details.hasElemMatchKey());
}
void run() {
client().ensureIndex( ns(), BSON( "a.b" << 1 ) );
client().insert( ns(), fromjson( "{ a:[ { b:1 } ] }" ) );
Client::Transaction transaction(DB_SERIALIZABLE);
Client::ReadContext context( ns(), mongo::unittest::EMPTY_STRING );
BSONObj query = BSON( "a.b" << 1 );
CoveredIndexMatcher matcher( query, BSON( "a.b" << 1 ) );
MatchDetails details;
details.requestElemMatchKey();
boost::shared_ptr<Cursor> cursor = getOptimizedCursor( ns(), query );
// Verify that the cursor is indexed.
ASSERT_EQUALS( "IndexCursor a.b_1", cursor->toString() );
// Verify that the cursor is not multikey.
ASSERT( !cursor->isMultiKey() );
ASSERT( matcher.matchesCurrent( cursor.get(), &details ) );
// The '0' entry of the 'a' array is matched.
ASSERT( details.hasElemMatchKey() );
ASSERT_EQUALS( string( "0" ), details.elemMatchKey() );
transaction.commit();
}
TEST(OrOp, ElemMatchKey) {
BSONObj baseOperand1 = BSON("a" << 1);
BSONObj baseOperand2 = BSON("b" << 2);
const CollatorInterface* collator = nullptr;
unique_ptr<ComparisonMatchExpression> sub1(new EqualityMatchExpression(collator));
ASSERT(sub1->init("a", baseOperand1["a"]).isOK());
unique_ptr<ComparisonMatchExpression> sub2(new EqualityMatchExpression(collator));
ASSERT(sub2->init("b", baseOperand2["b"]).isOK());
OrMatchExpression orOp;
orOp.add(sub1.release());
orOp.add(sub2.release());
MatchDetails details;
details.requestElemMatchKey();
ASSERT(!orOp.matchesBSON(BSONObj(), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(!orOp.matchesBSON(BSON("a" << BSON_ARRAY(10) << "b" << BSON_ARRAY(10)), &details));
ASSERT(!details.hasElemMatchKey());
ASSERT(orOp.matchesBSON(BSON("a" << BSON_ARRAY(1) << "b" << BSON_ARRAY(1 << 2)), &details));
// The elem match key feature is not implemented for $or.
ASSERT(!details.hasElemMatchKey());
}
TEST( OrOp, ElemMatchKey ) {
BSONObj baseOperand1 = BSON( "a" << 1 );
BSONObj baseOperand2 = BSON( "b" << 2 );
auto_ptr<ComparisonMatchExpression> sub1( new ComparisonMatchExpression() );
ASSERT( sub1->init( "a", ComparisonMatchExpression::EQ, baseOperand1[ "a" ] ).isOK() );
auto_ptr<ComparisonMatchExpression> sub2( new ComparisonMatchExpression() );
ASSERT( sub2->init( "b", ComparisonMatchExpression::EQ, baseOperand2[ "b" ] ).isOK() );
OrMatchExpression orOp;
orOp.add( sub1.release() );
orOp.add( sub2.release() );
MatchDetails details;
details.requestElemMatchKey();
ASSERT( !orOp.matches( BSONObj(), &details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( !orOp.matches( BSON( "a" << BSON_ARRAY( 10 ) << "b" << BSON_ARRAY( 10 ) ),
&details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( orOp.matches( BSON( "a" << BSON_ARRAY( 1 ) << "b" << BSON_ARRAY( 1 << 2 ) ),
&details ) );
// The elem match key feature is not implemented for $or.
ASSERT( !details.hasElemMatchKey() );
}
TEST( SizeMatchExpression, ElemMatchKey ) {
SizeMatchExpression size;
ASSERT( size.init( "a.b", 3 ).isOK() );
MatchDetails details;
details.requestElemMatchKey();
ASSERT( !size.matches( BSON( "a" << 1 ), &details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( size.matches( BSON( "a" << BSON( "b" << BSON_ARRAY( 1 << 2 << 3 ) ) ), &details ) );
ASSERT( !details.hasElemMatchKey() );
ASSERT( size.matches( BSON( "a" <<
BSON_ARRAY( 2 <<
BSON( "b" << BSON_ARRAY( 1 << 2 << 3 ) ) ) ),
&details ) );
ASSERT( details.hasElemMatchKey() );
ASSERT_EQUALS( "1", details.elemMatchKey() );
}
void UpdateStage::transformAndUpdate(BSONObj& oldObj, DiskLoc& loc) {
const UpdateRequest* request = _params.request;
UpdateDriver* driver = _params.driver;
CanonicalQuery* cq = _params.canonicalQuery;
UpdateLifecycle* lifecycle = request->getLifecycle();
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new
// document is needed to accomodate the new bson layout of the resulting document.
_doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled);
BSONObj logObj;
FieldRefSet updatedFields;
Status status = Status::OK();
if (!driver->needMatchDetails()) {
// If we don't need match details, avoid doing the rematch
status = driver->update(StringData(), &_doc, &logObj, &updatedFields);
}
else {
// If there was a matched field, obtain it.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
dassert(cq);
verify(cq->root()->matchesBSON(oldObj, &matchDetails));
string matchedField;
if (matchDetails.hasElemMatchKey())
matchedField = matchDetails.elemMatchKey();
// TODO: Right now, each mod checks in 'prepare' that if it needs positional
// data, that a non-empty StringData() was provided. In principle, we could do
// that check here in an else clause to the above conditional and remove the
// checks from the mods.
status = driver->update(matchedField, &_doc, &logObj, &updatedFields);
}
if (!status.isOK()) {
uasserted(16837, status.reason());
}
// Ensure _id exists and is first
uassertStatusOK(ensureIdAndFirst(_doc));
// If the driver applied the mods in place, we can ask the mutable for what
// changed. We call those changes "damages". :) We use the damages to inform the
// journal what was changed, and then apply them to the original document
// ourselves. If, however, the driver applied the mods out of place, we ask it to
// generate a new, modified document for us. In that case, the file manager will
// take care of the journaling details for us.
//
// This code flow is admittedly odd. But, right now, journaling is baked in the file
// manager. And if we aren't using the file manager, we have to do jounaling
// ourselves.
bool docWasModified = false;
BSONObj newObj;
const char* source = NULL;
bool inPlace = _doc.getInPlaceUpdates(&_damages, &source);
// If something changed in the document, verify that no immutable fields were changed
// and data is valid for storage.
if ((!inPlace || !_damages.empty()) ) {
if (!(request->isFromReplication() || request->isFromMigration())) {
const std::vector<FieldRef*>* immutableFields = NULL;
if (lifecycle)
immutableFields = lifecycle->getImmutableFields();
uassertStatusOK(validate(oldObj,
updatedFields,
_doc,
immutableFields,
driver->modOptions()) );
}
}
// Save state before making changes
saveState();
{
WriteUnitOfWork wunit(request->getOpCtx());
if (inPlace && !driver->modsAffectIndices()) {
// If a set of modifiers were all no-ops, we are still 'in place', but there
// is no work to do, in which case we want to consider the object unchanged.
if (!_damages.empty() ) {
// Don't actually do the write if this is an explain.
if (!request->isExplain()) {
invariant(_collection);
const RecordData oldRec(oldObj.objdata(), oldObj.objsize());
_collection->updateDocumentWithDamages(request->getOpCtx(), loc,
oldRec, source, _damages);
}
docWasModified = true;
_specificStats.fastmod = true;
}
newObj = oldObj;
}
else {
// The updates were not in place. Apply them through the file manager.
newObj = _doc.getObject();
uassert(17419,
str::stream() << "Resulting document after update is larger than "
<< BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
docWasModified = true;
// Don't actually do the write if this is an explain.
if (!request->isExplain()) {
invariant(_collection);
StatusWith<DiskLoc> res = _collection->updateDocument(request->getOpCtx(),
loc,
newObj,
true,
_params.opDebug);
uassertStatusOK(res.getStatus());
DiskLoc newLoc = res.getValue();
// If the document moved, we might see it again in a collection scan (maybe it's
// a document after our current document).
//
// If the document is indexed and the mod changes an indexed value, we might see
// it again. For an example, see the comment above near declaration of
// updatedLocs.
if (_updatedLocs && (newLoc != loc || driver->modsAffectIndices())) {
_updatedLocs->insert(newLoc);
}
}
}
// Call logOp if requested, and we're not an explain.
if (request->shouldCallLogOp() && !logObj.isEmpty() && !request->isExplain()) {
BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request->isMulti());
repl::logOp(request->getOpCtx(),
"u",
request->getNamespaceString().ns().c_str(),
logObj,
&idQuery,
NULL,
request->isFromMigration());
}
wunit.commit();
}
// Restore state after modification
// As restoreState may restore (recreate) cursors, make sure to restore the
// state outside of the WritUnitOfWork.
restoreState(request->getOpCtx());
// Only record doc modifications if they wrote (exclude no-ops). Explains get
// recorded as if they wrote.
if (docWasModified) {
_specificStats.nModified++;
}
}
UpdateResult update(
OperationContext* txn,
Database* db,
const UpdateRequest& request,
OpDebug* opDebug,
UpdateDriver* driver,
CanonicalQuery* cq) {
LOG(3) << "processing update : " << request;
std::auto_ptr<CanonicalQuery> cqHolder(cq);
const NamespaceString& nsString = request.getNamespaceString();
UpdateLifecycle* lifecycle = request.getLifecycle();
Collection* collection = db->getCollection(txn, nsString.ns());
validateUpdate(nsString.ns().c_str(), request.getUpdates(), request.getQuery());
// TODO: This seems a bit circuitious.
opDebug->updateobj = request.getUpdates();
if (lifecycle) {
lifecycle->setCollection(collection);
driver->refreshIndexKeys(lifecycle->getIndexKeys());
}
Runner* rawRunner;
Status status = cq ?
getRunner(collection, cqHolder.release(), &rawRunner) :
getRunner(collection, nsString.ns(), request.getQuery(), &rawRunner, &cq);
uassert(17243,
"could not get runner " + request.getQuery().toString() + "; " + causedBy(status),
status.isOK());
// Create the runner and setup all deps.
auto_ptr<Runner> runner(rawRunner);
// Register Runner with ClientCursor
const ScopedRunnerRegistration safety(runner.get());
//
// We'll start assuming we have one or more documents for this update. (Otherwise,
// we'll fall-back to insert case (if upsert is true).)
//
// We are an update until we fall into the insert case below.
driver->setContext(ModifierInterface::ExecInfo::UPDATE_CONTEXT);
int numMatched = 0;
// If the update was in-place, we may see it again. This only matters if we're doing
// a multi-update; if we're not doing a multi-update we stop after one update and we
// won't see any more docs.
//
// For example: If we're scanning an index {x:1} and performing {$inc:{x:5}}, we'll keep
// moving the document forward and it will continue to reappear in our index scan.
// Unless the index is multikey, the underlying query machinery won't de-dup.
//
// If the update wasn't in-place we may see it again. Our query may return the new
// document and we wouldn't want to update that.
//
// So, no matter what, we keep track of where the doc wound up.
typedef unordered_set<DiskLoc, DiskLoc::Hasher> DiskLocSet;
const scoped_ptr<DiskLocSet> updatedLocs(request.isMulti() ? new DiskLocSet : NULL);
// Reset these counters on each call. We might re-enter this function to retry this
// update if we throw a page fault exception below, and we rely on these counters
// reflecting only the actions taken locally. In particlar, we must have the no-op
// counter reset so that we can meaningfully comapre it with numMatched above.
opDebug->nscanned = 0;
opDebug->nscannedObjects = 0;
opDebug->nModified = 0;
// Get the cached document from the update driver.
mutablebson::Document& doc = driver->getDocument();
mutablebson::DamageVector damages;
// Used during iteration of docs
BSONObj oldObj;
// Get first doc, and location
Runner::RunnerState state = Runner::RUNNER_ADVANCED;
uassert(ErrorCodes::NotMaster,
mongoutils::str::stream() << "Not primary while updating " << nsString.ns(),
!request.shouldCallLogOp()
|| repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(
nsString.db()));
while (true) {
// Get next doc, and location
DiskLoc loc;
state = runner->getNext(&oldObj, &loc);
if (state != Runner::RUNNER_ADVANCED) {
if (state == Runner::RUNNER_EOF) {
// We have reached the logical end of the loop, so do yielding recovery
break;
}
else {
uassertStatusOK(Status(ErrorCodes::InternalError,
str::stream() << " Update query failed -- "
<< Runner::statestr(state)));
}
}
// We fill this with the new locs of moved doc so we don't double-update.
if (updatedLocs && updatedLocs->count(loc) > 0) {
continue;
}
// We count how many documents we scanned even though we may skip those that are
// deemed duplicated. The final 'numMatched' and 'nscanned' numbers may differ for
// that reason.
// TODO: Do we want to pull this out of the underlying query plan?
opDebug->nscanned++;
// Found a matching document
opDebug->nscannedObjects++;
numMatched++;
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new
// document is needed to accomodate the new bson layout of the resulting document.
doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled);
BSONObj logObj;
FieldRefSet updatedFields;
Status status = Status::OK();
if (!driver->needMatchDetails()) {
// If we don't need match details, avoid doing the rematch
status = driver->update(StringData(), &doc, &logObj, &updatedFields);
}
else {
// If there was a matched field, obtain it.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
dassert(cq);
verify(cq->root()->matchesBSON(oldObj, &matchDetails));
string matchedField;
if (matchDetails.hasElemMatchKey())
matchedField = matchDetails.elemMatchKey();
// TODO: Right now, each mod checks in 'prepare' that if it needs positional
// data, that a non-empty StringData() was provided. In principle, we could do
// that check here in an else clause to the above conditional and remove the
// checks from the mods.
status = driver->update(matchedField, &doc, &logObj, &updatedFields);
}
if (!status.isOK()) {
uasserted(16837, status.reason());
}
// Ensure _id exists and is first
uassertStatusOK(ensureIdAndFirst(doc));
// If the driver applied the mods in place, we can ask the mutable for what
// changed. We call those changes "damages". :) We use the damages to inform the
// journal what was changed, and then apply them to the original document
// ourselves. If, however, the driver applied the mods out of place, we ask it to
// generate a new, modified document for us. In that case, the file manager will
// take care of the journaling details for us.
//
// This code flow is admittedly odd. But, right now, journaling is baked in the file
// manager. And if we aren't using the file manager, we have to do jounaling
// ourselves.
bool docWasModified = false;
BSONObj newObj;
const char* source = NULL;
bool inPlace = doc.getInPlaceUpdates(&damages, &source);
// If something changed in the document, verify that no immutable fields were changed
// and data is valid for storage.
if ((!inPlace || !damages.empty()) ) {
if (!(request.isFromReplication() || request.isFromMigration())) {
const std::vector<FieldRef*>* immutableFields = NULL;
if (lifecycle)
immutableFields = lifecycle->getImmutableFields();
uassertStatusOK(validate(oldObj,
updatedFields,
doc,
immutableFields,
driver->modOptions()) );
}
}
// Save state before making changes
runner->saveState();
if (inPlace && !driver->modsAffectIndices()) {
// If a set of modifiers were all no-ops, we are still 'in place', but there is
// no work to do, in which case we want to consider the object unchanged.
if (!damages.empty() ) {
collection->updateDocumentWithDamages( txn, loc, source, damages );
docWasModified = true;
opDebug->fastmod = true;
}
newObj = oldObj;
}
else {
// The updates were not in place. Apply them through the file manager.
// XXX: With experimental document-level locking, we do not hold the sufficient
// locks, so this would cause corruption.
fassert(18516, !useExperimentalDocLocking);
newObj = doc.getObject();
uassert(17419,
str::stream() << "Resulting document after update is larger than "
<< BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
StatusWith<DiskLoc> res = collection->updateDocument(txn,
loc,
newObj,
true,
opDebug);
uassertStatusOK(res.getStatus());
DiskLoc newLoc = res.getValue();
docWasModified = true;
// If the document moved, we might see it again in a collection scan (maybe it's
// a document after our current document).
//
// If the document is indexed and the mod changes an indexed value, we might see it
// again. For an example, see the comment above near declaration of updatedLocs.
if (updatedLocs && (newLoc != loc || driver->modsAffectIndices())) {
updatedLocs->insert(newLoc);
}
}
// Restore state after modification
uassert(17278,
"Update could not restore runner state after updating a document.",
runner->restoreState(txn));
// Call logOp if requested.
if (request.shouldCallLogOp() && !logObj.isEmpty()) {
BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request.isMulti());
repl::logOp(txn, "u", nsString.ns().c_str(), logObj , &idQuery,
NULL, request.isFromMigration());
}
// Only record doc modifications if they wrote (exclude no-ops)
if (docWasModified)
opDebug->nModified++;
if (!request.isMulti()) {
break;
}
// Opportunity for journaling to write during the update.
txn->recoveryUnit()->commitIfNeeded();
}
// TODO: Can this be simplified?
if ((numMatched > 0) || (numMatched == 0 && !request.isUpsert()) ) {
opDebug->nMatched = numMatched;
return UpdateResult(numMatched > 0 /* updated existing object(s) */,
!driver->isDocReplacement() /* $mod or obj replacement */,
opDebug->nModified /* number of modified docs, no no-ops */,
numMatched /* # of docs matched/updated, even no-ops */,
BSONObj());
}
//
// We haven't found any existing document so an insert is done
// (upsert is true).
//
opDebug->upsert = true;
// Since this is an insert (no docs found and upsert:true), we will be logging it
// as an insert in the oplog. We don't need the driver's help to build the
// oplog record, then. We also set the context of the update driver to the INSERT_CONTEXT.
// Some mods may only work in that context (e.g. $setOnInsert).
driver->setLogOp(false);
driver->setContext(ModifierInterface::ExecInfo::INSERT_CONTEXT);
// Reset the document we will be writing to
doc.reset();
// This remains the empty object in the case of an object replacement, but in the case
// of an upsert where we are creating a base object from the query and applying mods,
// we capture the query as the original so that we can detect immutable field mutations.
BSONObj original = BSONObj();
// Calling createFromQuery will populate the 'doc' with fields from the query which
// creates the base of the update for the inserterd doc (because upsert was true)
if (cq) {
uassertStatusOK(driver->populateDocumentWithQueryFields(cq, doc));
// Validate the base doc, as taken from the query -- no fields means validate all.
FieldRefSet noFields;
uassertStatusOK(validate(BSONObj(), noFields, doc, NULL, driver->modOptions()));
if (!driver->isDocReplacement()) {
opDebug->fastmodinsert = true;
// We need all the fields from the query to compare against for validation below.
original = doc.getObject();
}
else {
original = request.getQuery();
}
}
else {
fassert(17354, CanonicalQuery::isSimpleIdQuery(request.getQuery()));
BSONElement idElt = request.getQuery()["_id"];
original = idElt.wrap();
fassert(17352, doc.root().appendElement(idElt));
}
// Apply the update modifications and then log the update as an insert manually.
FieldRefSet updatedFields;
status = driver->update(StringData(), &doc, NULL, &updatedFields);
if (!status.isOK()) {
uasserted(16836, status.reason());
}
// Ensure _id exists and is first
uassertStatusOK(ensureIdAndFirst(doc));
// Validate that the object replacement or modifiers resulted in a document
// that contains all the immutable keys and can be stored.
if (!(request.isFromReplication() || request.isFromMigration())){
const std::vector<FieldRef*>* immutableFields = NULL;
if (lifecycle)
immutableFields = lifecycle->getImmutableFields();
// This will only validate the modified fields if not a replacement.
uassertStatusOK(validate(original,
updatedFields,
doc,
immutableFields,
driver->modOptions()) );
}
// Only create the collection if the doc will be inserted.
if (!collection) {
collection = db->getCollection(txn, request.getNamespaceString().ns());
if (!collection) {
collection = db->createCollection(txn, request.getNamespaceString().ns());
}
}
// Insert the doc
BSONObj newObj = doc.getObject();
uassert(17420,
str::stream() << "Document to upsert is larger than " << BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
StatusWith<DiskLoc> newLoc = collection->insertDocument(txn,
newObj,
!request.isGod() /*enforceQuota*/);
uassertStatusOK(newLoc.getStatus());
if (request.shouldCallLogOp()) {
repl::logOp(txn, "i", nsString.ns().c_str(), newObj,
NULL, NULL, request.isFromMigration());
}
opDebug->nMatched = 1;
return UpdateResult(false /* updated a non existing document */,
!driver->isDocReplacement() /* $mod or obj replacement? */,
1 /* docs written*/,
1 /* count of updated documents */,
newObj /* object that was upserted */ );
}
bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
Timer t;
string ns = dbname + '.' + cmdObj.firstElement().valuestr();
string key = cmdObj["key"].valuestrsafe();
BSONObj keyPattern = BSON( key << 1 );
BSONObj query = getQuery( cmdObj );
int bufSize = BSONObjMaxUserSize - 4096;
BufBuilder bb( bufSize );
char * start = bb.buf();
BSONArrayBuilder arr( bb );
BSONElementSet values;
long long nscanned = 0; // locations looked at
long long nscannedObjects = 0; // full objects looked at
long long n = 0; // matches
MatchDetails md;
Collection *cl = getCollection( ns );
if ( ! cl ) {
result.appendArray( "values" , BSONObj() );
result.append( "stats" , BSON( "n" << 0 << "nscanned" << 0 << "nscannedObjects" << 0 ) );
return true;
}
shared_ptr<Cursor> cursor;
if ( ! query.isEmpty() ) {
cursor = getOptimizedCursor(ns.c_str() , query , BSONObj() );
}
else {
// query is empty, so lets see if we can find an index
// with the key so we don't have to hit the raw data
for (int i = 0; i < cl->nIndexes(); i++) {
IndexDetails &idx = cl->idx(i);
if (cl->isMultikey(i)) {
continue;
}
if ( idx.inKeyPattern( key ) ) {
cursor = getBestGuessCursor( ns.c_str() ,
BSONObj() ,
idx.keyPattern() );
if( cursor.get() ) break;
}
}
if ( ! cursor.get() ) {
cursor = getOptimizedCursor(ns.c_str() , query , BSONObj() );
}
}
verify( cursor );
string cursorName = cursor->toString();
auto_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns));
for ( ; cursor->ok(); cursor->advance() ) {
nscanned++;
bool loadedRecord = false;
if ( cursor->currentMatches( &md ) && !cursor->getsetdup( cursor->currPK() ) ) {
n++;
BSONObj holder;
BSONElementSet temp;
loadedRecord = ! cc->getFieldsDotted( key , temp, holder );
for ( BSONElementSet::iterator i=temp.begin(); i!=temp.end(); ++i ) {
BSONElement e = *i;
if ( values.count( e ) )
continue;
int now = bb.len();
uassert(10044, "distinct too big, 16mb cap", ( now + e.size() + 1024 ) < bufSize );
arr.append( e );
BSONElement x( start + now );
values.insert( x );
}
}
if ( loadedRecord || md.hasLoadedRecord() )
nscannedObjects++;
RARELY killCurrentOp.checkForInterrupt();
}
verify( start == bb.buf() );
result.appendArray( "values" , arr.done() );
{
BSONObjBuilder b;
b.appendNumber( "n" , n );
b.appendNumber( "nscanned" , nscanned );
b.appendNumber( "nscannedObjects" , nscannedObjects );
b.appendNumber( "timems" , t.millis() );
b.append( "cursor" , cursorName );
result.append( "stats" , b.obj() );
}
return true;
}
BSONObj UpdateStage::transformAndUpdate(const Snapshotted<BSONObj>& oldObj, RecordId& recordId) {
const UpdateRequest* request = _params.request;
UpdateDriver* driver = _params.driver;
CanonicalQuery* cq = _params.canonicalQuery;
UpdateLifecycle* lifecycle = request->getLifecycle();
// If asked to return new doc, default to the oldObj, in case nothing changes.
BSONObj newObj = oldObj.value();
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new document
// is needed to accomodate the new bson layout of the resulting document. In any event,
// only enable in-place mutations if the underlying storage engine offers support for
// writing damage events.
_doc.reset(oldObj.value(),
(_collection->updateWithDamagesSupported()
? mutablebson::Document::kInPlaceEnabled
: mutablebson::Document::kInPlaceDisabled));
BSONObj logObj;
bool docWasModified = false;
Status status = Status::OK();
const bool validateForStorage = getOpCtx()->writesAreReplicated() && _enforceOkForStorage;
FieldRefSet immutablePaths;
if (getOpCtx()->writesAreReplicated() && !request->isFromMigration()) {
if (lifecycle) {
auto immutablePathsVector =
getImmutableFields(getOpCtx(), request->getNamespaceString());
if (immutablePathsVector) {
immutablePaths.fillFrom(
transitional_tools_do_not_use::unspool_vector(*immutablePathsVector));
}
}
immutablePaths.keepShortest(&idFieldRef);
}
if (!driver->needMatchDetails()) {
// If we don't need match details, avoid doing the rematch
status = driver->update(
StringData(), &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified);
} else {
// If there was a matched field, obtain it.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
dassert(cq);
verify(cq->root()->matchesBSON(oldObj.value(), &matchDetails));
string matchedField;
if (matchDetails.hasElemMatchKey())
matchedField = matchDetails.elemMatchKey();
status = driver->update(
matchedField, &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified);
}
if (!status.isOK()) {
uasserted(16837, status.reason());
}
// Skip adding _id field if the collection is capped (since capped collection documents can
// neither grow nor shrink).
const auto createIdField = !_collection->isCapped();
// Ensure if _id exists it is first
status = ensureIdFieldIsFirst(&_doc);
if (status.code() == ErrorCodes::InvalidIdField) {
// Create ObjectId _id field if we are doing that
if (createIdField) {
addObjectIDIdField(&_doc);
}
} else {
uassertStatusOK(status);
}
// See if the changes were applied in place
const char* source = NULL;
const bool inPlace = _doc.getInPlaceUpdates(&_damages, &source);
if (inPlace && _damages.empty()) {
// An interesting edge case. A modifier didn't notice that it was really a no-op
// during its 'prepare' phase. That represents a missed optimization, but we still
// shouldn't do any real work. Toggle 'docWasModified' to 'false'.
//
// Currently, an example of this is '{ $push : { x : {$each: [], $sort: 1} } }' when the 'x'
// array exists and is already sorted.
docWasModified = false;
}
if (docWasModified) {
// Prepare to write back the modified document
WriteUnitOfWork wunit(getOpCtx());
RecordId newRecordId;
OplogUpdateEntryArgs args;
if (!request->isExplain()) {
invariant(_collection);
auto* css = CollectionShardingState::get(getOpCtx(), _collection->ns());
args.nss = _collection->ns();
args.uuid = _collection->uuid();
args.stmtId = request->getStmtId();
args.update = logObj;
args.criteria = css->getMetadata().extractDocumentKey(newObj);
uassert(16980,
"Multi-update operations require all documents to have an '_id' field",
!request->isMulti() || args.criteria.hasField("_id"_sd));
args.fromMigrate = request->isFromMigration();
args.storeDocOption = getStoreDocMode(*request);
if (args.storeDocOption == OplogUpdateEntryArgs::StoreDocOption::PreImage) {
args.preImageDoc = oldObj.value().getOwned();
}
}
if (inPlace) {
if (!request->isExplain()) {
newObj = oldObj.value();
const RecordData oldRec(oldObj.value().objdata(), oldObj.value().objsize());
Snapshotted<RecordData> snap(oldObj.snapshotId(), oldRec);
StatusWith<RecordData> newRecStatus = _collection->updateDocumentWithDamages(
getOpCtx(), recordId, std::move(snap), source, _damages, &args);
newObj = uassertStatusOK(std::move(newRecStatus)).releaseToBson();
}
newRecordId = recordId;
} else {
// The updates were not in place. Apply them through the file manager.
newObj = _doc.getObject();
uassert(17419,
str::stream() << "Resulting document after update is larger than "
<< BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
if (!request->isExplain()) {
newRecordId = _collection->updateDocument(getOpCtx(),
recordId,
oldObj,
newObj,
true,
driver->modsAffectIndices(),
_params.opDebug,
&args);
}
}
invariant(oldObj.snapshotId() == getOpCtx()->recoveryUnit()->getSnapshotId());
wunit.commit();
// If the document moved, we might see it again in a collection scan (maybe it's
// a document after our current document).
//
// If the document is indexed and the mod changes an indexed value, we might see
// it again. For an example, see the comment above near declaration of
// updatedRecordIds.
//
// This must be done after the wunit commits so we are sure we won't be rolling back.
if (_updatedRecordIds && (newRecordId != recordId || driver->modsAffectIndices())) {
_updatedRecordIds->insert(newRecordId);
}
}
// Only record doc modifications if they wrote (exclude no-ops). Explains get
// recorded as if they wrote.
if (docWasModified || request->isExplain()) {
_specificStats.nModified++;
}
return newObj;
}
UpdateResult _updateObjectsNEW( bool su,
const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
RemoveSaver* rs,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy,
bool forReplication ) {
// TODO
// + Separate UpdateParser from UpdateRunner (the latter should be "stage-y")
// + All the yield and deduplicate logic would move to the query stage
// portion of it
//
// + Replication related
// + fast path for update for query by _id
// + support for relaxing viable path constraint in replication
//
// + Field Management
// + Force all upsert to contain _id
// + Prevent changes to immutable fields (_id, and those mentioned by sharding)
//
// + Yiedling related
// + $atomic support (or better, support proper yielding if not)
// + page fault support
debug.updateobj = updateobj;
NamespaceDetails* d = nsdetails( ns );
NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get( ns );
UpdateDriver::Options opts;
opts.multi = multi;
opts.upsert = upsert;
opts.logOp = logop;
UpdateDriver driver( opts );
Status status = driver.parse( nsdt->indexKeys(), updateobj );
if ( !status.isOK() ) {
uasserted( 16840, status.reason() );
}
shared_ptr<Cursor> cursor = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy );
// The 'cursor' the optimizer gave us may contain query plans that generate duplicate
// diskloc's. We set up here the mechanims that will prevent us from processing those
// twice if we see them. We also set up a 'ClientCursor' so that we can support
// yielding.
const bool dedupHere = cursor->autoDedup();
shared_ptr<Cursor> cPtr = cursor;
auto_ptr<ClientCursor> clientCursor( new ClientCursor( QueryOption_NoCursorTimeout,
cPtr,
ns ) );
//
// Upsert Logic
//
// We may or may not have documents for this update. If we don't, then try to upsert,
// if allowed.
if ( !cursor->ok() && upsert ) {
// If this is a $mod base update, we need to generate a document by examining the
// query and the mods. Otherwise, we can use the object replacement sent by the user
// update command that was parsed by the driver before.
BSONObj oldObj;
if ( *updateobj.firstElementFieldName() == '$' ) {
if ( !driver.createFromQuery( patternOrig, &oldObj ) ) {
uasserted( 16835, "cannot create object to update" );
}
debug.fastmodinsert = true;
}
else {
debug.upsert = true;
}
// Since this is an upsert, we will be oplogging it as an insert. We don't
// need the driver's help to build the oplog record, then. We also set the
// context of the update driver to an "upsert". Some mods may only work in that
// context (e.g. $setOnInsert).
driver.setLogOp( false );
driver.setContext( ModifierInterface::ExecInfo::INSERT_CONTEXT );
mutablebson::Document doc( oldObj, mutablebson::Document::kInPlaceDisabled );
status = driver.update( StringData(), &doc, NULL /* no oplog record */);
if ( !status.isOK() ) {
uasserted( 16836, status.reason() );
}
BSONObj newObj = doc.getObject();
theDataFileMgr.insertWithObjMod( ns, newObj, false, su );
if ( logop ) {
logOp( "i", ns, newObj, 0, 0, fromMigrate, &newObj );
}
return UpdateResult( false /* updated a non existing document */,
driver.dollarModMode() /* $mod or obj replacement? */,
1 /* count of updated documents */,
newObj /* object that was upserted */ );
}
//
// We have one or more documents for this update.
//
// We record that this will not be an upsert, in case a mod doesn't want to be applied
// when in strict update mode.
driver.setContext( ModifierInterface::ExecInfo::UPDATE_CONTEXT );
// Let's fetch each of them and pipe them through the update expression, making sure to
// keep track of the necessary stats. Recall that we'll be pulling documents out of
// cursors and some of them do not deduplicate the entries they generate. We have
// deduping logic in here, too -- for now.
unordered_set<DiskLoc, DiskLoc::Hasher> seenLocs;
int numUpdated = 0;
debug.nscanned = 0;
while ( cursor->ok() ) {
// Let's fetch the next candidate object for this update.
Record* r = cursor->_current();
DiskLoc loc = cursor->currLoc();
const BSONObj oldObj = loc.obj();
// We count how many documents we scanned even though we may skip those that are
// deemed duplicated. The final 'numUpdated' and 'nscanned' numbers may differ for
// that reason.
debug.nscanned++;
// Skips this document if it:
// a) doesn't match the query portion of the update
// b) was deemed duplicate by the underlying cursor machinery
//
// Now, if we are going to update the document,
// c) we don't want to do so while the cursor is at it, as that may invalidate
// the cursor. So, we advance to next document, before issuing the update.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
if ( !cursor->currentMatches( &matchDetails ) ) {
// a)
cursor->advance();
continue;
}
else if ( cursor->getsetdup( loc ) && dedupHere ) {
// b)
cursor->advance();
continue;
}
else if (driver.dollarModMode() && multi) {
// c)
cursor->advance();
if ( dedupHere ) {
if ( seenLocs.count( loc ) ) {
continue;
}
}
// There are certain kind of cursors that hold multiple pointers to data
// underneath. $or cursors is one example. In a $or cursor, it may be the case
// that when we did the last advance(), we finished consuming documents from
// one of $or child and started consuming the next one. In that case, it is
// possible that the last document of the previous child is the same as the
// first document of the next (see SERVER-5198 and jstests/orp.js).
//
// So we advance the cursor here until we see a new diskloc.
//
// Note that we won't be yielding, and we may not do so for a while if we find
// a particularly duplicated sequence of loc's. That is highly unlikely,
// though. (See SERVER-5725, if curious, but "stage" based $or will make that
// ticket moot).
while( cursor->ok() && loc == cursor->currLoc() ) {
cursor->advance();
}
}
// For some (unfortunate) historical reasons, not all cursors would be valid after
// a write simply because we advanced them to a document not affected by the write.
// To protect in those cases, not only we engaged in the advance() logic above, but
// we also tell the cursor we're about to write a document that we've just seen.
// prepareToTouchEarlierIterate() requires calling later
// recoverFromTouchingEarlierIterate(), so we make a note here to do so.
bool touchPreviousDoc = multi && cursor->ok();
if ( touchPreviousDoc ) {
clientCursor->setDoingDeletes( true );
cursor->prepareToTouchEarlierIterate();
}
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new
// document is needed to accomodate the new bson layout of the resulting document.
mutablebson::Document doc( oldObj, mutablebson::Document::kInPlaceEnabled );
BSONObj logObj;
StringData matchedField = matchDetails.hasElemMatchKey() ?
matchDetails.elemMatchKey():
StringData();
status = driver.update( matchedField, &doc, &logObj );
if ( !status.isOK() ) {
uasserted( 16837, status.reason() );
}
// If the driver applied the mods in place, we can ask the mutable for what
// changed. We call those changes "damages". :) We use the damages to inform the
// journal what was changed, and then apply them to the original document
// ourselves. If, however, the driver applied the mods out of place, we ask it to
// generate a new, modified document for us. In that case, the file manager will
// take care of the journaling details for us.
//
// This code flow is admittedly odd. But, right now, journaling is baked in the file
// manager. And if we aren't using the file manager, we have to do jounaling
// ourselves.
BSONObj newObj;
const char* source = NULL;
mutablebson::DamageVector damages;
bool inPlace = doc.getInPlaceUpdates(&damages, &source);
if ( inPlace && !driver.modsAffectIndices() ) {
// All updates were in place. Apply them via durability and writing pointer.
mutablebson::DamageVector::const_iterator where = damages.begin();
const mutablebson::DamageVector::const_iterator end = damages.end();
for( ; where != end; ++where ) {
const char* sourcePtr = source + where->sourceOffset;
void* targetPtr = getDur().writingPtr(
const_cast<char*>(oldObj.objdata()) + where->targetOffset,
where->size);
std::memcpy(targetPtr, sourcePtr, where->size);
}
newObj = oldObj;
debug.fastmod = true;
}
else {
// The updates were not in place. Apply them through the file manager.
newObj = doc.getObject();
DiskLoc newLoc = theDataFileMgr.updateRecord(ns,
d,
nsdt,
r,
loc,
newObj.objdata(),
newObj.objsize(),
debug);
// If we've moved this object to a new location, make sure we don't apply
// that update again if our traversal picks the objecta again.
//
// We also take note that the diskloc if the updates are affecting indices.
// Chances are that we're traversing one of them and they may be multi key and
// therefore duplicate disklocs.
if ( newLoc != loc || driver.modsAffectIndices() ) {
seenLocs.insert( newLoc );
}
}
// Log Obj
if ( logop ) {
if ( !logObj.isEmpty() ) {
BSONObj pattern = patternOrig;
logOp("u", ns, logObj , &pattern, 0, fromMigrate, &newObj );
}
}
// One more document updated.
numUpdated++;
if (!multi) {
break;
}
// If we used the cursor mechanism that prepares an earlier seen document for a
// write we need to tell such mechanisms that the write is over.
if ( touchPreviousDoc ) {
cursor->recoverFromTouchingEarlierIterate();
}
getDur().commitIfNeeded();
}
return UpdateResult( true /* updated existing object(s) */,
driver.dollarModMode() /* $mod or obj replacement */,
numUpdated /* # of docments update */,
BSONObj() );
}
UpdateResult _updateObjects( const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy ) {
TOKULOG(2) << "update: " << ns
<< " update: " << updateobj
<< " query: " << patternOrig
<< " upsert: " << upsert << " multi: " << multi << endl;
debug.updateobj = updateobj;
NamespaceDetails *d = getAndMaybeCreateNS(ns, logop);
auto_ptr<ModSet> mods;
const bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$';
bool modsAreIndexed = false;
if ( isOperatorUpdate ) {
if ( d->indexBuildInProgress() ) {
set<string> bgKeys;
d->inProgIdx().keyPattern().getFieldNames(bgKeys);
mods.reset( new ModSet(updateobj, d->indexKeys(), &bgKeys) );
}
else {
mods.reset( new ModSet(updateobj, d->indexKeys()) );
}
modsAreIndexed = mods->isIndexed();
}
int idIdxNo = -1;
if ( planPolicy.permitOptimalIdPlan() && !multi && !modsAreIndexed &&
(idIdxNo = d->findIdIndex()) >= 0 && mayUpdateById(d, patternOrig) ) {
debug.idhack = true;
IndexDetails &idx = d->idx(idIdxNo);
BSONObj pk = idx.getKeyFromQuery(patternOrig);
TOKULOG(3) << "_updateObjects using simple _id query, pattern " << patternOrig << ", pk " << pk << endl;
UpdateResult result = _updateById( pk,
isOperatorUpdate,
mods.get(),
d,
ns,
updateobj,
patternOrig,
logop,
debug,
fromMigrate);
if ( result.existing || ! upsert ) {
return result;
}
else if ( upsert && ! isOperatorUpdate && ! logop) {
debug.upsert = true;
BSONObj objModified = updateobj;
insertAndLog( ns, d, objModified, logop, fromMigrate );
return UpdateResult( 0 , 0 , 1 , updateobj );
}
}
int numModded = 0;
debug.nscanned = 0;
shared_ptr<Cursor> c = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy );
if( c->ok() ) {
set<BSONObj> seenObjects;
MatchDetails details;
auto_ptr<ClientCursor> cc;
do {
debug.nscanned++;
if ( mods.get() && mods->hasDynamicArray() ) {
// The Cursor must have a Matcher to record an elemMatchKey. But currently
// a modifier on a dynamic array field may be applied even if there is no
// elemMatchKey, so a matcher cannot be required.
//verify( c->matcher() );
details.requestElemMatchKey();
}
if ( !c->currentMatches( &details ) ) {
c->advance();
continue;
}
BSONObj currPK = c->currPK();
if ( c->getsetdup( currPK ) ) {
c->advance();
continue;
}
BSONObj currentObj = c->current();
BSONObj pattern = patternOrig;
if ( logop ) {
BSONObjBuilder idPattern;
BSONElement id;
// NOTE: If the matching object lacks an id, we'll log
// with the original pattern. This isn't replay-safe.
// It might make sense to suppress the log instead
// if there's no id.
if ( currentObj.getObjectID( id ) ) {
idPattern.append( id );
pattern = idPattern.obj();
}
else {
uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi );
}
}
/* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some
regular ones at the moment. */
struct LogOpUpdateDetails loud;
loud.logop = logop;
loud.ns = ns;
loud.fromMigrate = fromMigrate;
if ( isOperatorUpdate ) {
if ( multi ) {
// Make our own copies of the currPK and currentObj before we invalidate
// them by advancing the cursor.
currPK = currPK.copy();
currentObj = currentObj.copy();
// Advance past the document to be modified. This used to be because of SERVER-5198,
// but TokuMX does it because we want to avoid needing to do manual deduplication
// of this PK on the next iteration if the current update modifies the next
// entry in the index. For example, an index scan over a:1 with mod {$inc: {a:1}}
// would cause every other key read to be a duplicate if we didn't advance here.
while ( c->ok() && currPK == c->currPK() ) {
c->advance();
}
// Multi updates need to do their own deduplication because updates may modify the
// keys the cursor is in the process of scanning over.
if ( seenObjects.count( currPK ) ) {
continue;
} else {
seenObjects.insert( currPK );
}
}
ModSet* useMods = mods.get();
auto_ptr<ModSet> mymodset;
if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) {
useMods = mods->fixDynamicArray( details.elemMatchKey() );
mymodset.reset( useMods );
}
auto_ptr<ModSetState> mss = useMods->prepare( currentObj );
updateUsingMods( d, currPK, currentObj, *mss, &loud );
numModded++;
if ( ! multi )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
continue;
} // end if operator is update
uassert( 10158 , "multi update only works with $ operators" , ! multi );
updateNoMods( d, currPK, currentObj, updateobj, &loud );
return UpdateResult( 1 , 0 , 1 , BSONObj() );
} while ( c->ok() );
} // endif
if ( numModded )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( upsert ) {
BSONObj newObj = updateobj;
if ( updateobj.firstElementFieldName()[0] == '$' ) {
// upsert of an $operation. build a default object
BSONObj newObj = mods->createNewFromQuery( patternOrig );
debug.fastmodinsert = true;
insertAndLog( ns, d, newObj, logop, fromMigrate );
return UpdateResult( 0 , 1 , 1 , newObj );
}
uassert( 10159 , "multi update only works with $ operators" , ! multi );
debug.upsert = true;
insertAndLog( ns, d, newObj, logop, fromMigrate );
return UpdateResult( 0 , 0 , 1 , newObj );
}
return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() );
}
QueryResult* processGetMore(const char* ns,
int ntoreturn,
long long cursorid,
CurOp& curop,
int pass,
bool& exhaust,
bool* isCursorAuthorized ) {
bool hasRunner = false;
// Scoped to kill the pin after seeing if the runner's there.
{
// See if there's a runner. We do this until agg. is behind a Runner instead of a CC.
ClientCursorPin p(cursorid);
ClientCursor *cc = p.c();
if (NULL != cc && NULL != cc->getRunner()) {
hasRunner = true;
}
}
if (hasRunner) {
return newGetMore(ns, ntoreturn, cursorid, curop, pass, exhaust,
isCursorAuthorized);
}
exhaust = false;
int bufSize = 512 + sizeof( QueryResult ) + MaxBytesToReturnToClientAtOnce;
BufBuilder b( bufSize );
b.skip(sizeof(QueryResult));
int resultFlags = ResultFlag_AwaitCapable;
int start = 0;
int n = 0;
scoped_ptr<Client::ReadContext> ctx(new Client::ReadContext(ns));
// call this readlocked so state can't change
replVerifyReadsOk();
ClientCursorPin p(cursorid);
ClientCursor *cc = p.c();
if ( unlikely(!cc) ) {
LOGSOME << "getMore: cursorid not found " << ns << " " << cursorid << endl;
cursorid = 0;
resultFlags = ResultFlag_CursorNotFound;
}
else {
// Some internal users create a ClientCursor with a Runner. Don't crash if this
// happens. Instead, hand them off to the new framework.
if (NULL != cc->getRunner()) {
p.release();
return newGetMore(ns, ntoreturn, cursorid, curop, pass, exhaust, isCursorAuthorized);
}
// check for spoofing of the ns such that it does not match the one originally there for the cursor
uassert(14833, "auth error", str::equals(ns, cc->ns().c_str()));
*isCursorAuthorized = true;
// This must be done after auth check to ensure proper cleanup.
uassert(16951, "failing getmore due to set failpoint",
!MONGO_FAIL_POINT(getMoreError));
// If the operation that spawned this cursor had a time limit set, apply leftover
// time to this getmore.
curop.setMaxTimeMicros( cc->getLeftoverMaxTimeMicros() );
killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
if ( pass == 0 )
cc->updateSlaveLocation( curop );
int queryOptions = cc->queryOptions();
curop.debug().query = cc->query();
curop.setQuery( cc->query() );
start = cc->pos();
Cursor *c = cc->c();
if (!c->requiresLock()) {
// make sure it won't be destroyed under us
fassert(16952, !c->shouldDestroyOnNSDeletion());
fassert(16953, !c->supportYields());
ctx.reset(); // unlocks
}
c->recoverFromYield();
DiskLoc last;
// This metadata may be stale, but it's the state of chunking when the cursor was
// created.
CollectionMetadataPtr metadata = cc->getCollMetadata();
KeyPattern keyPattern( metadata ? metadata->getKeyPattern() : BSONObj() );
while ( 1 ) {
if ( !c->ok() ) {
if ( c->tailable() ) {
// when a tailable cursor hits "EOF", ok() goes false, and current() is
// null. however advance() can still be retries as a reactivation attempt.
// when there is new data, it will return true. that's what we are doing
// here.
if ( c->advance() )
continue;
if( n == 0 && (queryOptions & QueryOption_AwaitData) && pass < 1000 ) {
return 0;
}
break;
}
p.release();
bool ok = ClientCursor::erase(cursorid);
verify(ok);
cursorid = 0;
cc = 0;
break;
}
MatchDetails details;
if ( cc->fields && cc->fields->getArrayOpType() == Projection::ARRAY_OP_POSITIONAL ) {
// field projection specified, and contains an array operator
details.requestElemMatchKey();
}
// in some cases (clone collection) there won't be a matcher
if ( !c->currentMatches( &details ) ) {
}
else if ( metadata && !metadata->keyBelongsToMe( extractKey(c, keyPattern ) ) ) {
LOG(2) << "cursor skipping document in un-owned chunk: " << c->current()
<< endl;
}
else {
if( c->getsetdup(c->currLoc()) ) {
//out() << " but it's a dup \n";
}
else {
last = c->currLoc();
n++;
// Fill out the fields requested by the query.
const Projection::KeyOnly *keyFieldsOnly = c->keyFieldsOnly();
if ( keyFieldsOnly ) {
fillQueryResultFromObj( b, 0, keyFieldsOnly->hydrate(
c->currKey() ), &details );
}
else {
DiskLoc loc = c->currLoc();
fillQueryResultFromObj( b, cc->fields.get(), c->current(), &details,
( ( cc->pq.get() && cc->pq->showDiskLoc() ) ? &loc : 0 ) );
}
if ( ( ntoreturn && n >= ntoreturn ) || b.len() > MaxBytesToReturnToClientAtOnce ) {
c->advance();
cc->incPos( n );
break;
}
}
}
c->advance();
if ( ! cc->yieldSometimes( ( c->ok() && c->keyFieldsOnly() ) ?
ClientCursor::DontNeed : ClientCursor::WillNeed ) ) {
ClientCursor::erase(cursorid);
cursorid = 0;
cc = 0;
break;
}
}
if ( cc ) {
if ( c->supportYields() ) {
ClientCursor::YieldData data;
verify( cc->prepareToYield( data ) );
}
else {
cc->c()->noteLocation();
}
cc->storeOpForSlave( last );
exhaust = cc->queryOptions() & QueryOption_Exhaust;
// If the getmore had a time limit, remaining time is "rolled over" back to the
// cursor (for use by future getmore ops).
cc->setLeftoverMaxTimeMicros( curop.getRemainingMaxTimeMicros() );
}
}
QueryResult *qr = (QueryResult *) b.buf();
qr->len = b.len();
qr->setOperation(opReply);
qr->_resultFlags() = resultFlags;
qr->cursorId = cursorid;
qr->startingFrom = start;
qr->nReturned = n;
b.decouple();
return qr;
}
Status ProjectionExec::transform(const BSONObj& in,
BSONObjBuilder* bob,
const MatchDetails* details) const {
const ArrayOpType& arrayOpType = _arrayOpType;
BSONObjIterator it(in);
while (it.more()) {
BSONElement elt = it.next();
// Case 1: _id
if (mongoutils::str::equals("_id", elt.fieldName())) {
if (_includeID) {
bob->append(elt);
}
continue;
}
// Case 2: no array projection for this field.
Matchers::const_iterator matcher = _matchers.find(elt.fieldName());
if (_matchers.end() == matcher) {
Status s = append(bob, elt, details, arrayOpType);
if (!s.isOK()) {
return s;
}
continue;
}
// Case 3: field has array projection with $elemMatch specified.
if (ARRAY_OP_ELEM_MATCH != arrayOpType) {
return Status(ErrorCodes::BadValue, "Matchers are only supported for $elemMatch");
}
MatchDetails arrayDetails;
arrayDetails.requestElemMatchKey();
if (matcher->second->matchesBSON(in, &arrayDetails)) {
FieldMap::const_iterator fieldIt = _fields.find(elt.fieldName());
if (_fields.end() == fieldIt) {
return Status(ErrorCodes::BadValue,
"$elemMatch specified, but projection field not found.");
}
BSONArrayBuilder arrBuilder;
BSONObjBuilder subBob;
if (in.getField(elt.fieldName()).eoo()) {
return Status(ErrorCodes::InternalError,
"$elemMatch called on document element with eoo");
}
if (in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey()).eoo()) {
return Status(ErrorCodes::InternalError,
"$elemMatch called on array element with eoo");
}
arrBuilder.append(
in.getField(elt.fieldName()).Obj().getField(arrayDetails.elemMatchKey()));
subBob.appendArray(matcher->first, arrBuilder.arr());
Status status = append(bob, subBob.done().firstElement(), details, arrayOpType);
if (!status.isOK()) {
return status;
}
}
}
return Status::OK();
}
UpdateResult update(const UpdateRequest& request, OpDebug* opDebug, UpdateDriver* driver) {
LOG(3) << "processing update : " << request;
const NamespaceString& nsString = request.getNamespaceString();
validateUpdate( nsString.ns().c_str(), request.getUpdates(), request.getQuery() );
NamespaceDetails* nsDetails = nsdetails( nsString.ns() );
NamespaceDetailsTransient* nsDetailsTransient =
&NamespaceDetailsTransient::get( nsString.ns().c_str() );
// TODO: This seems a bit circuitious.
opDebug->updateobj = request.getUpdates();
driver->refreshIndexKeys( nsDetailsTransient->indexKeys() );
shared_ptr<Cursor> cursor = getOptimizedCursor(
nsString.ns(), request.getQuery(), BSONObj(), request.getQueryPlanSelectionPolicy() );
// If the update was marked with '$isolated' (a.k.a '$atomic'), we are not allowed to
// yield while evaluating the update loop below.
//
// TODO: Old code checks this repeatedly within the update loop. Is that necessary? It seems
// that once atomic should be always atomic.
const bool isolated =
cursor->ok() &&
cursor->matcher() &&
cursor->matcher()->docMatcher().atomic();
// The 'cursor' the optimizer gave us may contain query plans that generate duplicate
// diskloc's. We set up here the mechanims that will prevent us from processing those
// twice if we see them. We also set up a 'ClientCursor' so that we can support
// yielding.
//
// TODO: Is it valid to call this on a non-ok cursor?
const bool dedupHere = cursor->autoDedup();
//
// We'll start assuming we have one or more documents for this update. (Othwerwise,
// we'll fallback to upserting.)
//
// We record that this will not be an upsert, in case a mod doesn't want to be applied
// when in strict update mode.
driver->setContext( ModifierInterface::ExecInfo::UPDATE_CONTEXT );
// Let's fetch each of them and pipe them through the update expression, making sure to
// keep track of the necessary stats. Recall that we'll be pulling documents out of
// cursors and some of them do not deduplicate the entries they generate. We have
// deduping logic in here, too -- for now.
unordered_set<DiskLoc, DiskLoc::Hasher> seenLocs;
int numMatched = 0;
// Reset these counters on each call. We might re-enter this function to retry this
// update if we throw a page fault exception below, and we rely on these counters
// reflecting only the actions taken locally. In particlar, we must have the no-op
// counter reset so that we can meaningfully comapre it with numMatched above.
opDebug->nscanned = 0;
opDebug->nupdateNoops = 0;
Client& client = cc();
mutablebson::Document doc;
mutablebson::DamageVector damages;
// If we are going to be yielding, we will need a ClientCursor scoped to this loop. We
// only loop as long as the underlying cursor is OK.
for ( auto_ptr<ClientCursor> clientCursor; cursor->ok(); ) {
// If we haven't constructed a ClientCursor, and if the client allows us to throw
// page faults, and if we are referring to a location that is likely not in
// physical memory, then throw a PageFaultException. The entire operation will be
// restarted.
if ( clientCursor.get() == NULL &&
client.allowedToThrowPageFaultException() &&
!cursor->currLoc().isNull() &&
!cursor->currLoc().rec()->likelyInPhysicalMemory() ) {
// We should never throw a PFE if we have already updated items. The numMatched
// variable includes no-ops, which do not prevent us from raising a PFE, so if
// numMatched is non-zero, we are still OK to throw as long all matched items
// resulted in a no-op.
dassert((numMatched == 0) || (numMatched == opDebug->nupdateNoops));
throw PageFaultException( cursor->currLoc().rec() );
}
if ( !isolated && opDebug->nscanned != 0 ) {
// We are permitted to yield. To do so we need a ClientCursor, so create one
// now if we have not yet done so.
if ( !clientCursor.get() )
clientCursor.reset(
new ClientCursor( QueryOption_NoCursorTimeout, cursor, nsString.ns() ) );
// Ask the client cursor to yield. We get two bits of state back: whether or not
// we yielded, and whether or not we correctly recovered from yielding.
bool yielded = false;
const bool recovered = clientCursor->yieldSometimes(
ClientCursor::WillNeed, &yielded );
if ( !recovered ) {
// If we failed to recover from the yield, then the ClientCursor is already
// gone. Release it so we don't destroy it a second time.
clientCursor.release();
break;
}
if ( !cursor->ok() ) {
// If the cursor died while we were yielded, just get out of the update loop.
break;
}
if ( yielded ) {
// We yielded and recovered OK, and our cursor is still good. Details about
// our namespace may have changed while we were yielded, so we re-acquire
// them here. If we can't do so, escape the update loop. Otherwise, refresh
// the driver so that it knows about what is currently indexed.
nsDetails = nsdetails( nsString.ns() );
if ( !nsDetails )
break;
nsDetailsTransient = &NamespaceDetailsTransient::get( nsString.ns().c_str() );
// TODO: This copies the index keys, but it may not need to do so.
driver->refreshIndexKeys( nsDetailsTransient->indexKeys() );
}
}
// Let's fetch the next candidate object for this update.
Record* record = cursor->_current();
DiskLoc loc = cursor->currLoc();
const BSONObj oldObj = loc.obj();
// We count how many documents we scanned even though we may skip those that are
// deemed duplicated. The final 'numUpdated' and 'nscanned' numbers may differ for
// that reason.
opDebug->nscanned++;
// Skips this document if it:
// a) doesn't match the query portion of the update
// b) was deemed duplicate by the underlying cursor machinery
//
// Now, if we are going to update the document,
// c) we don't want to do so while the cursor is at it, as that may invalidate
// the cursor. So, we advance to next document, before issuing the update.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
if ( !cursor->currentMatches( &matchDetails ) ) {
// a)
cursor->advance();
continue;
}
else if ( cursor->getsetdup( loc ) && dedupHere ) {
// b)
cursor->advance();
continue;
}
else if (!driver->isDocReplacement() && request.isMulti()) {
// c)
cursor->advance();
if ( dedupHere ) {
if ( seenLocs.count( loc ) ) {
continue;
}
}
// There are certain kind of cursors that hold multiple pointers to data
// underneath. $or cursors is one example. In a $or cursor, it may be the case
// that when we did the last advance(), we finished consuming documents from
// one of $or child and started consuming the next one. In that case, it is
// possible that the last document of the previous child is the same as the
// first document of the next (see SERVER-5198 and jstests/orp.js).
//
// So we advance the cursor here until we see a new diskloc.
//
// Note that we won't be yielding, and we may not do so for a while if we find
// a particularly duplicated sequence of loc's. That is highly unlikely,
// though. (See SERVER-5725, if curious, but "stage" based $or will make that
// ticket moot).
while( cursor->ok() && loc == cursor->currLoc() ) {
cursor->advance();
}
}
// For some (unfortunate) historical reasons, not all cursors would be valid after
// a write simply because we advanced them to a document not affected by the write.
// To protect in those cases, not only we engaged in the advance() logic above, but
// we also tell the cursor we're about to write a document that we've just seen.
// prepareToTouchEarlierIterate() requires calling later
// recoverFromTouchingEarlierIterate(), so we make a note here to do so.
bool touchPreviousDoc = request.isMulti() && cursor->ok();
if ( touchPreviousDoc ) {
if ( clientCursor.get() )
clientCursor->setDoingDeletes( true );
cursor->prepareToTouchEarlierIterate();
}
// Found a matching document
numMatched++;
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new
// document is needed to accomodate the new bson layout of the resulting document.
doc.reset( oldObj, mutablebson::Document::kInPlaceEnabled );
BSONObj logObj;
// If there was a matched field, obtain it.
string matchedField;
if (matchDetails.hasElemMatchKey())
matchedField = matchDetails.elemMatchKey();
Status status = driver->update( matchedField, &doc, &logObj );
if ( !status.isOK() ) {
uasserted( 16837, status.reason() );
}
// If the driver applied the mods in place, we can ask the mutable for what
// changed. We call those changes "damages". :) We use the damages to inform the
// journal what was changed, and then apply them to the original document
// ourselves. If, however, the driver applied the mods out of place, we ask it to
// generate a new, modified document for us. In that case, the file manager will
// take care of the journaling details for us.
//
// This code flow is admittedly odd. But, right now, journaling is baked in the file
// manager. And if we aren't using the file manager, we have to do jounaling
// ourselves.
bool objectWasChanged = false;
BSONObj newObj;
const char* source = NULL;
bool inPlace = doc.getInPlaceUpdates(&damages, &source);
if ( inPlace && !driver->modsAffectIndices() ) {
// If a set of modifiers were all no-ops, we are still 'in place', but there is
// no work to do, in which case we want to consider the object unchanged.
if (!damages.empty() ) {
nsDetails->paddingFits();
// All updates were in place. Apply them via durability and writing pointer.
mutablebson::DamageVector::const_iterator where = damages.begin();
const mutablebson::DamageVector::const_iterator end = damages.end();
for( ; where != end; ++where ) {
const char* sourcePtr = source + where->sourceOffset;
void* targetPtr = getDur().writingPtr(
const_cast<char*>(oldObj.objdata()) + where->targetOffset,
where->size);
std::memcpy(targetPtr, sourcePtr, where->size);
}
objectWasChanged = true;
opDebug->fastmod = true;
}
newObj = oldObj;
}
else {
// The updates were not in place. Apply them through the file manager.
newObj = doc.getObject();
DiskLoc newLoc = theDataFileMgr.updateRecord(nsString.ns().c_str(),
nsDetails,
nsDetailsTransient,
record,
loc,
newObj.objdata(),
newObj.objsize(),
*opDebug);
// If we've moved this object to a new location, make sure we don't apply
// that update again if our traversal picks the objecta again.
//
// We also take note that the diskloc if the updates are affecting indices.
// Chances are that we're traversing one of them and they may be multi key and
// therefore duplicate disklocs.
if ( newLoc != loc || driver->modsAffectIndices() ) {
seenLocs.insert( newLoc );
}
objectWasChanged = true;
}
// Log Obj
if ( request.shouldUpdateOpLog() ) {
if ( driver->isDocReplacement() || !logObj.isEmpty() ) {
BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request.isMulti());
logOp("u", nsString.ns().c_str(), logObj , &idQuery,
NULL, request.isFromMigration(), &newObj);
}
}
// If it was noop since the document didn't change, record that.
if (!objectWasChanged)
opDebug->nupdateNoops++;
if (!request.isMulti()) {
break;
}
// If we used the cursor mechanism that prepares an earlier seen document for a
// write we need to tell such mechanisms that the write is over.
if ( touchPreviousDoc ) {
cursor->recoverFromTouchingEarlierIterate();
}
getDur().commitIfNeeded();
}
// TODO: Can this be simplified?
if ((numMatched > 0) || (numMatched == 0 && !request.isUpsert()) ) {
opDebug->nupdated = numMatched;
return UpdateResult( numMatched > 0 /* updated existing object(s) */,
!driver->isDocReplacement() /* $mod or obj replacement */,
numMatched /* # of docments update, even no-ops */,
BSONObj() );
}
//
// We haven't found any existing document so an insert is done
// (upsert is true).
//
opDebug->upsert = true;
// Since this is an insert (no docs found and upsert:true), we will be logging it
// as an insert in the oplog. We don't need the driver's help to build the
// oplog record, then. We also set the context of the update driver to the INSERT_CONTEXT.
// Some mods may only work in that context (e.g. $setOnInsert).
driver->setLogOp( false );
driver->setContext( ModifierInterface::ExecInfo::INSERT_CONTEXT );
BSONObj baseObj;
// Reset the document we will be writing to
doc.reset( baseObj, mutablebson::Document::kInPlaceDisabled );
if ( request.getQuery().hasElement("_id") ) {
uassertStatusOK(doc.root().appendElement(request.getQuery().getField("_id")));
}
// If this is a $mod base update, we need to generate a document by examining the
// query and the mods. Otherwise, we can use the object replacement sent by the user
// update command that was parsed by the driver before.
// In the following block we handle the query part, and then do the regular mods after.
if ( *request.getUpdates().firstElementFieldName() == '$' ) {
uassertStatusOK(UpdateDriver::createFromQuery(request.getQuery(), doc));
opDebug->fastmodinsert = true;
}
// Apply the update modifications and then log the update as an insert manually.
Status status = driver->update( StringData(), &doc, NULL /* no oplog record */);
if ( !status.isOK() ) {
uasserted( 16836, status.reason() );
}
BSONObj newObj = doc.getObject();
theDataFileMgr.insertWithObjMod( nsString.ns().c_str(), newObj, false, request.isGod() );
if ( request.shouldUpdateOpLog() ) {
logOp( "i", nsString.ns().c_str(), newObj,
NULL, NULL, request.isFromMigration(), &newObj );
}
opDebug->nupdated = 1;
return UpdateResult( false /* updated a non existing document */,
!driver->isDocReplacement() /* $mod or obj replacement? */,
1 /* count of updated documents */,
newObj /* object that was upserted */ );
}
Status ProjectionExec::transform(WorkingSetMember* member) const {
if (_hasReturnKey) {
BSONObj keyObj;
if (member->hasComputed(WSM_INDEX_KEY)) {
const IndexKeyComputedData* key =
static_cast<const IndexKeyComputedData*>(member->getComputed(WSM_INDEX_KEY));
keyObj = key->getKey();
}
member->obj = Snapshotted<BSONObj>(SnapshotId(), keyObj.getOwned());
member->keyData.clear();
member->loc = RecordId();
member->transitionToOwnedObj();
return Status::OK();
}
BSONObjBuilder bob;
if (member->hasObj()) {
MatchDetails matchDetails;
// If it's a positional projection we need a MatchDetails.
if (transformRequiresDetails()) {
matchDetails.requestElemMatchKey();
verify(NULL != _queryExpression);
verify(_queryExpression->matchesBSON(member->obj.value(), &matchDetails));
}
Status projStatus = transform(member->obj.value(), &bob, &matchDetails);
if (!projStatus.isOK()) {
return projStatus;
}
} else {
verify(!requiresDocument());
// Go field by field.
if (_includeID) {
BSONElement elt;
// Sometimes the _id field doesn't exist...
if (member->getFieldDotted("_id", &elt) && !elt.eoo()) {
bob.appendAs(elt, "_id");
}
}
BSONObjIterator it(_source);
while (it.more()) {
BSONElement specElt = it.next();
if (mongoutils::str::equals("_id", specElt.fieldName())) {
continue;
}
BSONElement keyElt;
// We can project a field that doesn't exist. We just ignore it.
if (member->getFieldDotted(specElt.fieldName(), &keyElt) && !keyElt.eoo()) {
bob.appendAs(keyElt, specElt.fieldName());
}
}
}
for (MetaMap::const_iterator it = _meta.begin(); it != _meta.end(); ++it) {
if (META_GEONEAR_DIST == it->second) {
if (member->hasComputed(WSM_COMPUTED_GEO_DISTANCE)) {
const GeoDistanceComputedData* dist = static_cast<const GeoDistanceComputedData*>(
member->getComputed(WSM_COMPUTED_GEO_DISTANCE));
bob.append(it->first, dist->getDist());
} else {
return Status(ErrorCodes::InternalError,
"near loc dist requested but no data available");
}
} else if (META_GEONEAR_POINT == it->second) {
if (member->hasComputed(WSM_GEO_NEAR_POINT)) {
const GeoNearPointComputedData* point =
static_cast<const GeoNearPointComputedData*>(
member->getComputed(WSM_GEO_NEAR_POINT));
BSONObj ptObj = point->getPoint();
if (ptObj.couldBeArray()) {
bob.appendArray(it->first, ptObj);
} else {
bob.append(it->first, ptObj);
}
} else {
return Status(ErrorCodes::InternalError,
"near loc proj requested but no data available");
}
} else if (META_TEXT_SCORE == it->second) {
if (member->hasComputed(WSM_COMPUTED_TEXT_SCORE)) {
const TextScoreComputedData* score = static_cast<const TextScoreComputedData*>(
member->getComputed(WSM_COMPUTED_TEXT_SCORE));
bob.append(it->first, score->getScore());
} else {
bob.append(it->first, 0.0);
}
} else if (META_RECORDID == it->second) {
bob.append(it->first, static_cast<long long>(member->loc.repr()));
}
}
BSONObj newObj = bob.obj();
member->obj = Snapshotted<BSONObj>(SnapshotId(), newObj);
member->keyData.clear();
member->loc = RecordId();
member->transitionToOwnedObj();
return Status::OK();
}
UpdateResult _updateObjects( bool su,
const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
RemoveSaver* rs,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy,
bool forReplication ) {
DEBUGUPDATE( "update: " << ns
<< " update: " << updateobj
<< " query: " << patternOrig
<< " upsert: " << upsert << " multi: " << multi );
Client& client = cc();
debug.updateobj = updateobj;
// The idea with these here it to make them loop invariant for
// multi updates, and thus be a bit faster for that case. The
// pointers may be left invalid on a failed or terminal yield
// recovery.
NamespaceDetails* d = nsdetails(ns); // can be null if an upsert...
NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get(ns);
auto_ptr<ModSet> mods;
bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$';
int modsIsIndexed = false; // really the # of indexes
if ( isOperatorUpdate ) {
mods.reset( new ModSet(updateobj, nsdt->indexKeys(), forReplication) );
modsIsIndexed = mods->maxNumIndexUpdated();
}
if( planPolicy.permitOptimalIdPlan() && !multi && isSimpleIdQuery(patternOrig) && d &&
!modsIsIndexed ) {
int idxNo = d->findIdIndex();
if( idxNo >= 0 ) {
debug.idhack = true;
UpdateResult result = _updateById( isOperatorUpdate,
idxNo,
mods.get(),
d,
nsdt,
su,
ns,
updateobj,
patternOrig,
logop,
debug,
fromMigrate);
if ( result.existing || ! upsert ) {
return result;
}
else if ( upsert && ! isOperatorUpdate ) {
// this handles repl inserts
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, false, su);
if ( logop )
logOp( "i", ns, no, 0, 0, fromMigrate, &no );
return UpdateResult( 0 , 0 , 1 , no );
}
}
}
int numModded = 0;
debug.nscanned = 0;
shared_ptr<Cursor> c = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy );
d = nsdetails(ns);
nsdt = &NamespaceDetailsTransient::get(ns);
bool autoDedup = c->autoDedup();
if( c->ok() ) {
set<DiskLoc> seenObjects;
MatchDetails details;
auto_ptr<ClientCursor> cc;
do {
if ( cc.get() == 0 &&
client.allowedToThrowPageFaultException() &&
! c->currLoc().isNull() &&
! c->currLoc().rec()->likelyInPhysicalMemory() ) {
throw PageFaultException( c->currLoc().rec() );
}
bool atomic = c->matcher() && c->matcher()->docMatcher().atomic();
if ( ! atomic && debug.nscanned > 0 ) {
// we need to use a ClientCursor to yield
if ( cc.get() == 0 ) {
shared_ptr< Cursor > cPtr = c;
cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
}
bool didYield;
if ( ! cc->yieldSometimes( ClientCursor::WillNeed, &didYield ) ) {
cc.release();
break;
}
if ( !c->ok() ) {
break;
}
if ( didYield ) {
d = nsdetails(ns);
if ( ! d )
break;
nsdt = &NamespaceDetailsTransient::get(ns);
if ( mods.get() ) {
mods->setIndexedStatus( nsdt->indexKeys() );
modsIsIndexed = mods->maxNumIndexUpdated();
}
}
} // end yielding block
debug.nscanned++;
if ( mods.get() && mods->hasDynamicArray() ) {
details.requestElemMatchKey();
}
if ( !c->currentMatches( &details ) ) {
c->advance();
continue;
}
Record* r = c->_current();
DiskLoc loc = c->currLoc();
if ( c->getsetdup( loc ) && autoDedup ) {
c->advance();
continue;
}
BSONObj js = BSONObj::make(r);
BSONObj pattern = patternOrig;
if ( logop ) {
BSONObjBuilder idPattern;
BSONElement id;
// NOTE: If the matching object lacks an id, we'll log
// with the original pattern. This isn't replay-safe.
// It might make sense to suppress the log instead
// if there's no id.
if ( js.getObjectID( id ) ) {
idPattern.append( id );
pattern = idPattern.obj();
}
else {
uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi );
}
}
/* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some
regular ones at the moment. */
if ( isOperatorUpdate ) {
if ( multi ) {
// go to next record in case this one moves
c->advance();
// Update operations are deduped for cursors that implement their own
// deduplication. In particular, some geo cursors are excluded.
if ( autoDedup ) {
if ( seenObjects.count( loc ) ) {
continue;
}
// SERVER-5198 Advance past the document to be modified, provided
// deduplication is enabled, but see SERVER-5725.
while( c->ok() && loc == c->currLoc() ) {
c->advance();
}
}
}
const BSONObj& onDisk = loc.obj();
ModSet* useMods = mods.get();
auto_ptr<ModSet> mymodset;
if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) {
useMods = mods->fixDynamicArray( details.elemMatchKey() );
mymodset.reset( useMods );
}
auto_ptr<ModSetState> mss = useMods->prepare( onDisk,
false /* not an insertion */ );
bool willAdvanceCursor = multi && c->ok() && ( modsIsIndexed || ! mss->canApplyInPlace() );
if ( willAdvanceCursor ) {
if ( cc.get() ) {
cc->setDoingDeletes( true );
}
c->prepareToTouchEarlierIterate();
}
// If we've made it this far, "ns" must contain a valid collection name, and so
// is of the form "db.collection". Therefore, the following expression must
// always be valid. "system.users" updates must never be done in place, in
// order to ensure that they are validated inside DataFileMgr::updateRecord(.).
bool isSystemUsersMod = (NamespaceString(ns).coll == "system.users");
BSONObj newObj;
if ( !mss->isUpdateIndexed() && mss->canApplyInPlace() && !isSystemUsersMod ) {
mss->applyModsInPlace( true );// const_cast<BSONObj&>(onDisk) );
DEBUGUPDATE( "\t\t\t doing in place update" );
if ( !multi )
debug.fastmod = true;
if ( modsIsIndexed ) {
seenObjects.insert( loc );
}
newObj = loc.obj();
d->paddingFits();
}
else {
newObj = mss->createNewFromMods();
checkTooLarge(newObj);
DiskLoc newLoc = theDataFileMgr.updateRecord(ns,
d,
nsdt,
r,
loc,
newObj.objdata(),
newObj.objsize(),
debug);
if ( newLoc != loc || modsIsIndexed ){
// log() << "Moved obj " << newLoc.obj()["_id"] << " from " << loc << " to " << newLoc << endl;
// object moved, need to make sure we don' get again
seenObjects.insert( newLoc );
}
}
if ( logop ) {
DEV verify( mods->size() );
BSONObj logObj = mss->getOpLogRewrite();
DEBUGUPDATE( "\t rewrite update: " << logObj );
// It is possible that the entire mod set was a no-op over this
// document. We would have an empty log record in that case. If we
// call logOp, with an empty record, that would be replicated as "clear
// this record", which is not what we want. Therefore, to get a no-op
// in the replica, we simply don't log.
if ( logObj.nFields() ) {
logOp("u", ns, logObj , &pattern, 0, fromMigrate, &newObj );
}
}
numModded++;
if ( ! multi )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( willAdvanceCursor )
c->recoverFromTouchingEarlierIterate();
getDur().commitIfNeeded();
continue;
}
uassert( 10158 , "multi update only works with $ operators" , ! multi );
BSONElementManipulator::lookForTimestamps( updateobj );
checkNoMods( updateobj );
theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, su);
if ( logop ) {
DEV wassert( !su ); // super used doesn't get logged, this would be bad.
logOp("u", ns, updateobj, &pattern, 0, fromMigrate, &updateobj );
}
return UpdateResult( 1 , 0 , 1 , BSONObj() );
} while ( c->ok() );
} // endif
if ( numModded )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( upsert ) {
if ( updateobj.firstElementFieldName()[0] == '$' ) {
// upsert of an $operation. build a default object
BSONObj newObj = mods->createNewFromQuery( patternOrig );
checkNoMods( newObj );
debug.fastmodinsert = true;
theDataFileMgr.insertWithObjMod(ns, newObj, false, su);
if ( logop )
logOp( "i", ns, newObj, 0, 0, fromMigrate, &newObj );
return UpdateResult( 0 , 1 , 1 , newObj );
}
uassert( 10159 , "multi update only works with $ operators" , ! multi );
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, false, su);
if ( logop )
logOp( "i", ns, no, 0, 0, fromMigrate, &no );
return UpdateResult( 0 , 0 , 1 , no );
}
return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() );
}
