void Helpers::upsert(OperationContext* txn, const string& ns, const BSONObj& o, bool fromMigrate) {
BSONElement e = o["_id"];
verify(e.type());
BSONObj id = e.wrap();
OldClientContext context(txn, ns);
const NamespaceString requestNs(ns);
UpdateRequest request(requestNs);
request.setQuery(id);
request.setUpdates(o);
request.setUpsert();
request.setFromMigration(fromMigrate);
UpdateLifecycleImpl updateLifecycle(requestNs);
request.setLifecycle(&updateLifecycle);
update(txn, context.db(), request);
}
void DataFileMgr::deleteRecord(NamespaceDetails* d, const StringData& ns, Record *todelete,
const DiskLoc& dl, bool cappedOK, bool noWarn, bool doLog ) {
dassert( todelete == dl.rec() );
if ( d->isCapped() && !cappedOK ) {
out() << "failing remove on a capped ns " << ns << endl;
uassert( 10089 , "can't remove from a capped collection" , 0 );
return;
}
BSONObj obj = BSONObj::make( todelete );
BSONObj toDelete;
if ( doLog ) {
BSONElement e = obj["_id"];
if ( e.type() ) {
toDelete = e.wrap();
}
}
Collection* collection = cc().database()->getCollection( ns );
verify( collection );
/* check if any cursors point to us. if so, advance them. */
ClientCursor::aboutToDelete(ns, d, dl);
collection->getIndexCatalog()->unindexRecord( obj, dl, noWarn );
_deleteRecord(d, ns, todelete, dl);
collection->infoCache()->notifyOfWriteOp();
if ( ! toDelete.isEmpty() ) {
// TODO: this is crazy, need to fix logOp
const char* raw = ns.rawData();
if ( strlen(raw) == ns.size() ) {
logOp( "d", raw, toDelete );
}
else {
string temp = ns.toString();
logOp( "d", temp.c_str(), toDelete );
}
}
}
void Helpers::upsert( const string& ns , const BSONObj& o, bool fromMigrate ) {
BSONElement e = o["_id"];
verify( e.type() );
BSONObj id = e.wrap();
OpDebug debug;
Client::Context context(ns);
const NamespaceString requestNs(ns);
UpdateRequest request(requestNs);
request.setQuery(id);
request.setUpdates(o);
request.setUpsert();
request.setUpdateOpLog();
request.setFromMigration(fromMigrate);
update(request, &debug);
}
void MigrationSourceManager::logDeleteOp(OperationContext* txn,
const char* ns,
const BSONObj& obj,
bool notInActiveChunk) {
ensureShardVersionOKOrThrow(txn, ns);
if (notInActiveChunk)
return;
dassert(txn->lockState()->isWriteLocked()); // Must have Global IX.
BSONElement ide = obj["_id"];
if (ide.eoo()) {
warning() << "logDeleteOp got mod with no _id, ignoring obj: " << obj << migrateLog;
return;
}
BSONObj idObj(ide.wrap());
txn->recoveryUnit()->registerChange(new LogOpForShardingHandler(this, idObj, 'd'));
}
// used to establish a slave for 'w' write concern
void Client::gotHandshake( const BSONObj& o ) {
BSONObjIterator i(o);
{
BSONElement id = i.next();
verify( id.type() );
_remoteId = id.wrap( "_id" );
}
BSONObjBuilder b;
while ( i.more() )
b.append( i.next() );
b.appendElementsUnique( _handshake );
_handshake = b.obj();
if (theReplSet && o.hasField("member")) {
theReplSet->registerSlave(_remoteId, o["member"].Int());
}
}
FieldParser::FieldState FieldParser::extractID(BSONElement elem,
const BSONField<BSONObj>& field,
BSONObj* out,
string* errMsg) {
if (elem.eoo()) {
if (field.hasDefault()) {
*out = field.getDefault().firstElement().wrap("");
return FIELD_DEFAULT;
} else {
return FIELD_NONE;
}
}
if (elem.type() != Array) {
*out = elem.wrap("").getOwned();
return FIELD_SET;
}
_genFieldErrMsg(elem, field, "id", errMsg);
return FIELD_INVALID;
}
bool GeoQuery::parseLegacyQuery(const BSONObj &obj) {
// The only legacy syntax is {$within: {.....}}
BSONObjIterator outerIt(obj);
if (!outerIt.more()) {
return false;
}
BSONElement withinElt = outerIt.next();
if (outerIt.more()) {
return false;
}
if (!withinElt.isABSONObj()) {
return false;
}
if (!equals(withinElt.fieldName(), "$within") && !equals(withinElt.fieldName(), "$geoWithin")) {
return false;
}
BSONObj withinObj = withinElt.embeddedObject();
bool hasGeometry = false;
BSONObjIterator withinIt(withinObj);
while (withinIt.more()) {
BSONElement elt = withinIt.next();
if (equals(elt.fieldName(), "$uniqueDocs")) {
warning() << "deprecated $uniqueDocs option: " << obj.toString() << endl;
// return false;
}
else if (elt.isABSONObj()) {
hasGeometry = geoContainer.parseFrom(elt.wrap());
}
else {
warning() << "bad geo query: " << obj.toString() << endl;
return false;
}
}
predicate = GeoQuery::WITHIN;
return hasGeometry;
}
void MigrationSourceManager::logUpdateOp(OperationContext* txn,
const char* ns,
const BSONObj& updatedDoc) {
dassert(txn->lockState()->isWriteLocked()); // Must have Global IX.
if (!_sessionId || (_nss != ns))
return;
BSONElement idElement = updatedDoc["_id"];
if (idElement.eoo()) {
warning() << "logUpdateOp got a document with no _id field, ignoring updatedDoc: "
<< updatedDoc << migrateLog;
return;
}
BSONObj idObj(idElement.wrap());
if (!isInRange(updatedDoc, _min, _max, _shardKeyPattern)) {
return;
}
txn->recoveryUnit()->registerChange(new LogOpForShardingHandler(this, idObj, 'u'));
}
void Helpers::upsert( TransactionExperiment* txn,
const string& ns,
const BSONObj& o,
bool fromMigrate ) {
BSONElement e = o["_id"];
verify( e.type() );
BSONObj id = e.wrap();
OpDebug debug;
Client::Context context(ns);
const NamespaceString requestNs(ns);
UpdateRequest request(requestNs);
request.setQuery(id);
request.setUpdates(o);
request.setUpsert();
request.setUpdateOpLog();
request.setFromMigration(fromMigrate);
UpdateLifecycleImpl updateLifecycle(true, requestNs);
request.setLifecycle(&updateLifecycle);
update(txn, context.db(), request, &debug);
}
void MigrationSourceManager::logUpdateOp(OperationContext* txn,
const char* ns,
const BSONObj& pattern,
bool notInActiveChunk) {
ensureShardVersionOKOrThrow(txn, ns);
if (notInActiveChunk)
return;
dassert(txn->lockState()->isWriteLocked()); // Must have Global IX.
if (!_active || (_nss != ns))
return;
BSONElement ide = pattern.getField("_id");
if (ide.eoo()) {
warning() << "logUpdateOp got mod with no _id, ignoring obj: " << pattern << migrateLog;
return;
}
BSONObj idObj(ide.wrap());
BSONObj fullDoc;
OldClientContext ctx(txn, _nss.ns(), false);
if (!Helpers::findById(txn, ctx.db(), _nss.ns().c_str(), idObj, fullDoc)) {
warning() << "logUpdateOp couldn't find: " << idObj << " even though should have"
<< migrateLog;
dassert(false); // TODO: Abort the migration.
return;
}
if (!isInRange(fullDoc, _min, _max, _shardKeyPattern)) {
return;
}
txn->recoveryUnit()->registerChange(new LogOpForShardingHandler(this, idObj, 'u'));
}
bool Client::gotHandshake( const BSONObj& o ) {
BSONObjIterator i(o);
{
BSONElement id = i.next();
verify( id.type() );
_remoteId = id.wrap( "_id" );
}
BSONObjBuilder b;
while ( i.more() )
b.append( i.next() );
b.appendElementsUnique( _handshake );
_handshake = b.obj();
if (repl::getGlobalReplicationCoordinator()->getReplicationMode() !=
repl::ReplicationCoordinator::modeReplSet || !o.hasField("member")) {
return false;
}
return repl::theReplSet->registerSlave(_remoteId, o["member"].Int());
}
/**
* actually applies a reduce, to a list of tuples (key, value).
* After the call, tuples will hold a single tuple {"0": key, "1": value}
*/
void JSReducer::_reduce( const BSONList& tuples , BSONObj& key , int& endSizeEstimate ) {
uassert( 10074 , "need values" , tuples.size() );
int sizeEstimate = ( tuples.size() * tuples.begin()->getField( "value" ).size() ) + 128;
// need to build the reduce args: ( key, [values] )
BSONObjBuilder reduceArgs( sizeEstimate );
boost::scoped_ptr<BSONArrayBuilder> valueBuilder;
int sizeSoFar = 0;
unsigned n = 0;
for ( ; n<tuples.size(); n++ ) {
BSONObjIterator j(tuples[n]);
BSONElement keyE = j.next();
if ( n == 0 ) {
reduceArgs.append( keyE );
key = keyE.wrap();
sizeSoFar = 5 + keyE.size();
valueBuilder.reset(new BSONArrayBuilder( reduceArgs.subarrayStart( "tuples" ) ));
}
BSONElement ee = j.next();
uassert( 13070 , "value too large to reduce" , ee.size() < ( BSONObjMaxUserSize / 2 ) );
if ( sizeSoFar + ee.size() > BSONObjMaxUserSize ) {
assert( n > 1 ); // if not, inf. loop
break;
}
valueBuilder->append( ee );
sizeSoFar += ee.size();
}
assert(valueBuilder);
valueBuilder->done();
BSONObj args = reduceArgs.obj();
Scope * s = _func.scope();
s->invokeSafe( _func.func() , args );
if ( s->type( "return" ) == Array ) {
uasserted( 10075 , "reduce -> multiple not supported yet");
return;
}
endSizeEstimate = key.objsize() + ( args.objsize() / tuples.size() );
if ( n == tuples.size() )
return;
// the input list was too large, add the rest of elmts to new tuples and reduce again
// note: would be better to use loop instead of recursion to avoid stack overflow
BSONList x;
for ( ; n < tuples.size(); n++ ) {
x.push_back( tuples[n] );
}
BSONObjBuilder temp( endSizeEstimate );
temp.append( key.firstElement() );
s->append( temp , "1" , "return" );
x.push_back( temp.obj() );
_reduce( x , key , endSizeEstimate );
}
ProjectionExec::ProjectionExec(const BSONObj& spec,
const MatchExpression* queryExpression,
const MatchExpressionParser::WhereCallback& whereCallback)
: _include(true),
_special(false),
_source(spec),
_includeID(true),
_skip(0),
_limit(-1),
_arrayOpType(ARRAY_OP_NORMAL),
_hasNonSimple(false),
_hasDottedField(false),
_queryExpression(queryExpression),
_hasReturnKey(false) {
// Are we including or excluding fields?
// -1 when we haven't initialized it.
// 1 when we're including
// 0 when we're excluding.
int include_exclude = -1;
BSONObjIterator it(_source);
while (it.more()) {
BSONElement e = it.next();
if (!e.isNumber() && !e.isBoolean()) {
_hasNonSimple = true;
}
if (Object == e.type()) {
BSONObj obj = e.embeddedObject();
verify(1 == obj.nFields());
BSONElement e2 = obj.firstElement();
if (mongoutils::str::equals(e2.fieldName(), "$slice")) {
if (e2.isNumber()) {
int i = e2.numberInt();
if (i < 0) {
add(e.fieldName(), i, -i); // limit is now positive
} else {
add(e.fieldName(), 0, i);
}
} else {
verify(e2.type() == Array);
BSONObj arr = e2.embeddedObject();
verify(2 == arr.nFields());
BSONObjIterator it(arr);
int skip = it.next().numberInt();
int limit = it.next().numberInt();
verify(limit > 0);
add(e.fieldName(), skip, limit);
}
} else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) {
_arrayOpType = ARRAY_OP_ELEM_MATCH;
// Create a MatchExpression for the elemMatch.
BSONObj elemMatchObj = e.wrap();
verify(elemMatchObj.isOwned());
_elemMatchObjs.push_back(elemMatchObj);
StatusWithMatchExpression swme =
MatchExpressionParser::parse(elemMatchObj, whereCallback);
verify(swme.isOK());
// And store it in _matchers.
_matchers[mongoutils::str::before(e.fieldName(), '.').c_str()] = swme.getValue();
add(e.fieldName(), true);
} else if (mongoutils::str::equals(e2.fieldName(), "$meta")) {
verify(String == e2.type());
if (e2.valuestr() == LiteParsedQuery::metaTextScore) {
_meta[e.fieldName()] = META_TEXT_SCORE;
} else if (e2.valuestr() == LiteParsedQuery::metaRecordId) {
_meta[e.fieldName()] = META_RECORDID;
} else if (e2.valuestr() == LiteParsedQuery::metaGeoNearPoint) {
_meta[e.fieldName()] = META_GEONEAR_POINT;
} else if (e2.valuestr() == LiteParsedQuery::metaGeoNearDistance) {
_meta[e.fieldName()] = META_GEONEAR_DIST;
} else if (e2.valuestr() == LiteParsedQuery::metaIndexKey) {
_hasReturnKey = true;
// The index key clobbers everything so just stop parsing here.
return;
} else {
// This shouldn't happen, should be caught by parsing.
verify(0);
}
} else {
verify(0);
}
} else if (mongoutils::str::equals(e.fieldName(), "_id") && !e.trueValue()) {
_includeID = false;
} else {
add(e.fieldName(), e.trueValue());
// Projections of dotted fields aren't covered.
if (mongoutils::str::contains(e.fieldName(), '.')) {
_hasDottedField = true;
}
// Validate input.
if (include_exclude == -1) {
// If we haven't specified an include/exclude, initialize include_exclude.
// We expect further include/excludes to match it.
include_exclude = e.trueValue();
_include = !e.trueValue();
}
}
if (mongoutils::str::contains(e.fieldName(), ".$")) {
_arrayOpType = ARRAY_OP_POSITIONAL;
}
}
}
WrappedObjectMatcher(BSONElement matchCondition,
const boost::intrusive_ptr<ExpressionContext>& expCtx)
: _matchExpr(matchCondition.wrap(""),
expCtx,
stdx::make_unique<ExtensionsCallbackNoop>(),
MatchExpressionParser::kBanAllSpecialFeatures) {}
void Model::save( bool safe ) {
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getScopedDbConnection (modelServer() ) );
BSONObjBuilder b;
serialize( b );
BSONElement myId;
{
BSONObjIterator i = b.iterator();
while ( i.more() ) {
BSONElement e = i.next();
if ( strcmp( e.fieldName() , "_id" ) == 0 ) {
myId = e;
break;
}
}
}
if ( myId.type() ) {
if ( _id.isEmpty() ) {
_id = myId.wrap();
}
else if ( myId.woCompare( _id.firstElement() ) ) {
stringstream ss;
ss << "_id from serialize and stored differ: ";
ss << '[' << myId << "] != ";
ss << '[' << _id.firstElement() << ']';
throw UserException( 13121 , ss.str() );
}
}
if ( _id.isEmpty() ) {
OID oid;
oid.init();
b.appendOID( "_id" , &oid );
BSONObj o = b.obj();
conn->get()->insert( getNS() , o );
_id = o["_id"].wrap().getOwned();
LOG(4) << "inserted new model " << getNS() << " " << o << endl;
}
else {
if ( myId.eoo() ) {
myId = _id["_id"];
b.append( myId );
}
verify( ! myId.eoo() );
BSONObjBuilder qb;
qb.append( myId );
BSONObj q = qb.obj();
BSONObj o = b.obj();
LOG(4) << "updated model" << getNS() << " " << q << " " << o << endl;
conn->get()->update( getNS() , q , o , true );
}
string errmsg = "";
if ( safe )
errmsg = conn->get()->getLastError();
conn->done();
if ( safe && errmsg.size() )
throw UserException( 9003 , (string)"error on Model::save: " + errmsg );
}
// static
Status ParsedProjection::make(const BSONObj& spec,
const MatchExpression* const query,
ParsedProjection** out,
const MatchExpressionParser::WhereCallback& whereCallback) {
// Are we including or excluding fields? Values:
// -1 when we haven't initialized it.
// 1 when we're including
// 0 when we're excluding.
int include_exclude = -1;
// If any of these are 'true' the projection isn't covered.
bool include = true;
bool hasNonSimple = false;
bool hasDottedField = false;
bool includeID = true;
bool hasIndexKeyProjection = false;
bool wantGeoNearPoint = false;
bool wantGeoNearDistance = false;
// Until we see a positional or elemMatch operator we're normal.
ArrayOpType arrayOpType = ARRAY_OP_NORMAL;
BSONObjIterator it(spec);
while (it.more()) {
BSONElement e = it.next();
if (!e.isNumber() && !e.isBoolean()) {
hasNonSimple = true;
}
if (Object == e.type()) {
BSONObj obj = e.embeddedObject();
if (1 != obj.nFields()) {
return Status(ErrorCodes::BadValue, ">1 field in obj: " + obj.toString());
}
BSONElement e2 = obj.firstElement();
if (mongoutils::str::equals(e2.fieldName(), "$slice")) {
if (e2.isNumber()) {
// This is A-OK.
} else if (e2.type() == Array) {
BSONObj arr = e2.embeddedObject();
if (2 != arr.nFields()) {
return Status(ErrorCodes::BadValue, "$slice array wrong size");
}
BSONObjIterator it(arr);
// Skip over 'skip'.
it.next();
int limit = it.next().numberInt();
if (limit <= 0) {
return Status(ErrorCodes::BadValue, "$slice limit must be positive");
}
} else {
return Status(ErrorCodes::BadValue,
"$slice only supports numbers and [skip, limit] arrays");
}
} else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) {
// Validate $elemMatch arguments and dependencies.
if (Object != e2.type()) {
return Status(ErrorCodes::BadValue,
"elemMatch: Invalid argument, object required.");
}
if (ARRAY_OP_POSITIONAL == arrayOpType) {
return Status(ErrorCodes::BadValue,
"Cannot specify positional operator and $elemMatch.");
}
if (mongoutils::str::contains(e.fieldName(), '.')) {
return Status(ErrorCodes::BadValue,
"Cannot use $elemMatch projection on a nested field.");
}
arrayOpType = ARRAY_OP_ELEM_MATCH;
// Create a MatchExpression for the elemMatch.
BSONObj elemMatchObj = e.wrap();
verify(elemMatchObj.isOwned());
// TODO: Is there a faster way of validating the elemMatchObj?
StatusWithMatchExpression swme =
MatchExpressionParser::parse(elemMatchObj, whereCallback);
if (!swme.isOK()) {
return swme.getStatus();
}
delete swme.getValue();
} else if (mongoutils::str::equals(e2.fieldName(), "$meta")) {
// Field for meta must be top level. We can relax this at some point.
if (mongoutils::str::contains(e.fieldName(), '.')) {
return Status(ErrorCodes::BadValue, "field for $meta cannot be nested");
}
// Make sure the argument to $meta is something we recognize.
// e.g. {x: {$meta: "textScore"}}
if (String != e2.type()) {
return Status(ErrorCodes::BadValue, "unexpected argument to $meta in proj");
}
if (e2.valuestr() != LiteParsedQuery::metaTextScore &&
e2.valuestr() != LiteParsedQuery::metaRecordId &&
e2.valuestr() != LiteParsedQuery::metaIndexKey &&
e2.valuestr() != LiteParsedQuery::metaGeoNearDistance &&
e2.valuestr() != LiteParsedQuery::metaGeoNearPoint) {
return Status(ErrorCodes::BadValue, "unsupported $meta operator: " + e2.str());
}
// This clobbers everything else.
if (e2.valuestr() == LiteParsedQuery::metaIndexKey) {
hasIndexKeyProjection = true;
} else if (e2.valuestr() == LiteParsedQuery::metaGeoNearDistance) {
wantGeoNearDistance = true;
} else if (e2.valuestr() == LiteParsedQuery::metaGeoNearPoint) {
wantGeoNearPoint = true;
}
} else {
return Status(ErrorCodes::BadValue,
string("Unsupported projection option: ") + e.toString());
}
} else if (mongoutils::str::equals(e.fieldName(), "_id") && !e.trueValue()) {
includeID = false;
} else {
// Projections of dotted fields aren't covered.
if (mongoutils::str::contains(e.fieldName(), '.')) {
hasDottedField = true;
}
// Validate input.
if (include_exclude == -1) {
// If we haven't specified an include/exclude, initialize include_exclude.
// We expect further include/excludes to match it.
include_exclude = e.trueValue();
include = !e.trueValue();
} else if (static_cast<bool>(include_exclude) != e.trueValue()) {
// Make sure that the incl./excl. matches the previous.
return Status(ErrorCodes::BadValue,
"Projection cannot have a mix of inclusion and exclusion.");
}
}
if (_isPositionalOperator(e.fieldName())) {
// Validate the positional op.
if (!e.trueValue()) {
return Status(ErrorCodes::BadValue,
"Cannot exclude array elements with the positional operator.");
}
if (ARRAY_OP_POSITIONAL == arrayOpType) {
return Status(ErrorCodes::BadValue,
"Cannot specify more than one positional proj. per query.");
}
if (ARRAY_OP_ELEM_MATCH == arrayOpType) {
return Status(ErrorCodes::BadValue,
"Cannot specify positional operator and $elemMatch.");
}
std::string after = mongoutils::str::after(e.fieldName(), ".$");
if (mongoutils::str::contains(after, ".$")) {
mongoutils::str::stream ss;
ss << "Positional projection '" << e.fieldName() << "' contains "
<< "the positional operator more than once.";
return Status(ErrorCodes::BadValue, ss);
}
std::string matchfield = mongoutils::str::before(e.fieldName(), '.');
if (!_hasPositionalOperatorMatch(query, matchfield)) {
mongoutils::str::stream ss;
ss << "Positional projection '" << e.fieldName() << "' does not "
<< "match the query document.";
return Status(ErrorCodes::BadValue, ss);
}
arrayOpType = ARRAY_OP_POSITIONAL;
}
}
// Fill out the returned obj.
unique_ptr<ParsedProjection> pp(new ParsedProjection());
// The positional operator uses the MatchDetails from the query
// expression to know which array element was matched.
pp->_requiresMatchDetails = arrayOpType == ARRAY_OP_POSITIONAL;
// Save the raw spec. It should be owned by the LiteParsedQuery.
verify(spec.isOwned());
pp->_source = spec;
pp->_returnKey = hasIndexKeyProjection;
// Dotted fields aren't covered, non-simple require match details, and as for include, "if
// we default to including then we can't use an index because we don't know what we're
// missing."
pp->_requiresDocument = include || hasNonSimple || hasDottedField;
// Add geoNear projections.
pp->_wantGeoNearPoint = wantGeoNearPoint;
pp->_wantGeoNearDistance = wantGeoNearDistance;
// If it's possible to compute the projection in a covered fashion, populate _requiredFields
// so the planner can perform projection analysis.
if (!pp->_requiresDocument) {
if (includeID) {
pp->_requiredFields.push_back("_id");
}
// The only way we could be here is if spec is only simple non-dotted-field projections.
// Therefore we can iterate over spec to get the fields required.
BSONObjIterator srcIt(spec);
while (srcIt.more()) {
BSONElement elt = srcIt.next();
// We've already handled the _id field before entering this loop.
if (includeID && mongoutils::str::equals(elt.fieldName(), "_id")) {
continue;
}
if (elt.trueValue()) {
pp->_requiredFields.push_back(elt.fieldName());
}
}
}
// returnKey clobbers everything.
if (hasIndexKeyProjection) {
pp->_requiresDocument = false;
}
*out = pp.release();
return Status::OK();
}
ProjectionExec::ProjectionExec(const BSONObj& spec,
const MatchExpression* queryExpression,
const CollatorInterface* collator,
const ExtensionsCallback& extensionsCallback)
: _include(true),
_special(false),
_source(spec),
_includeID(true),
_skip(0),
_limit(-1),
_arrayOpType(ARRAY_OP_NORMAL),
_queryExpression(queryExpression),
_hasReturnKey(false),
_collator(collator) {
// Whether we're including or excluding fields.
enum class IncludeExclude { kUninitialized, kInclude, kExclude };
IncludeExclude includeExclude = IncludeExclude::kUninitialized;
BSONObjIterator it(_source);
while (it.more()) {
BSONElement e = it.next();
if (Object == e.type()) {
BSONObj obj = e.embeddedObject();
verify(1 == obj.nFields());
BSONElement e2 = obj.firstElement();
if (mongoutils::str::equals(e2.fieldName(), "$slice")) {
if (e2.isNumber()) {
int i = e2.numberInt();
if (i < 0) {
add(e.fieldName(), i, -i); // limit is now positive
} else {
add(e.fieldName(), 0, i);
}
} else {
verify(e2.type() == Array);
BSONObj arr = e2.embeddedObject();
verify(2 == arr.nFields());
BSONObjIterator it(arr);
int skip = it.next().numberInt();
int limit = it.next().numberInt();
verify(limit > 0);
add(e.fieldName(), skip, limit);
}
} else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) {
_arrayOpType = ARRAY_OP_ELEM_MATCH;
// Create a MatchExpression for the elemMatch.
BSONObj elemMatchObj = e.wrap();
verify(elemMatchObj.isOwned());
_elemMatchObjs.push_back(elemMatchObj);
StatusWithMatchExpression statusWithMatcher =
MatchExpressionParser::parse(elemMatchObj, extensionsCallback, _collator);
verify(statusWithMatcher.isOK());
// And store it in _matchers.
_matchers[mongoutils::str::before(e.fieldName(), '.').c_str()] =
statusWithMatcher.getValue().release();
add(e.fieldName(), true);
} else if (mongoutils::str::equals(e2.fieldName(), "$meta")) {
verify(String == e2.type());
if (e2.valuestr() == QueryRequest::metaTextScore) {
_meta[e.fieldName()] = META_TEXT_SCORE;
} else if (e2.valuestr() == QueryRequest::metaSortKey) {
_sortKeyMetaFields.push_back(e.fieldName());
_meta[_sortKeyMetaFields.back()] = META_SORT_KEY;
} else if (e2.valuestr() == QueryRequest::metaRecordId) {
_meta[e.fieldName()] = META_RECORDID;
} else if (e2.valuestr() == QueryRequest::metaGeoNearPoint) {
_meta[e.fieldName()] = META_GEONEAR_POINT;
} else if (e2.valuestr() == QueryRequest::metaGeoNearDistance) {
_meta[e.fieldName()] = META_GEONEAR_DIST;
} else if (e2.valuestr() == QueryRequest::metaIndexKey) {
_hasReturnKey = true;
} else {
// This shouldn't happen, should be caught by parsing.
verify(0);
}
} else {
verify(0);
}
} else if (mongoutils::str::equals(e.fieldName(), "_id") && !e.trueValue()) {
_includeID = false;
} else {
add(e.fieldName(), e.trueValue());
// If we haven't specified an include/exclude, initialize includeExclude.
if (includeExclude == IncludeExclude::kUninitialized) {
includeExclude =
e.trueValue() ? IncludeExclude::kInclude : IncludeExclude::kExclude;
_include = !e.trueValue();
}
}
if (mongoutils::str::contains(e.fieldName(), ".$")) {
_arrayOpType = ARRAY_OP_POSITIONAL;
}
}
}
void UpdateStage::doInsert() {
_specificStats.inserted = true;
const UpdateRequest* request = _params.request;
UpdateDriver* driver = _params.driver;
CanonicalQuery* cq = _params.canonicalQuery;
UpdateLifecycle* lifecycle = request->getLifecycle();
// Since this is an insert (no docs found and upsert:true), we will be logging it
// as an insert in the oplog. We don't need the driver's help to build the
// oplog record, then. We also set the context of the update driver to the INSERT_CONTEXT.
// Some mods may only work in that context (e.g. $setOnInsert).
driver->setLogOp(false);
driver->setContext(ModifierInterface::ExecInfo::INSERT_CONTEXT);
// Reset the document we will be writing to
_doc.reset();
// The original document we compare changes to - immutable paths must not change
BSONObj original;
bool isInternalRequest = request->isFromReplication() || request->isFromMigration();
const vector<FieldRef*>* immutablePaths = NULL;
if (!isInternalRequest && lifecycle)
immutablePaths = lifecycle->getImmutableFields();
// Calling populateDocumentWithQueryFields will populate the '_doc' with fields from the
// query which creates the base of the update for the inserted doc (because upsert
// was true).
if (cq) {
uassertStatusOK(driver->populateDocumentWithQueryFields(cq, immutablePaths, _doc));
if (driver->isDocReplacement())
_specificStats.fastmodinsert = true;
original = _doc.getObject();
}
else {
fassert(17354, CanonicalQuery::isSimpleIdQuery(request->getQuery()));
BSONElement idElt = request->getQuery()[idFieldName];
original = idElt.wrap();
fassert(17352, _doc.root().appendElement(idElt));
}
// Apply the update modifications and then log the update as an insert manually.
Status status = driver->update(StringData(), &_doc);
if (!status.isOK()) {
uasserted(16836, status.reason());
}
// Ensure _id exists and is first
uassertStatusOK(ensureIdAndFirst(_doc));
// Validate that the object replacement or modifiers resulted in a document
// that contains all the immutable keys and can be stored if it isn't coming
// from a migration or via replication.
if (!isInternalRequest){
FieldRefSet noFields;
// This will only validate the modified fields if not a replacement.
uassertStatusOK(validate(original,
noFields,
_doc,
immutablePaths,
driver->modOptions()) );
}
// Insert the doc
BSONObj newObj = _doc.getObject();
uassert(17420,
str::stream() << "Document to upsert is larger than " << BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
_specificStats.objInserted = newObj;
// If this is an explain, bail out now without doing the insert.
if (request->isExplain()) {
return;
}
WriteUnitOfWork wunit(request->getOpCtx());
invariant(_collection);
StatusWith<DiskLoc> newLoc = _collection->insertDocument(request->getOpCtx(),
newObj,
!request->isGod()/*enforceQuota*/);
uassertStatusOK(newLoc.getStatus());
if (request->shouldCallLogOp()) {
repl::logOp(request->getOpCtx(),
"i",
request->getNamespaceString().ns().c_str(),
newObj,
NULL,
NULL,
request->isFromMigration());
}
wunit.commit();
}
void Projection::init( const BSONObj& o ) {
massert( 10371 , "can only add to Projection once", _source.isEmpty());
_source = o;
BSONObjIterator i( o );
int true_false = -1;
while ( i.more() ) {
BSONElement e = i.next();
if ( ! e.isNumber() )
_hasNonSimple = true;
if (e.type() == Object) {
BSONObj obj = e.embeddedObject();
BSONElement e2 = obj.firstElement();
if ( mongoutils::str::equals( e2.fieldName(), "$slice" ) ) {
if (e2.isNumber()) {
int i = e2.numberInt();
if (i < 0)
add(e.fieldName(), i, -i); // limit is now positive
else
add(e.fieldName(), 0, i);
}
else if (e2.type() == Array) {
BSONObj arr = e2.embeddedObject();
uassert(13099, "$slice array wrong size", arr.nFields() == 2 );
BSONObjIterator it(arr);
int skip = it.next().numberInt();
int limit = it.next().numberInt();
uassert(13100, "$slice limit must be positive", limit > 0 );
add(e.fieldName(), skip, limit);
}
else {
uassert(13098, "$slice only supports numbers and [skip, limit] arrays", false);
}
}
else if ( mongoutils::str::equals( e2.fieldName(), "$elemMatch" ) ) {
// validate $elemMatch arguments and dependencies
uassert( 16342, "elemMatch: invalid argument. object required.",
e2.type() == Object );
uassert( 16343, "Cannot specify positional operator and $elemMatch"
" (currently unsupported).",
_arrayOpType != ARRAY_OP_POSITIONAL );
uassert( 16344, "Cannot use $elemMatch projection on a nested field"
" (currently unsupported).",
! mongoutils::str::contains( e.fieldName(), '.' ) );
_arrayOpType = ARRAY_OP_ELEM_MATCH;
// initialize new Matcher object(s)
_matchers.insert( make_pair( mongoutils::str::before( e.fieldName(), '.' ),
boost::make_shared<Matcher>( e.wrap(), true ) ) );
add( e.fieldName(), true );
}
else {
uasserted(13097, string("Unsupported projection option: ") +
obj.firstElementFieldName() );
}
}
else if (!strcmp(e.fieldName(), "_id") && !e.trueValue()) {
_includeID = false;
}
else {
add( e.fieldName(), e.trueValue() );
// validate input
if (true_false == -1) {
true_false = e.trueValue();
_include = !e.trueValue();
}
else {
uassert( 10053 , "You cannot currently mix including and excluding fields. "
"Contact us if this is an issue." ,
(bool)true_false == e.trueValue() );
}
}
if ( mongoutils::str::contains( e.fieldName(), ".$" ) ) {
// positional op found; verify dependencies
uassert( 16345, "Cannot exclude array elements with the positional operator"
" (currently unsupported).", e.trueValue() );
uassert( 16346, "Cannot specify more than one positional array element per query"
" (currently unsupported).", _arrayOpType != ARRAY_OP_POSITIONAL );
uassert( 16347, "Cannot specify positional operator and $elemMatch"
" (currently unsupported).", _arrayOpType != ARRAY_OP_ELEM_MATCH );
_arrayOpType = ARRAY_OP_POSITIONAL;
}
}
}
Status ShardingCatalogClientImpl::insertConfigDocument(OperationContext* opCtx,
const NamespaceString& nss,
const BSONObj& doc,
const WriteConcernOptions& writeConcern) {
invariant(nss.db() == NamespaceString::kAdminDb || nss.db() == NamespaceString::kConfigDb);
const BSONElement idField = doc.getField("_id");
invariant(!idField.eoo());
BatchedCommandRequest request([&] {
write_ops::Insert insertOp(nss);
insertOp.setDocuments({doc});
return insertOp;
}());
request.setWriteConcern(writeConcern.toBSON());
auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
for (int retry = 1; retry <= kMaxWriteRetry; retry++) {
auto response = configShard->runBatchWriteCommand(
opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kNoRetry);
Status status = response.toStatus();
if (retry < kMaxWriteRetry &&
configShard->isRetriableError(status.code(), Shard::RetryPolicy::kIdempotent)) {
// Pretend like the operation is idempotent because we're handling DuplicateKey errors
// specially
continue;
}
// If we get DuplicateKey error on the first attempt to insert, this definitively means that
// we are trying to insert the same entry a second time, so error out. If it happens on a
// retry attempt though, it is not clear whether we are actually inserting a duplicate key
// or it is because we failed to wait for write concern on the first attempt. In order to
// differentiate, fetch the entry and check.
if (retry > 1 && status == ErrorCodes::DuplicateKey) {
LOG(1) << "Insert retry failed because of duplicate key error, rechecking.";
auto fetchDuplicate =
_exhaustiveFindOnConfig(opCtx,
ReadPreferenceSetting{ReadPreference::PrimaryOnly},
repl::ReadConcernLevel::kMajorityReadConcern,
nss,
idField.wrap(),
BSONObj(),
boost::none);
if (!fetchDuplicate.isOK()) {
return fetchDuplicate.getStatus();
}
auto existingDocs = fetchDuplicate.getValue().value;
if (existingDocs.empty()) {
return {status.withContext(
stream() << "DuplicateKey error was returned after a retry attempt, but no "
"documents were found. This means a concurrent change occurred "
"together with the retries.")};
}
invariant(existingDocs.size() == 1);
BSONObj existing = std::move(existingDocs.front());
if (existing.woCompare(doc) == 0) {
// Documents match, so treat the operation as success
return Status::OK();
}
}
return status;
}
MONGO_UNREACHABLE;
}
void MigrationSourceManager::logOp(OperationContext* txn,
const char* opstr,
const char* ns,
const BSONObj& obj,
BSONObj* patt,
bool notInActiveChunk) {
ensureShardVersionOKOrThrow(txn->getClient(), ns);
const char op = opstr[0];
if (notInActiveChunk) {
// Ignore writes that came from the migration process like cleanup so they
// won't be transferred to the recipient shard. Also ignore ops from
// _migrateClone and _transferMods since it is impossible to move a chunk
// to self.
return;
}
dassert(txn->lockState()->isWriteLocked()); // Must have Global IX.
if (!_active)
return;
if (_ns != ns)
return;
// no need to log if this is not an insertion, an update, or an actual deletion
// note: opstr 'db' isn't a deletion but a mention that a database exists
// (for replication machinery mostly).
if (op == 'n' || op == 'c' || (op == 'd' && opstr[1] == 'b'))
return;
BSONElement ide;
if (patt)
ide = patt->getField("_id");
else
ide = obj["_id"];
if (ide.eoo()) {
warning() << "logOpForSharding got mod with no _id, ignoring obj: " << obj << migrateLog;
return;
}
if (op == 'i' && (!isInRange(obj, _min, _max, _shardKeyPattern))) {
return;
}
BSONObj idObj(ide.wrap());
if (op == 'u') {
BSONObj fullDoc;
OldClientContext ctx(txn, _ns, false);
if (!Helpers::findById(txn, ctx.db(), _ns.c_str(), idObj, fullDoc)) {
warning() << "logOpForSharding couldn't find: " << idObj << " even though should have"
<< migrateLog;
dassert(false); // TODO: Abort the migration.
return;
}
if (!isInRange(fullDoc, _min, _max, _shardKeyPattern)) {
return;
}
}
// Note: can't check if delete is in active chunk since the document is gone!
txn->recoveryUnit()->registerChange(new LogOpForShardingHandler(this, idObj, op));
}
Status LiteParsedQuery::initFullQuery(const BSONObj& top) {
BSONObjIterator i(top);
while (i.more()) {
BSONElement e = i.next();
const char* name = e.fieldName();
if (0 == strcmp("$orderby", name) || 0 == strcmp("orderby", name)) {
if (Object == e.type()) {
_sort = e.embeddedObject();
}
else if (Array == e.type()) {
_sort = e.embeddedObject();
// TODO: Is this ever used? I don't think so.
// Quote:
// This is for languages whose "objects" are not well ordered (JSON is well
// ordered).
// [ { a : ... } , { b : ... } ] -> { a : ..., b : ... }
// note: this is slow, but that is ok as order will have very few pieces
BSONObjBuilder b;
char p[2] = "0";
while (1) {
BSONObj j = _sort.getObjectField(p);
if (j.isEmpty()) { break; }
BSONElement e = j.firstElement();
if (e.eoo()) {
return Status(ErrorCodes::BadValue, "bad order array");
}
if (!e.isNumber()) {
return Status(ErrorCodes::BadValue, "bad order array [2]");
}
b.append(e);
(*p)++;
if (!(*p <= '9')) {
return Status(ErrorCodes::BadValue, "too many ordering elements");
}
}
_sort = b.obj();
}
else {
return Status(ErrorCodes::BadValue, "sort must be object or array");
}
}
else if ('$' == *name) {
name++;
if (str::equals("explain", name)) {
// Won't throw.
_explain = e.trueValue();
}
else if (str::equals("snapshot", name)) {
// Won't throw.
_snapshot = e.trueValue();
}
else if (str::equals("min", name)) {
if (!e.isABSONObj()) {
return Status(ErrorCodes::BadValue, "$min must be a BSONObj");
}
_min = e.embeddedObject();
}
else if (str::equals("max", name)) {
if (!e.isABSONObj()) {
return Status(ErrorCodes::BadValue, "$max must be a BSONObj");
}
_max = e.embeddedObject();
}
else if (str::equals("hint", name)) {
if (e.isABSONObj()) {
_hint = e.embeddedObject();
}
else {
// Hint can be specified as an object or as a string. Wrap takes care of
// it.
_hint = e.wrap();
}
}
else if (str::equals("returnKey", name)) {
// Won't throw.
_returnKey = e.trueValue();
}
else if (str::equals("maxScan", name)) {
// Won't throw.
_maxScan = e.numberInt();
}
else if (str::equals("showDiskLoc", name)) {
// Won't throw.
_showDiskLoc = e.trueValue();
}
else if (str::equals("maxTimeMS", name)) {
StatusWith<int> maxTimeMS = parseMaxTimeMS(e);
if (!maxTimeMS.isOK()) {
return maxTimeMS.getStatus();
}
_maxTimeMS = maxTimeMS.getValue();
}
}
}
if (_snapshot) {
if (!_sort.isEmpty()) {
return Status(ErrorCodes::BadValue, "E12001 can't use sort with $snapshot");
}
if (!_hint.isEmpty()) {
return Status(ErrorCodes::BadValue, "E12002 can't use hint with $snapshot");
}
}
return Status::OK();
}
UpdateResult update(
OperationContext* txn,
Database* db,
const UpdateRequest& request,
OpDebug* opDebug,
UpdateDriver* driver,
CanonicalQuery* cq) {
LOG(3) << "processing update : " << request;
std::auto_ptr<CanonicalQuery> cqHolder(cq);
const NamespaceString& nsString = request.getNamespaceString();
UpdateLifecycle* lifecycle = request.getLifecycle();
Collection* collection = db->getCollection(txn, nsString.ns());
validateUpdate(nsString.ns().c_str(), request.getUpdates(), request.getQuery());
// TODO: This seems a bit circuitious.
opDebug->updateobj = request.getUpdates();
if (lifecycle) {
lifecycle->setCollection(collection);
driver->refreshIndexKeys(lifecycle->getIndexKeys());
}
Runner* rawRunner;
Status status = cq ?
getRunner(collection, cqHolder.release(), &rawRunner) :
getRunner(collection, nsString.ns(), request.getQuery(), &rawRunner, &cq);
uassert(17243,
"could not get runner " + request.getQuery().toString() + "; " + causedBy(status),
status.isOK());
// Create the runner and setup all deps.
auto_ptr<Runner> runner(rawRunner);
// Register Runner with ClientCursor
const ScopedRunnerRegistration safety(runner.get());
//
// We'll start assuming we have one or more documents for this update. (Otherwise,
// we'll fall-back to insert case (if upsert is true).)
//
// We are an update until we fall into the insert case below.
driver->setContext(ModifierInterface::ExecInfo::UPDATE_CONTEXT);
int numMatched = 0;
// If the update was in-place, we may see it again. This only matters if we're doing
// a multi-update; if we're not doing a multi-update we stop after one update and we
// won't see any more docs.
//
// For example: If we're scanning an index {x:1} and performing {$inc:{x:5}}, we'll keep
// moving the document forward and it will continue to reappear in our index scan.
// Unless the index is multikey, the underlying query machinery won't de-dup.
//
// If the update wasn't in-place we may see it again. Our query may return the new
// document and we wouldn't want to update that.
//
// So, no matter what, we keep track of where the doc wound up.
typedef unordered_set<DiskLoc, DiskLoc::Hasher> DiskLocSet;
const scoped_ptr<DiskLocSet> updatedLocs(request.isMulti() ? new DiskLocSet : NULL);
// Reset these counters on each call. We might re-enter this function to retry this
// update if we throw a page fault exception below, and we rely on these counters
// reflecting only the actions taken locally. In particlar, we must have the no-op
// counter reset so that we can meaningfully comapre it with numMatched above.
opDebug->nscanned = 0;
opDebug->nscannedObjects = 0;
opDebug->nModified = 0;
// Get the cached document from the update driver.
mutablebson::Document& doc = driver->getDocument();
mutablebson::DamageVector damages;
// Used during iteration of docs
BSONObj oldObj;
// Get first doc, and location
Runner::RunnerState state = Runner::RUNNER_ADVANCED;
uassert(ErrorCodes::NotMaster,
mongoutils::str::stream() << "Not primary while updating " << nsString.ns(),
!request.shouldCallLogOp()
|| repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(
nsString.db()));
while (true) {
// Get next doc, and location
DiskLoc loc;
state = runner->getNext(&oldObj, &loc);
if (state != Runner::RUNNER_ADVANCED) {
if (state == Runner::RUNNER_EOF) {
// We have reached the logical end of the loop, so do yielding recovery
break;
}
else {
uassertStatusOK(Status(ErrorCodes::InternalError,
str::stream() << " Update query failed -- "
<< Runner::statestr(state)));
}
}
// We fill this with the new locs of moved doc so we don't double-update.
if (updatedLocs && updatedLocs->count(loc) > 0) {
continue;
}
// We count how many documents we scanned even though we may skip those that are
// deemed duplicated. The final 'numMatched' and 'nscanned' numbers may differ for
// that reason.
// TODO: Do we want to pull this out of the underlying query plan?
opDebug->nscanned++;
// Found a matching document
opDebug->nscannedObjects++;
numMatched++;
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new
// document is needed to accomodate the new bson layout of the resulting document.
doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled);
BSONObj logObj;
FieldRefSet updatedFields;
Status status = Status::OK();
if (!driver->needMatchDetails()) {
// If we don't need match details, avoid doing the rematch
status = driver->update(StringData(), &doc, &logObj, &updatedFields);
}
else {
// If there was a matched field, obtain it.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
dassert(cq);
verify(cq->root()->matchesBSON(oldObj, &matchDetails));
string matchedField;
if (matchDetails.hasElemMatchKey())
matchedField = matchDetails.elemMatchKey();
// TODO: Right now, each mod checks in 'prepare' that if it needs positional
// data, that a non-empty StringData() was provided. In principle, we could do
// that check here in an else clause to the above conditional and remove the
// checks from the mods.
status = driver->update(matchedField, &doc, &logObj, &updatedFields);
}
if (!status.isOK()) {
uasserted(16837, status.reason());
}
// Ensure _id exists and is first
uassertStatusOK(ensureIdAndFirst(doc));
// If the driver applied the mods in place, we can ask the mutable for what
// changed. We call those changes "damages". :) We use the damages to inform the
// journal what was changed, and then apply them to the original document
// ourselves. If, however, the driver applied the mods out of place, we ask it to
// generate a new, modified document for us. In that case, the file manager will
// take care of the journaling details for us.
//
// This code flow is admittedly odd. But, right now, journaling is baked in the file
// manager. And if we aren't using the file manager, we have to do jounaling
// ourselves.
bool docWasModified = false;
BSONObj newObj;
const char* source = NULL;
bool inPlace = doc.getInPlaceUpdates(&damages, &source);
// If something changed in the document, verify that no immutable fields were changed
// and data is valid for storage.
if ((!inPlace || !damages.empty()) ) {
if (!(request.isFromReplication() || request.isFromMigration())) {
const std::vector<FieldRef*>* immutableFields = NULL;
if (lifecycle)
immutableFields = lifecycle->getImmutableFields();
uassertStatusOK(validate(oldObj,
updatedFields,
doc,
immutableFields,
driver->modOptions()) );
}
}
// Save state before making changes
runner->saveState();
if (inPlace && !driver->modsAffectIndices()) {
// If a set of modifiers were all no-ops, we are still 'in place', but there is
// no work to do, in which case we want to consider the object unchanged.
if (!damages.empty() ) {
collection->updateDocumentWithDamages( txn, loc, source, damages );
docWasModified = true;
opDebug->fastmod = true;
}
newObj = oldObj;
}
else {
// The updates were not in place. Apply them through the file manager.
// XXX: With experimental document-level locking, we do not hold the sufficient
// locks, so this would cause corruption.
fassert(18516, !useExperimentalDocLocking);
newObj = doc.getObject();
uassert(17419,
str::stream() << "Resulting document after update is larger than "
<< BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
StatusWith<DiskLoc> res = collection->updateDocument(txn,
loc,
newObj,
true,
opDebug);
uassertStatusOK(res.getStatus());
DiskLoc newLoc = res.getValue();
docWasModified = true;
// If the document moved, we might see it again in a collection scan (maybe it's
// a document after our current document).
//
// If the document is indexed and the mod changes an indexed value, we might see it
// again. For an example, see the comment above near declaration of updatedLocs.
if (updatedLocs && (newLoc != loc || driver->modsAffectIndices())) {
updatedLocs->insert(newLoc);
}
}
// Restore state after modification
uassert(17278,
"Update could not restore runner state after updating a document.",
runner->restoreState(txn));
// Call logOp if requested.
if (request.shouldCallLogOp() && !logObj.isEmpty()) {
BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request.isMulti());
repl::logOp(txn, "u", nsString.ns().c_str(), logObj , &idQuery,
NULL, request.isFromMigration());
}
// Only record doc modifications if they wrote (exclude no-ops)
if (docWasModified)
opDebug->nModified++;
if (!request.isMulti()) {
break;
}
// Opportunity for journaling to write during the update.
txn->recoveryUnit()->commitIfNeeded();
}
// TODO: Can this be simplified?
if ((numMatched > 0) || (numMatched == 0 && !request.isUpsert()) ) {
opDebug->nMatched = numMatched;
return UpdateResult(numMatched > 0 /* updated existing object(s) */,
!driver->isDocReplacement() /* $mod or obj replacement */,
opDebug->nModified /* number of modified docs, no no-ops */,
numMatched /* # of docs matched/updated, even no-ops */,
BSONObj());
}
//
// We haven't found any existing document so an insert is done
// (upsert is true).
//
opDebug->upsert = true;
// Since this is an insert (no docs found and upsert:true), we will be logging it
// as an insert in the oplog. We don't need the driver's help to build the
// oplog record, then. We also set the context of the update driver to the INSERT_CONTEXT.
// Some mods may only work in that context (e.g. $setOnInsert).
driver->setLogOp(false);
driver->setContext(ModifierInterface::ExecInfo::INSERT_CONTEXT);
// Reset the document we will be writing to
doc.reset();
// This remains the empty object in the case of an object replacement, but in the case
// of an upsert where we are creating a base object from the query and applying mods,
// we capture the query as the original so that we can detect immutable field mutations.
BSONObj original = BSONObj();
// Calling createFromQuery will populate the 'doc' with fields from the query which
// creates the base of the update for the inserterd doc (because upsert was true)
if (cq) {
uassertStatusOK(driver->populateDocumentWithQueryFields(cq, doc));
// Validate the base doc, as taken from the query -- no fields means validate all.
FieldRefSet noFields;
uassertStatusOK(validate(BSONObj(), noFields, doc, NULL, driver->modOptions()));
if (!driver->isDocReplacement()) {
opDebug->fastmodinsert = true;
// We need all the fields from the query to compare against for validation below.
original = doc.getObject();
}
else {
original = request.getQuery();
}
}
else {
fassert(17354, CanonicalQuery::isSimpleIdQuery(request.getQuery()));
BSONElement idElt = request.getQuery()["_id"];
original = idElt.wrap();
fassert(17352, doc.root().appendElement(idElt));
}
// Apply the update modifications and then log the update as an insert manually.
FieldRefSet updatedFields;
status = driver->update(StringData(), &doc, NULL, &updatedFields);
if (!status.isOK()) {
uasserted(16836, status.reason());
}
// Ensure _id exists and is first
uassertStatusOK(ensureIdAndFirst(doc));
// Validate that the object replacement or modifiers resulted in a document
// that contains all the immutable keys and can be stored.
if (!(request.isFromReplication() || request.isFromMigration())){
const std::vector<FieldRef*>* immutableFields = NULL;
if (lifecycle)
immutableFields = lifecycle->getImmutableFields();
// This will only validate the modified fields if not a replacement.
uassertStatusOK(validate(original,
updatedFields,
doc,
immutableFields,
driver->modOptions()) );
}
// Only create the collection if the doc will be inserted.
if (!collection) {
collection = db->getCollection(txn, request.getNamespaceString().ns());
if (!collection) {
collection = db->createCollection(txn, request.getNamespaceString().ns());
}
}
// Insert the doc
BSONObj newObj = doc.getObject();
uassert(17420,
str::stream() << "Document to upsert is larger than " << BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
StatusWith<DiskLoc> newLoc = collection->insertDocument(txn,
newObj,
!request.isGod() /*enforceQuota*/);
uassertStatusOK(newLoc.getStatus());
if (request.shouldCallLogOp()) {
repl::logOp(txn, "i", nsString.ns().c_str(), newObj,
NULL, NULL, request.isFromMigration());
}
opDebug->nMatched = 1;
return UpdateResult(false /* updated a non existing document */,
!driver->isDocReplacement() /* $mod or obj replacement? */,
1 /* docs written*/,
1 /* count of updated documents */,
newObj /* object that was upserted */ );
}
// static
Status ParsedProjection::make(const BSONObj& spec,
const MatchExpression* const query,
ParsedProjection** out,
const ExtensionsCallback& extensionsCallback) {
// Whether we're including or excluding fields.
enum class IncludeExclude { kUninitialized, kInclude, kExclude };
IncludeExclude includeExclude = IncludeExclude::kUninitialized;
bool requiresDocument = false;
bool includeID = true;
bool hasIndexKeyProjection = false;
bool wantGeoNearPoint = false;
bool wantGeoNearDistance = false;
bool wantSortKey = false;
// Until we see a positional or elemMatch operator we're normal.
ArrayOpType arrayOpType = ARRAY_OP_NORMAL;
BSONObjIterator it(spec);
while (it.more()) {
BSONElement e = it.next();
if (Object == e.type()) {
BSONObj obj = e.embeddedObject();
if (1 != obj.nFields()) {
return Status(ErrorCodes::BadValue, ">1 field in obj: " + obj.toString());
}
BSONElement e2 = obj.firstElement();
if (mongoutils::str::equals(e2.fieldName(), "$slice")) {
if (e2.isNumber()) {
// This is A-OK.
} else if (e2.type() == Array) {
BSONObj arr = e2.embeddedObject();
if (2 != arr.nFields()) {
return Status(ErrorCodes::BadValue, "$slice array wrong size");
}
BSONObjIterator it(arr);
// Skip over 'skip'.
it.next();
int limit = it.next().numberInt();
if (limit <= 0) {
return Status(ErrorCodes::BadValue, "$slice limit must be positive");
}
} else {
return Status(ErrorCodes::BadValue,
"$slice only supports numbers and [skip, limit] arrays");
}
// Projections with $slice aren't covered.
requiresDocument = true;
} else if (mongoutils::str::equals(e2.fieldName(), "$elemMatch")) {
// Validate $elemMatch arguments and dependencies.
if (Object != e2.type()) {
return Status(ErrorCodes::BadValue,
"elemMatch: Invalid argument, object required.");
}
if (ARRAY_OP_POSITIONAL == arrayOpType) {
return Status(ErrorCodes::BadValue,
"Cannot specify positional operator and $elemMatch.");
}
if (mongoutils::str::contains(e.fieldName(), '.')) {
return Status(ErrorCodes::BadValue,
"Cannot use $elemMatch projection on a nested field.");
}
arrayOpType = ARRAY_OP_ELEM_MATCH;
// Create a MatchExpression for the elemMatch.
BSONObj elemMatchObj = e.wrap();
verify(elemMatchObj.isOwned());
// TODO: Is there a faster way of validating the elemMatchObj?
StatusWithMatchExpression statusWithMatcher =
MatchExpressionParser::parse(elemMatchObj, extensionsCallback);
if (!statusWithMatcher.isOK()) {
return statusWithMatcher.getStatus();
}
// Projections with $elemMatch aren't covered.
requiresDocument = true;
} else if (mongoutils::str::equals(e2.fieldName(), "$meta")) {
// Field for meta must be top level. We can relax this at some point.
if (mongoutils::str::contains(e.fieldName(), '.')) {
return Status(ErrorCodes::BadValue, "field for $meta cannot be nested");
}
// Make sure the argument to $meta is something we recognize.
// e.g. {x: {$meta: "textScore"}}
if (String != e2.type()) {
return Status(ErrorCodes::BadValue, "unexpected argument to $meta in proj");
}
if (e2.valuestr() != LiteParsedQuery::metaTextScore &&
e2.valuestr() != LiteParsedQuery::metaRecordId &&
e2.valuestr() != LiteParsedQuery::metaIndexKey &&
e2.valuestr() != LiteParsedQuery::metaGeoNearDistance &&
e2.valuestr() != LiteParsedQuery::metaGeoNearPoint &&
e2.valuestr() != LiteParsedQuery::metaSortKey) {
return Status(ErrorCodes::BadValue, "unsupported $meta operator: " + e2.str());
}
// This clobbers everything else.
if (e2.valuestr() == LiteParsedQuery::metaIndexKey) {
hasIndexKeyProjection = true;
} else if (e2.valuestr() == LiteParsedQuery::metaGeoNearDistance) {
wantGeoNearDistance = true;
} else if (e2.valuestr() == LiteParsedQuery::metaGeoNearPoint) {
wantGeoNearPoint = true;
} else if (e2.valuestr() == LiteParsedQuery::metaSortKey) {
wantSortKey = true;
}
// Of the $meta projections, only sortKey can be covered.
if (e2.valuestr() != LiteParsedQuery::metaSortKey) {
requiresDocument = true;
}
} else {
return Status(ErrorCodes::BadValue,
string("Unsupported projection option: ") + e.toString());
}
} else if (mongoutils::str::equals(e.fieldName(), "_id") && !e.trueValue()) {
includeID = false;
} else {
// Projections of dotted fields aren't covered.
if (mongoutils::str::contains(e.fieldName(), '.')) {
requiresDocument = true;
}
// If we haven't specified an include/exclude, initialize includeExclude. We expect
// further include/excludes to match it.
if (includeExclude == IncludeExclude::kUninitialized) {
includeExclude =
e.trueValue() ? IncludeExclude::kInclude : IncludeExclude::kExclude;
} else if ((includeExclude == IncludeExclude::kInclude && !e.trueValue()) ||
(includeExclude == IncludeExclude::kExclude && e.trueValue())) {
return Status(ErrorCodes::BadValue,
"Projection cannot have a mix of inclusion and exclusion.");
}
}
if (_isPositionalOperator(e.fieldName())) {
// Validate the positional op.
if (!e.trueValue()) {
return Status(ErrorCodes::BadValue,
"Cannot exclude array elements with the positional operator.");
}
if (ARRAY_OP_POSITIONAL == arrayOpType) {
return Status(ErrorCodes::BadValue,
"Cannot specify more than one positional proj. per query.");
}
if (ARRAY_OP_ELEM_MATCH == arrayOpType) {
return Status(ErrorCodes::BadValue,
"Cannot specify positional operator and $elemMatch.");
}
std::string after = mongoutils::str::after(e.fieldName(), ".$");
if (mongoutils::str::contains(after, ".$")) {
mongoutils::str::stream ss;
ss << "Positional projection '" << e.fieldName() << "' contains "
<< "the positional operator more than once.";
return Status(ErrorCodes::BadValue, ss);
}
std::string matchfield = mongoutils::str::before(e.fieldName(), '.');
if (!_hasPositionalOperatorMatch(query, matchfield)) {
mongoutils::str::stream ss;
ss << "Positional projection '" << e.fieldName() << "' does not "
<< "match the query document.";
return Status(ErrorCodes::BadValue, ss);
}
arrayOpType = ARRAY_OP_POSITIONAL;
}
}
// If includeExclude is uninitialized or set to exclude fields, then we can't use an index
// because we don't know what fields we're missing.
if (includeExclude == IncludeExclude::kUninitialized ||
includeExclude == IncludeExclude::kExclude) {
requiresDocument = true;
}
// Fill out the returned obj.
unique_ptr<ParsedProjection> pp(new ParsedProjection());
// The positional operator uses the MatchDetails from the query
// expression to know which array element was matched.
pp->_requiresMatchDetails = arrayOpType == ARRAY_OP_POSITIONAL;
// Save the raw spec. It should be owned by the LiteParsedQuery.
verify(spec.isOwned());
pp->_source = spec;
pp->_returnKey = hasIndexKeyProjection;
pp->_requiresDocument = requiresDocument;
// Add meta-projections.
pp->_wantGeoNearPoint = wantGeoNearPoint;
pp->_wantGeoNearDistance = wantGeoNearDistance;
pp->_wantSortKey = wantSortKey;
// If it's possible to compute the projection in a covered fashion, populate _requiredFields
// so the planner can perform projection analysis.
if (!pp->_requiresDocument) {
if (includeID) {
pp->_requiredFields.push_back("_id");
}
// The only way we could be here is if spec is only simple non-dotted-field inclusions or
// the $meta sortKey projection. Therefore we can iterate over spec to get the fields
// required.
BSONObjIterator srcIt(spec);
while (srcIt.more()) {
BSONElement elt = srcIt.next();
// We've already handled the _id field before entering this loop.
if (includeID && mongoutils::str::equals(elt.fieldName(), "_id")) {
continue;
}
// $meta sortKey should not be checked as a part of _requiredFields, since it can
// potentially produce a covered projection as long as the sort key is covered.
if (BSONType::Object == elt.type()) {
dassert(elt.Obj() == BSON("$meta"
<< "sortKey"));
continue;
}
if (elt.trueValue()) {
pp->_requiredFields.push_back(elt.fieldName());
}
}
}
// returnKey clobbers everything except for sortKey meta-projection.
if (hasIndexKeyProjection && !wantSortKey) {
pp->_requiresDocument = false;
}
*out = pp.release();
return Status::OK();
}
void ParsedQuery::_initTop( const BSONObj& top ) {
BSONObjIterator i( top );
while ( i.more() ) {
BSONElement e = i.next();
const char * name = e.fieldName();
if ( strcmp( "$orderby" , name ) == 0 ||
strcmp( "orderby" , name ) == 0 ) {
if ( e.type() == Object ) {
_order = e.embeddedObject();
}
else if ( e.type() == Array ) {
_order = transformOrderFromArrayFormat( _order );
}
else {
uasserted(13513, "sort must be an object or array");
}
continue;
}
if( *name == '$' ) {
name++;
if ( strcmp( "explain" , name ) == 0 ) {
_explain = e.trueValue();
}
else if ( strcmp( "snapshot" , name ) == 0 ) {
_snapshot = e.trueValue();
}
else if ( strcmp( "min" , name ) == 0 ) {
_min = e.embeddedObject();
}
else if ( strcmp( "max" , name ) == 0 ) {
_max = e.embeddedObject();
}
else if ( strcmp( "hint" , name ) == 0 ) {
_hint = e.wrap();
}
else if ( strcmp( "returnKey" , name ) == 0 ) {
_returnKey = e.trueValue();
}
else if ( strcmp( "maxScan" , name ) == 0 ) {
_maxScan = e.numberInt();
}
else if ( strcmp( "showDiskLoc" , name ) == 0 ) {
_showDiskLoc = e.trueValue();
}
else if ( strcmp( "maxTimeMS" , name ) == 0 ) {
StatusWith<int> maxTimeMS = LiteParsedQuery::parseMaxTimeMSQuery( top );
uassert(17131,
maxTimeMS.getStatus().reason(),
maxTimeMS.isOK());
_maxTimeMS = maxTimeMS.getValue();
}
else if ( strcmp( "comment" , name ) == 0 ) {
; // no-op
}
}
}
if ( _snapshot ) {
uassert( 12001 , "E12001 can't sort with $snapshot", _order.isEmpty() );
uassert( 12002 , "E12002 can't use hint with $snapshot", _hint.isEmpty() );
}
}