bool Helpers::findById(Client& c, const char *ns, BSONObj query, BSONObj& result ,
bool * nsFound , bool * indexFound ) {
Lock::assertAtLeastReadLocked(ns);
Database *database = c.database();
verify( database );
NamespaceDetails *d = database->namespaceIndex.details(ns);
if ( ! d )
return false;
if ( nsFound )
*nsFound = 1;
int idxNo = d->findIdIndex();
if ( idxNo < 0 )
return false;
if ( indexFound )
*indexFound = 1;
IndexDetails& i = d->idx( idxNo );
BSONObj key = i.getKeyFromQuery( query );
DiskLoc loc = QueryRunner::fastFindSingle(i, key);
if ( loc.isNull() )
return false;
result = loc.obj();
return true;
}
shared_ptr<Cursor> NamespaceDetailsTransient::getCursor( const char *ns, const BSONObj &query, const BSONObj &order ) {
if ( query.isEmpty() && order.isEmpty() ) {
// TODO This will not use a covered index currently.
return theDataFileMgr.findAll( ns );
}
if ( isSimpleIdQuery( query ) ) {
Database *database = cc().database();
assert( database );
NamespaceDetails *d = database->namespaceIndex.details(ns);
if ( d ) {
int idxNo = d->findIdIndex();
if ( idxNo >= 0 ) {
IndexDetails& i = d->idx( idxNo );
BSONObj key = i.getKeyFromQuery( query );
return shared_ptr<Cursor>( BtreeCursor::make( d, idxNo, i, key, key, true, 1 ) );
}
}
}
auto_ptr<MultiPlanScanner> mps( new MultiPlanScanner( ns, query, order ) ); // mayYield == false
shared_ptr<Cursor> single = mps->singleCursor();
if ( single ) {
if ( !query.isEmpty() && !single->matcher() ) {
shared_ptr<CoveredIndexMatcher> matcher( new CoveredIndexMatcher( query, single->indexKeyPattern() ) );
single->setMatcher( matcher );
}
return single;
}
return newQueryOptimizerCursor( mps );
}
void run() {
// This will succeed, but not insert anything because they are changed to upserts
for (int i=0; i<150; i++) {
insertSucceed();
}
// Just to be sure, no _id index, right?
Client::Context ctx(cappedNs());
NamespaceDetails *nsd = nsdetails(cappedNs().c_str());
verify(nsd->findIdIndex() == -1);
}
void run() {
// This will succeed, but not insert anything because they are changed to upserts
for (int i=0; i<150; i++) {
insertSucceed();
}
// this changed in 2.1.2
// we now have indexes on capped collections
Client::Context ctx(cappedNs());
NamespaceDetails *nsd = nsdetails(cappedNs().c_str());
verify(nsd->findIdIndex() >= 0);
}
void run() {
// RARELY shoud be once/128x
for (int i=0; i<150; i++) {
insert();
updateSucceed();
}
DBDirectClient client;
int count = (int) client.count(cappedNs(), BSONObj());
verify(count > 1);
// Just to be sure, no _id index, right?
Client::Context ctx(cappedNs());
NamespaceDetails *nsd = nsdetails(cappedNs().c_str());
verify(nsd->findIdIndex() == -1);
}
void CursorGenerator::setArgumentsHint() {
if ( useHints && _parsedQuery ) {
_argumentsHint = _parsedQuery->getHint();
}
if ( snapshot() ) {
NamespaceDetails *d = nsdetails( _ns );
if ( d ) {
int i = d->findIdIndex();
if( i < 0 ) {
if ( _ns.find( ".system." ) == string::npos )
log() << "warning: no _id index on $snapshot query, ns:" << _ns << endl;
}
else {
/* [dm] the name of an _id index tends to vary, so we build the hint the hard
way here. probably need a better way to specify "use the _id index" as a hint.
if someone is in the query optimizer please fix this then!
*/
_argumentsHint = BSON( "$hint" << d->idx(i).indexName() );
}
}
}
}
shared_ptr<Cursor> CursorGenerator::shortcutCursor() const {
if ( !mayShortcutQueryOptimizer() ) {
return shared_ptr<Cursor>();
}
if ( _planPolicy.permitOptimalNaturalPlan() && _query.isEmpty() && _order.isEmpty() ) {
return theDataFileMgr.findAll( _ns );
}
if ( _planPolicy.permitOptimalIdPlan() && isSimpleIdQuery( _query ) ) {
Database *database = cc().database();
verify( database );
NamespaceDetails *d = database->namespaceIndex.details( _ns );
if ( d ) {
int idxNo = d->findIdIndex();
if ( idxNo >= 0 ) {
IndexDetails& i = d->idx( idxNo );
BSONObj key = i.getKeyFromQuery( _query );
return shared_ptr<Cursor>( BtreeCursor::make( d, i, key, key, true, 1 ) );
}
}
}
return shared_ptr<Cursor>();
}
UpdateResult _updateObjects( const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy ) {
TOKULOG(2) << "update: " << ns
<< " update: " << updateobj
<< " query: " << patternOrig
<< " upsert: " << upsert << " multi: " << multi << endl;
debug.updateobj = updateobj;
NamespaceDetails *d = getAndMaybeCreateNS(ns, logop);
auto_ptr<ModSet> mods;
const bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$';
bool modsAreIndexed = false;
if ( isOperatorUpdate ) {
if ( d->indexBuildInProgress() ) {
set<string> bgKeys;
d->inProgIdx().keyPattern().getFieldNames(bgKeys);
mods.reset( new ModSet(updateobj, d->indexKeys(), &bgKeys) );
}
else {
mods.reset( new ModSet(updateobj, d->indexKeys()) );
}
modsAreIndexed = mods->isIndexed();
}
int idIdxNo = -1;
if ( planPolicy.permitOptimalIdPlan() && !multi && !modsAreIndexed &&
(idIdxNo = d->findIdIndex()) >= 0 && mayUpdateById(d, patternOrig) ) {
debug.idhack = true;
IndexDetails &idx = d->idx(idIdxNo);
BSONObj pk = idx.getKeyFromQuery(patternOrig);
TOKULOG(3) << "_updateObjects using simple _id query, pattern " << patternOrig << ", pk " << pk << endl;
UpdateResult result = _updateById( pk,
isOperatorUpdate,
mods.get(),
d,
ns,
updateobj,
patternOrig,
logop,
debug,
fromMigrate);
if ( result.existing || ! upsert ) {
return result;
}
else if ( upsert && ! isOperatorUpdate && ! logop) {
debug.upsert = true;
BSONObj objModified = updateobj;
insertAndLog( ns, d, objModified, logop, fromMigrate );
return UpdateResult( 0 , 0 , 1 , updateobj );
}
}
int numModded = 0;
debug.nscanned = 0;
shared_ptr<Cursor> c = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy );
if( c->ok() ) {
set<BSONObj> seenObjects;
MatchDetails details;
auto_ptr<ClientCursor> cc;
do {
debug.nscanned++;
if ( mods.get() && mods->hasDynamicArray() ) {
// The Cursor must have a Matcher to record an elemMatchKey. But currently
// a modifier on a dynamic array field may be applied even if there is no
// elemMatchKey, so a matcher cannot be required.
//verify( c->matcher() );
details.requestElemMatchKey();
}
if ( !c->currentMatches( &details ) ) {
c->advance();
continue;
}
BSONObj currPK = c->currPK();
if ( c->getsetdup( currPK ) ) {
c->advance();
continue;
}
BSONObj currentObj = c->current();
BSONObj pattern = patternOrig;
if ( logop ) {
BSONObjBuilder idPattern;
BSONElement id;
// NOTE: If the matching object lacks an id, we'll log
// with the original pattern. This isn't replay-safe.
// It might make sense to suppress the log instead
// if there's no id.
if ( currentObj.getObjectID( id ) ) {
idPattern.append( id );
pattern = idPattern.obj();
}
else {
uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi );
}
}
/* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some
regular ones at the moment. */
struct LogOpUpdateDetails loud;
loud.logop = logop;
loud.ns = ns;
loud.fromMigrate = fromMigrate;
if ( isOperatorUpdate ) {
if ( multi ) {
// Make our own copies of the currPK and currentObj before we invalidate
// them by advancing the cursor.
currPK = currPK.copy();
currentObj = currentObj.copy();
// Advance past the document to be modified. This used to be because of SERVER-5198,
// but TokuMX does it because we want to avoid needing to do manual deduplication
// of this PK on the next iteration if the current update modifies the next
// entry in the index. For example, an index scan over a:1 with mod {$inc: {a:1}}
// would cause every other key read to be a duplicate if we didn't advance here.
while ( c->ok() && currPK == c->currPK() ) {
c->advance();
}
// Multi updates need to do their own deduplication because updates may modify the
// keys the cursor is in the process of scanning over.
if ( seenObjects.count( currPK ) ) {
continue;
} else {
seenObjects.insert( currPK );
}
}
ModSet* useMods = mods.get();
auto_ptr<ModSet> mymodset;
if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) {
useMods = mods->fixDynamicArray( details.elemMatchKey() );
mymodset.reset( useMods );
}
auto_ptr<ModSetState> mss = useMods->prepare( currentObj );
updateUsingMods( d, currPK, currentObj, *mss, &loud );
numModded++;
if ( ! multi )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
continue;
} // end if operator is update
uassert( 10158 , "multi update only works with $ operators" , ! multi );
updateNoMods( d, currPK, currentObj, updateobj, &loud );
return UpdateResult( 1 , 0 , 1 , BSONObj() );
} while ( c->ok() );
} // endif
if ( numModded )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( upsert ) {
BSONObj newObj = updateobj;
if ( updateobj.firstElementFieldName()[0] == '$' ) {
// upsert of an $operation. build a default object
BSONObj newObj = mods->createNewFromQuery( patternOrig );
debug.fastmodinsert = true;
insertAndLog( ns, d, newObj, logop, fromMigrate );
return UpdateResult( 0 , 1 , 1 , newObj );
}
uassert( 10159 , "multi update only works with $ operators" , ! multi );
debug.upsert = true;
insertAndLog( ns, d, newObj, logop, fromMigrate );
return UpdateResult( 0 , 0 , 1 , newObj );
}
return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() );
}
UpdateResult _updateObjects( bool su,
const char* ns,
const BSONObj& updateobj,
const BSONObj& patternOrig,
bool upsert,
bool multi,
bool logop ,
OpDebug& debug,
RemoveSaver* rs,
bool fromMigrate,
const QueryPlanSelectionPolicy& planPolicy,
bool forReplication ) {
DEBUGUPDATE( "update: " << ns
<< " update: " << updateobj
<< " query: " << patternOrig
<< " upsert: " << upsert << " multi: " << multi );
Client& client = cc();
debug.updateobj = updateobj;
// The idea with these here it to make them loop invariant for
// multi updates, and thus be a bit faster for that case. The
// pointers may be left invalid on a failed or terminal yield
// recovery.
NamespaceDetails* d = nsdetails(ns); // can be null if an upsert...
NamespaceDetailsTransient* nsdt = &NamespaceDetailsTransient::get(ns);
auto_ptr<ModSet> mods;
bool isOperatorUpdate = updateobj.firstElementFieldName()[0] == '$';
int modsIsIndexed = false; // really the # of indexes
if ( isOperatorUpdate ) {
mods.reset( new ModSet(updateobj, nsdt->indexKeys(), forReplication) );
modsIsIndexed = mods->maxNumIndexUpdated();
}
if( planPolicy.permitOptimalIdPlan() && !multi && isSimpleIdQuery(patternOrig) && d &&
!modsIsIndexed ) {
int idxNo = d->findIdIndex();
if( idxNo >= 0 ) {
debug.idhack = true;
UpdateResult result = _updateById( isOperatorUpdate,
idxNo,
mods.get(),
d,
nsdt,
su,
ns,
updateobj,
patternOrig,
logop,
debug,
fromMigrate);
if ( result.existing || ! upsert ) {
return result;
}
else if ( upsert && ! isOperatorUpdate ) {
// this handles repl inserts
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, false, su);
if ( logop )
logOp( "i", ns, no, 0, 0, fromMigrate, &no );
return UpdateResult( 0 , 0 , 1 , no );
}
}
}
int numModded = 0;
debug.nscanned = 0;
shared_ptr<Cursor> c = getOptimizedCursor( ns, patternOrig, BSONObj(), planPolicy );
d = nsdetails(ns);
nsdt = &NamespaceDetailsTransient::get(ns);
bool autoDedup = c->autoDedup();
if( c->ok() ) {
set<DiskLoc> seenObjects;
MatchDetails details;
auto_ptr<ClientCursor> cc;
do {
if ( cc.get() == 0 &&
client.allowedToThrowPageFaultException() &&
! c->currLoc().isNull() &&
! c->currLoc().rec()->likelyInPhysicalMemory() ) {
throw PageFaultException( c->currLoc().rec() );
}
bool atomic = c->matcher() && c->matcher()->docMatcher().atomic();
if ( ! atomic && debug.nscanned > 0 ) {
// we need to use a ClientCursor to yield
if ( cc.get() == 0 ) {
shared_ptr< Cursor > cPtr = c;
cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
}
bool didYield;
if ( ! cc->yieldSometimes( ClientCursor::WillNeed, &didYield ) ) {
cc.release();
break;
}
if ( !c->ok() ) {
break;
}
if ( didYield ) {
d = nsdetails(ns);
if ( ! d )
break;
nsdt = &NamespaceDetailsTransient::get(ns);
if ( mods.get() ) {
mods->setIndexedStatus( nsdt->indexKeys() );
modsIsIndexed = mods->maxNumIndexUpdated();
}
}
} // end yielding block
debug.nscanned++;
if ( mods.get() && mods->hasDynamicArray() ) {
details.requestElemMatchKey();
}
if ( !c->currentMatches( &details ) ) {
c->advance();
continue;
}
Record* r = c->_current();
DiskLoc loc = c->currLoc();
if ( c->getsetdup( loc ) && autoDedup ) {
c->advance();
continue;
}
BSONObj js = BSONObj::make(r);
BSONObj pattern = patternOrig;
if ( logop ) {
BSONObjBuilder idPattern;
BSONElement id;
// NOTE: If the matching object lacks an id, we'll log
// with the original pattern. This isn't replay-safe.
// It might make sense to suppress the log instead
// if there's no id.
if ( js.getObjectID( id ) ) {
idPattern.append( id );
pattern = idPattern.obj();
}
else {
uassert( 10157 , "multi-update requires all modified objects to have an _id" , ! multi );
}
}
/* look for $inc etc. note as listed here, all fields to inc must be this type, you can't set some
regular ones at the moment. */
if ( isOperatorUpdate ) {
if ( multi ) {
// go to next record in case this one moves
c->advance();
// Update operations are deduped for cursors that implement their own
// deduplication. In particular, some geo cursors are excluded.
if ( autoDedup ) {
if ( seenObjects.count( loc ) ) {
continue;
}
// SERVER-5198 Advance past the document to be modified, provided
// deduplication is enabled, but see SERVER-5725.
while( c->ok() && loc == c->currLoc() ) {
c->advance();
}
}
}
const BSONObj& onDisk = loc.obj();
ModSet* useMods = mods.get();
auto_ptr<ModSet> mymodset;
if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) {
useMods = mods->fixDynamicArray( details.elemMatchKey() );
mymodset.reset( useMods );
}
auto_ptr<ModSetState> mss = useMods->prepare( onDisk,
false /* not an insertion */ );
bool willAdvanceCursor = multi && c->ok() && ( modsIsIndexed || ! mss->canApplyInPlace() );
if ( willAdvanceCursor ) {
if ( cc.get() ) {
cc->setDoingDeletes( true );
}
c->prepareToTouchEarlierIterate();
}
// If we've made it this far, "ns" must contain a valid collection name, and so
// is of the form "db.collection". Therefore, the following expression must
// always be valid. "system.users" updates must never be done in place, in
// order to ensure that they are validated inside DataFileMgr::updateRecord(.).
bool isSystemUsersMod = (NamespaceString(ns).coll == "system.users");
BSONObj newObj;
if ( !mss->isUpdateIndexed() && mss->canApplyInPlace() && !isSystemUsersMod ) {
mss->applyModsInPlace( true );// const_cast<BSONObj&>(onDisk) );
DEBUGUPDATE( "\t\t\t doing in place update" );
if ( !multi )
debug.fastmod = true;
if ( modsIsIndexed ) {
seenObjects.insert( loc );
}
newObj = loc.obj();
d->paddingFits();
}
else {
newObj = mss->createNewFromMods();
checkTooLarge(newObj);
DiskLoc newLoc = theDataFileMgr.updateRecord(ns,
d,
nsdt,
r,
loc,
newObj.objdata(),
newObj.objsize(),
debug);
if ( newLoc != loc || modsIsIndexed ){
// log() << "Moved obj " << newLoc.obj()["_id"] << " from " << loc << " to " << newLoc << endl;
// object moved, need to make sure we don' get again
seenObjects.insert( newLoc );
}
}
if ( logop ) {
DEV verify( mods->size() );
BSONObj logObj = mss->getOpLogRewrite();
DEBUGUPDATE( "\t rewrite update: " << logObj );
// It is possible that the entire mod set was a no-op over this
// document. We would have an empty log record in that case. If we
// call logOp, with an empty record, that would be replicated as "clear
// this record", which is not what we want. Therefore, to get a no-op
// in the replica, we simply don't log.
if ( logObj.nFields() ) {
logOp("u", ns, logObj , &pattern, 0, fromMigrate, &newObj );
}
}
numModded++;
if ( ! multi )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( willAdvanceCursor )
c->recoverFromTouchingEarlierIterate();
getDur().commitIfNeeded();
continue;
}
uassert( 10158 , "multi update only works with $ operators" , ! multi );
BSONElementManipulator::lookForTimestamps( updateobj );
checkNoMods( updateobj );
theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, su);
if ( logop ) {
DEV wassert( !su ); // super used doesn't get logged, this would be bad.
logOp("u", ns, updateobj, &pattern, 0, fromMigrate, &updateobj );
}
return UpdateResult( 1 , 0 , 1 , BSONObj() );
} while ( c->ok() );
} // endif
if ( numModded )
return UpdateResult( 1 , 1 , numModded , BSONObj() );
if ( upsert ) {
if ( updateobj.firstElementFieldName()[0] == '$' ) {
// upsert of an $operation. build a default object
BSONObj newObj = mods->createNewFromQuery( patternOrig );
checkNoMods( newObj );
debug.fastmodinsert = true;
theDataFileMgr.insertWithObjMod(ns, newObj, false, su);
if ( logop )
logOp( "i", ns, newObj, 0, 0, fromMigrate, &newObj );
return UpdateResult( 0 , 1 , 1 , newObj );
}
uassert( 10159 , "multi update only works with $ operators" , ! multi );
checkNoMods( updateobj );
debug.upsert = true;
BSONObj no = updateobj;
theDataFileMgr.insertWithObjMod(ns, no, false, su);
if ( logop )
logOp( "i", ns, no, 0, 0, fromMigrate, &no );
return UpdateResult( 0 , 0 , 1 , no );
}
return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() );
}
void QueryPlanSet::init() {
DEBUGQO( "QueryPlanSet::init " << ns << "\t" << _originalQuery );
_plans.clear();
_mayRecordPlan = true;
_usingPrerecordedPlan = false;
const char *ns = _frsp->ns();
NamespaceDetails *d = nsdetails( ns );
if ( !d || !_frsp->matchPossible() ) {
// Table scan plan, when no matches are possible
_plans.push_back( QueryPlanPtr( new QueryPlan( d, -1, *_frsp, *_originalFrsp, _originalQuery, _order ) ) );
return;
}
BSONElement hint = _hint.firstElement();
if ( !hint.eoo() ) {
_mayRecordPlan = false;
IndexDetails *id = parseHint( hint, d );
if ( id ) {
addHint( *id );
}
else {
massert( 10366 , "natural order cannot be specified with $min/$max", _min.isEmpty() && _max.isEmpty() );
// Table scan plan
_plans.push_back( QueryPlanPtr( new QueryPlan( d, -1, *_frsp, *_originalFrsp, _originalQuery, _order ) ) );
}
return;
}
if ( !_min.isEmpty() || !_max.isEmpty() ) {
string errmsg;
BSONObj keyPattern;
IndexDetails *idx = indexDetailsForRange( ns, errmsg, _min, _max, keyPattern );
massert( 10367 , errmsg, idx );
_plans.push_back( QueryPlanPtr( new QueryPlan( d, d->idxNo(*idx), *_frsp, *_originalFrsp, _originalQuery, _order, _min, _max ) ) );
return;
}
if ( isSimpleIdQuery( _originalQuery ) ) {
int idx = d->findIdIndex();
if ( idx >= 0 ) {
_usingPrerecordedPlan = true;
_mayRecordPlan = false;
_plans.push_back( QueryPlanPtr( new QueryPlan( d , idx , *_frsp , *_originalFrsp , _originalQuery, _order ) ) );
return;
}
}
if ( _originalQuery.isEmpty() && _order.isEmpty() ) {
_plans.push_back( QueryPlanPtr( new QueryPlan( d, -1, *_frsp, *_originalFrsp, _originalQuery, _order ) ) );
return;
}
DEBUGQO( "\t special : " << _frsp->getSpecial() );
if ( _frsp->getSpecial().size() ) {
_special = _frsp->getSpecial();
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
int j = i.pos();
IndexDetails& ii = i.next();
const IndexSpec& spec = ii.getSpec();
if ( spec.getTypeName() == _special && spec.suitability( _originalQuery , _order ) ) {
_usingPrerecordedPlan = true;
_mayRecordPlan = false;
_plans.push_back( QueryPlanPtr( new QueryPlan( d , j , *_frsp , *_originalFrsp , _originalQuery, _order ,
BSONObj() , BSONObj() , _special ) ) );
return;
}
}
uassert( 13038 , (string)"can't find special index: " + _special + " for: " + _originalQuery.toString() , 0 );
}
if ( _honorRecordedPlan ) {
pair< BSONObj, long long > best = QueryUtilIndexed::bestIndexForPatterns( *_frsp, _order );
BSONObj bestIndex = best.first;
long long oldNScanned = best.second;
if ( !bestIndex.isEmpty() ) {
QueryPlanPtr p;
_oldNScanned = oldNScanned;
if ( !strcmp( bestIndex.firstElement().fieldName(), "$natural" ) ) {
// Table scan plan
p.reset( new QueryPlan( d, -1, *_frsp, *_originalFrsp, _originalQuery, _order ) );
}
NamespaceDetails::IndexIterator i = d->ii();
while( i.more() ) {
int j = i.pos();
IndexDetails& ii = i.next();
if( ii.keyPattern().woCompare(bestIndex) == 0 ) {
p.reset( new QueryPlan( d, j, *_frsp, *_originalFrsp, _originalQuery, _order ) );
}
}
massert( 10368 , "Unable to locate previously recorded index", p.get() );
if ( !( _bestGuessOnly && p->scanAndOrderRequired() ) ) {
_usingPrerecordedPlan = true;
_mayRecordPlan = false;
_plans.push_back( p );
return;
}
}
}
addOtherPlans( false );
}