PlanStage::StageState FetchStage::fetchCompleted(WorkingSetID* out) {
WorkingSetMember* member = _ws->get(_idBeingPagedIn);
// The DiskLoc we're waiting to page in was invalidated (forced fetch). Test for
// matching and maybe pass it up.
if (member->state == WorkingSetMember::OWNED_OBJ) {
WorkingSetID memberID = _idBeingPagedIn;
_idBeingPagedIn = WorkingSet::INVALID_ID;
return returnIfMatches(member, memberID, out);
}
// Assume that the caller has fetched appropriately.
// TODO: Do we want to double-check the runner? Not sure how reliable likelyInMemory is
// on all platforms.
verify(member->hasLoc());
verify(!member->hasObj());
// Make the (unowned) object.
Record* record = member->loc.rec();
const char* data = record->dataNoThrowing();
member->obj = BSONObj(data);
// Don't need index data anymore as we have an obj.
member->keyData.clear();
member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ;
verify(!member->obj.isOwned());
// Return the obj if it passes our filter.
WorkingSetID memberID = _idBeingPagedIn;
_idBeingPagedIn = WorkingSet::INVALID_ID;
return returnIfMatches(member, memberID, out);
}
bool MultiPlanRunner::workAllPlans(BSONObj* objOut) {
bool planHitEOF = false;
for (size_t i = 0; i < _candidates.size(); ++i) {
CandidatePlan& candidate = _candidates[i];
if (candidate.failed) { continue; }
// Yield, if we can yield ourselves.
if (NULL != _yieldPolicy.get() && _yieldPolicy->shouldYield()) {
saveState();
_yieldPolicy->yield();
if (_failure || _killed) { return false; }
restoreState();
}
WorkingSetID id = WorkingSet::INVALID_ID;
PlanStage::StageState state = candidate.root->work(&id);
if (PlanStage::ADVANCED == state) {
// Save result for later.
candidate.results.push_back(id);
}
else if (PlanStage::NEED_TIME == state) {
// Fall through to yield check at end of large conditional.
}
else if (PlanStage::NEED_FETCH == state) {
// id has a loc and refers to an obj we need to fetch.
WorkingSetMember* member = candidate.ws->get(id);
// This must be true for somebody to request a fetch and can only change when an
// invalidation happens, which is when we give up a lock. Don't give up the
// lock between receiving the NEED_FETCH and actually fetching(?).
verify(member->hasLoc());
// Actually bring record into memory.
Record* record = member->loc.rec();
// If we're allowed to, go to disk outside of the lock.
if (NULL != _yieldPolicy.get()) {
saveState();
_yieldPolicy->yield(record);
if (_failure || _killed) { return false; }
restoreState();
}
else {
// We're set to manually yield. We go to disk in the lock.
record->touch();
}
// Record should be in memory now. Log if it's not.
if (!Record::likelyInPhysicalMemory(record->dataNoThrowing())) {
OCCASIONALLY {
warning() << "Record wasn't in memory immediately after fetch: "
<< member->loc.toString() << endl;
}
}
// Note that we're not freeing id. Fetch semantics say that we shouldn't.
}
PlanStage::StageState FetchStage::work(WorkingSetID* out) {
++_commonStats.works;
if (isEOF()) { return PlanStage::IS_EOF; }
// If we asked our parent for a page-in last time work(...) was called, finish the fetch.
if (WorkingSet::INVALID_ID != _idBeingPagedIn) {
cout << "fetch completed, id being paged on " << _idBeingPagedIn << endl;
return fetchCompleted(out);
}
// If we're here, we're not waiting for a DiskLoc to be fetched. Get another to-be-fetched
// result from our child.
WorkingSetID id;
StageState status = _child->work(&id);
if (PlanStage::ADVANCED == status) {
WorkingSetMember* member = _ws->get(id);
// If there's an obj there, there is no fetching to perform.
if (member->hasObj()) {
++_specificStats.alreadyHasObj;
return returnIfMatches(member, id, out);
}
// We need a valid loc to fetch from and this is the only state that has one.
verify(WorkingSetMember::LOC_AND_IDX == member->state);
verify(member->hasLoc());
Record* record = member->loc.rec();
const char* data = record->dataNoThrowing();
if (!recordInMemory(data)) {
// member->loc points to a record that's NOT in memory. Pass a fetch request up.
verify(WorkingSet::INVALID_ID == _idBeingPagedIn);
_idBeingPagedIn = id;
*out = id;
++_commonStats.needFetch;
return PlanStage::NEED_FETCH;
}
else {
// Don't need index data anymore as we have an obj.
member->keyData.clear();
member->obj = BSONObj(data);
member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ;
return returnIfMatches(member, id, out);
}
}
else {
if (PlanStage::NEED_FETCH == status) {
*out = id;
++_commonStats.needFetch;
}
else if (PlanStage::NEED_TIME == status) {
++_commonStats.needTime;
}
return status;
}
}
bool MultiPlanRunner::workAllPlans() {
for (size_t i = 0; i < _candidates.size(); ++i) {
CandidatePlan& candidate = _candidates[i];
WorkingSetID id;
PlanStage::StageState state = candidate.root->work(&id);
if (PlanStage::ADVANCED == state) {
// Save result for later.
candidate.results.push(id);
}
else if (PlanStage::NEED_TIME == state) {
// Nothing to do here.
}
else if (PlanStage::NEED_FETCH == state) {
// XXX: We can yield to do this. We have to deal with synchronization issues with
// regards to the working set and invalidation. What if another thread invalidates
// the thing we're fetching? The loc could vanish between hasLoc() and the actual
// fetch...
// id has a loc and refers to an obj we need to fetch.
WorkingSetMember* member = candidate.ws->get(id);
// This must be true for somebody to request a fetch and can only change when an
// invalidation happens, which is when we give up a lock. Don't give up the
// lock between receiving the NEED_FETCH and actually fetching(?).
verify(member->hasLoc());
// Actually bring record into memory.
Record* record = member->loc.rec();
record->touch();
// Record should be in memory now. Log if it's not.
if (!Record::likelyInPhysicalMemory(record->dataNoThrowing())) {
OCCASIONALLY {
warning() << "Record wasn't in memory immediately after fetch: "
<< member->loc.toString() << endl;
}
}
// Note that we're not freeing id. Fetch semantics say that we shouldn't.
}
Runner::RunnerState IDHackRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) {
if (_killed) { return Runner::RUNNER_DEAD; }
if (_done) { return Runner::RUNNER_EOF; }
// Use the index catalog to get the id index.
IndexCatalog* catalog = _collection->getIndexCatalog();
// Find the index we use.
const IndexDescriptor* idDesc = catalog->findIdIndex();
if (NULL == idDesc) {
_done = true;
return Runner::RUNNER_EOF;
}
BtreeBasedAccessMethod* accessMethod = catalog->getBtreeBasedIndex( idDesc );
BSONObj key = _query->getQueryObj()["_id"].wrap();
// Look up the key by going directly to the Btree.
DiskLoc loc = accessMethod->findSingle( key );
_done = true;
// Key not found.
if (loc.isNull()) {
return Runner::RUNNER_EOF;
}
// Set out parameters and note that we're done w/lookup.
if (NULL != objOut) {
Record* record = loc.rec();
// If the record isn't in memory...
if (!Record::likelyInPhysicalMemory(record->dataNoThrowing())) {
// And we're allowed to yield ourselves...
if (Runner::YIELD_AUTO == _policy) {
// Note what we're yielding to fetch so that we don't crash if the loc is
// deleted during a yield.
_locFetching = loc;
// Yield. TODO: Do we want to bother yielding if micros < 0?
int micros = ClientCursor::suggestYieldMicros();
ClientCursor::staticYield(micros, "", record);
// This can happen when we're yielded for various reasons (e.g. db/idx dropped).
if (_killed) {
return Runner::RUNNER_DEAD;
}
}
}
// Either the data was in memory or we paged it in.
*objOut = loc.obj();
// If we're sharded make sure the key belongs to us. We need the object to do this.
if (shardingState.needCollectionMetadata(_query->ns())) {
CollectionMetadataPtr m = shardingState.getCollectionMetadata(_query->ns());
if (m) {
KeyPattern kp(m->getKeyPattern());
if (!m->keyBelongsToMe( kp.extractSingleKey(*objOut))) {
// We have something with a matching _id but it doesn't belong to me.
return Runner::RUNNER_EOF;
}
}
}
// If there is a projection...
if (NULL != _query->getProj()) {
// Create something to execute it.
auto_ptr<ProjectionExec> projExec(new ProjectionExec(_query->getParsed().getProj(),
_query->root()));
projExec->transform(*objOut, objOut);
}
}
// Return the DiskLoc if the caller wants it.
if (NULL != dlOut) {
*dlOut = loc;
}
return Runner::RUNNER_ADVANCED;
}
Runner::RunnerState IDHackRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) {
if (_killed) { return Runner::RUNNER_DEAD; }
if (_done) { return Runner::RUNNER_EOF; }
// Use the index catalog to get the id index.
const IndexCatalog* catalog = _collection->getIndexCatalog();
// Find the index we use.
IndexDescriptor* idDesc = catalog->findIdIndex();
if (NULL == idDesc) {
_done = true;
return Runner::RUNNER_EOF;
}
// This may not be valid always. See SERVER-12397.
const BtreeBasedAccessMethod* accessMethod =
static_cast<const BtreeBasedAccessMethod*>(catalog->getIndex(idDesc));
// Look up the key by going directly to the Btree.
DiskLoc loc = accessMethod->findSingle( _key );
_done = true;
// Key not found.
if (loc.isNull()) {
return Runner::RUNNER_EOF;
}
_nscanned++;
// Set out parameters and note that we're done w/lookup.
if (NULL == objOut) {
// No object requested - nothing to do.
}
else if (hasIDProjection(_query.get())) {
// Covered query on _id field only.
// Set object to search key.
// Search key is retrieved from the canonical query at
// construction and always contains the _id field name.
// It is possible to construct the ID hack runner with just the collection
// and the key object (which could be {"": my_obj_id}) but _query would be null
// in that case and the query would never be seen as covered.
*objOut = _key.getOwned();
}
else {
invariant(!hasIDProjection(_query.get()));
// Fetch object from storage.
Record* record = loc.rec();
_nscannedObjects++;
// If the record isn't in memory...
if (!Record::likelyInPhysicalMemory(record->dataNoThrowing())) {
// And we're allowed to yield ourselves...
if (Runner::YIELD_AUTO == _policy) {
// Note what we're yielding to fetch so that we don't crash if the loc is
// deleted during a yield.
_locFetching = loc;
// Yield. TODO: Do we want to bother yielding if micros < 0?
int micros = ClientCursor::suggestYieldMicros();
ClientCursor::staticYield(micros, "", record);
// This can happen when we're yielded for various reasons (e.g. db/idx dropped).
if (_killed) {
return Runner::RUNNER_DEAD;
}
}
}
// Either the data was in memory or we paged it in.
*objOut = loc.obj();
// If we're sharded make sure the key belongs to us. We need the object to do this.
if (shardingState.needCollectionMetadata(_collection->ns().ns())) {
CollectionMetadataPtr m = shardingState.getCollectionMetadata(_collection->ns().ns());
if (m) {
KeyPattern kp(m->getKeyPattern());
if (!m->keyBelongsToMe( kp.extractSingleKey(*objOut))) {
// We have something with a matching _id but it doesn't belong to me.
return Runner::RUNNER_EOF;
}
}
}
}
// Return the DiskLoc if the caller wants it.
if (NULL != dlOut) {
*dlOut = loc;
}
return Runner::RUNNER_ADVANCED;
}