void SortKeyGenerator::getBoundsForSort(OperationContext* txn,
const BSONObj& queryObj,
const BSONObj& sortObj) {
QueryPlannerParams params;
params.options = QueryPlannerParams::NO_TABLE_SCAN;
// We're creating a "virtual index" with key pattern equal to the sort order.
IndexEntry sortOrder(sortObj,
IndexNames::BTREE,
true,
MultikeyPaths{},
false,
false,
"doesnt_matter",
NULL,
BSONObj());
params.indices.push_back(sortOrder);
auto statusWithQueryForSort = CanonicalQuery::canonicalize(
txn, NamespaceString("fake.ns"), queryObj, ExtensionsCallbackNoop());
verify(statusWithQueryForSort.isOK());
std::unique_ptr<CanonicalQuery> queryForSort = std::move(statusWithQueryForSort.getValue());
std::vector<QuerySolution*> solns;
LOG(5) << "Sort key generation: Planning to obtain bounds for sort.";
QueryPlanner::plan(*queryForSort, params, &solns);
// TODO: are there ever > 1 solns? If so, do we look for a specific soln?
if (1 == solns.size()) {
IndexScanNode* ixScan = NULL;
QuerySolutionNode* rootNode = solns[0]->root.get();
if (rootNode->getType() == STAGE_FETCH) {
FetchNode* fetchNode = static_cast<FetchNode*>(rootNode);
if (fetchNode->children[0]->getType() != STAGE_IXSCAN) {
delete solns[0];
// No bounds.
return;
}
ixScan = static_cast<IndexScanNode*>(fetchNode->children[0]);
} else if (rootNode->getType() == STAGE_IXSCAN) {
ixScan = static_cast<IndexScanNode*>(rootNode);
}
if (ixScan) {
_bounds.fields.swap(ixScan->bounds.fields);
_hasBounds = true;
}
}
for (size_t i = 0; i < solns.size(); ++i) {
delete solns[i];
}
}
/**
* If possible, turn the provided QuerySolution into a QuerySolution that uses a DistinctNode
* to provide results for the distinct command.
*
* If the provided solution could be mutated successfully, returns true, otherwise returns
* false.
*/
bool turnIxscanIntoDistinctIxscan(QuerySolution* soln, const string& field) {
QuerySolutionNode* root = soln->root.get();
// We're looking for a project on top of an ixscan.
if (STAGE_PROJECTION == root->getType() && (STAGE_IXSCAN == root->children[0]->getType())) {
IndexScanNode* isn = static_cast<IndexScanNode*>(root->children[0]);
// An additional filter must be applied to the data in the key, so we can't just skip
// all the keys with a given value; we must examine every one to find the one that (may)
// pass the filter.
if (NULL != isn->filter.get()) {
return false;
}
// We only set this when we have special query modifiers (.max() or .min()) or other
// special cases. Don't want to handle the interactions between those and distinct.
// Don't think this will ever really be true but if it somehow is, just ignore this
// soln.
if (isn->bounds.isSimpleRange) {
return false;
}
// Make a new DistinctNode. We swap this for the ixscan in the provided solution.
DistinctNode* dn = new DistinctNode();
dn->indexKeyPattern = isn->indexKeyPattern;
dn->direction = isn->direction;
dn->bounds = isn->bounds;
// Figure out which field we're skipping to the next value of. TODO: We currently only
// try to distinct-hack when there is an index prefixed by the field we're distinct-ing
// over. Consider removing this code if we stick with that policy.
dn->fieldNo = 0;
BSONObjIterator it(isn->indexKeyPattern);
while (it.more()) {
if (field == it.next().fieldName()) {
break;
}
dn->fieldNo++;
}
// Delete the old index scan, set the child of project to the fast distinct scan.
delete root->children[0];
root->children[0] = dn;
return true;
}
return false;
}
void SortStageKeyGenerator::getBoundsForSort(const BSONObj& queryObj, const BSONObj& sortObj) {
QueryPlannerParams params;
params.options = QueryPlannerParams::NO_TABLE_SCAN;
// We're creating a "virtual index" with key pattern equal to the sort order.
IndexEntry sortOrder(sortObj, IndexNames::BTREE, true, false, false, "doesnt_matter",
BSONObj());
params.indices.push_back(sortOrder);
CanonicalQuery* rawQueryForSort;
verify(CanonicalQuery::canonicalize(
"fake_ns", queryObj, &rawQueryForSort, WhereCallbackNoop()).isOK());
auto_ptr<CanonicalQuery> queryForSort(rawQueryForSort);
vector<QuerySolution*> solns;
QLOG() << "Sort stage: Planning to obtain bounds for sort." << endl;
QueryPlanner::plan(*queryForSort, params, &solns);
// TODO: are there ever > 1 solns? If so, do we look for a specific soln?
if (1 == solns.size()) {
IndexScanNode* ixScan = NULL;
QuerySolutionNode* rootNode = solns[0]->root.get();
if (rootNode->getType() == STAGE_FETCH) {
FetchNode* fetchNode = static_cast<FetchNode*>(rootNode);
if (fetchNode->children[0]->getType() != STAGE_IXSCAN) {
delete solns[0];
// No bounds.
return;
}
ixScan = static_cast<IndexScanNode*>(fetchNode->children[0]);
}
else if (rootNode->getType() == STAGE_IXSCAN) {
ixScan = static_cast<IndexScanNode*>(rootNode);
}
if (ixScan) {
_bounds.fields.swap(ixScan->bounds.fields);
_hasBounds = true;
}
}
for (size_t i = 0; i < solns.size(); ++i) {
delete solns[i];
}
}
//static
bool StageBuilder::build(const QuerySolution& solution, PlanStage** rootOut,
WorkingSet** wsOut) {
QuerySolutionNode* root = solution.root.get();
if (NULL == root) { return false; }
if (STAGE_COLLSCAN == root->getType()) {
const CollectionScanNode* csn = static_cast<const CollectionScanNode*>(root);
CollectionScanParams params;
params.ns = csn->name;
params.tailable = csn->tailable;
params.direction = (csn->direction == 1) ? CollectionScanParams::FORWARD
: CollectionScanParams::BACKWARD;
*wsOut = new WorkingSet();
*rootOut = new CollectionScan(params, *wsOut, csn->filter);
return true;
}
else {
return false;
}
}
Status SubplanStage::choosePlanForSubqueries(PlanYieldPolicy* yieldPolicy) {
// This is what we annotate with the index selections and then turn into a solution.
auto_ptr<OrMatchExpression> orExpr(
static_cast<OrMatchExpression*>(_query->root()->shallowClone()));
// This is the skeleton of index selections that is inserted into the cache.
auto_ptr<PlanCacheIndexTree> cacheData(new PlanCacheIndexTree());
for (size_t i = 0; i < orExpr->numChildren(); ++i) {
MatchExpression* orChild = orExpr->getChild(i);
BranchPlanningResult* branchResult = _branchResults[i];
if (branchResult->cachedSolution.get()) {
// We can get the index tags we need out of the cache.
Status tagStatus = tagOrChildAccordingToCache(
cacheData.get(),
branchResult->cachedSolution->plannerData[0],
orChild,
_indexMap);
if (!tagStatus.isOK()) {
return tagStatus;
}
}
else if (1 == branchResult->solutions.size()) {
QuerySolution* soln = branchResult->solutions.front();
Status tagStatus = tagOrChildAccordingToCache(cacheData.get(),
soln->cacheData.get(),
orChild,
_indexMap);
if (!tagStatus.isOK()) {
return tagStatus;
}
}
else {
// N solutions, rank them.
// We already checked for zero solutions in planSubqueries(...).
invariant(!branchResult->solutions.empty());
_ws->clear();
_child.reset(new MultiPlanStage(_txn, _collection,
branchResult->canonicalQuery.get()));
MultiPlanStage* multiPlanStage = static_cast<MultiPlanStage*>(_child.get());
// Dump all the solutions into the MPS.
for (size_t ix = 0; ix < branchResult->solutions.size(); ++ix) {
PlanStage* nextPlanRoot;
invariant(StageBuilder::build(_txn,
_collection,
*branchResult->solutions[ix],
_ws,
&nextPlanRoot));
// Takes ownership of solution with index 'ix' and 'nextPlanRoot'.
multiPlanStage->addPlan(branchResult->solutions.releaseAt(ix),
nextPlanRoot,
_ws);
}
Status planSelectStat = multiPlanStage->pickBestPlan(yieldPolicy);
if (!planSelectStat.isOK()) {
return planSelectStat;
}
if (!multiPlanStage->bestPlanChosen()) {
mongoutils::str::stream ss;
ss << "Failed to pick best plan for subchild "
<< branchResult->canonicalQuery->toString();
return Status(ErrorCodes::BadValue, ss);
}
QuerySolution* bestSoln = multiPlanStage->bestSolution();
// Check that we have good cache data. For example, we don't cache things
// for 2d indices.
if (NULL == bestSoln->cacheData.get()) {
mongoutils::str::stream ss;
ss << "No cache data for subchild " << orChild->toString();
return Status(ErrorCodes::BadValue, ss);
}
if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) {
mongoutils::str::stream ss;
ss << "No indexed cache data for subchild "
<< orChild->toString();
return Status(ErrorCodes::BadValue, ss);
}
// Add the index assignments to our original query.
Status tagStatus = QueryPlanner::tagAccordingToCache(
orChild, bestSoln->cacheData->tree.get(), _indexMap);
if (!tagStatus.isOK()) {
mongoutils::str::stream ss;
ss << "Failed to extract indices from subchild "
<< orChild->toString();
return Status(ErrorCodes::BadValue, ss);
}
cacheData->children.push_back(bestSoln->cacheData->tree->clone());
}
}
// Must do this before using the planner functionality.
sortUsingTags(orExpr.get());
// Use the cached index assignments to build solnRoot. Takes ownership of 'orExpr'.
QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(
*_query, orExpr.release(), false, _plannerParams.indices, _plannerParams);
if (NULL == solnRoot) {
mongoutils::str::stream ss;
ss << "Failed to build indexed data path for subplanned query\n";
return Status(ErrorCodes::BadValue, ss);
}
QLOG() << "Subplanner: fully tagged tree is " << solnRoot->toString();
// Takes ownership of 'solnRoot'
_compositeSolution.reset(QueryPlannerAnalysis::analyzeDataAccess(*_query,
_plannerParams,
solnRoot));
if (NULL == _compositeSolution.get()) {
mongoutils::str::stream ss;
ss << "Failed to analyze subplanned query";
return Status(ErrorCodes::BadValue, ss);
}
QLOG() << "Subplanner: Composite solution is " << _compositeSolution->toString() << endl;
// Use the index tags from planning each branch to construct the composite solution,
// and set that solution as our child stage.
_ws->clear();
PlanStage* root;
invariant(StageBuilder::build(_txn, _collection, *_compositeSolution.get(), _ws, &root));
_child.reset(root);
return Status::OK();
}
bool SubplanRunner::runSubplans() {
// This is what we annotate with the index selections and then turn into a solution.
auto_ptr<OrMatchExpression> theOr(
static_cast<OrMatchExpression*>(_query->root()->shallowClone()));
// This is the skeleton of index selections that is inserted into the cache.
auto_ptr<PlanCacheIndexTree> cacheData(new PlanCacheIndexTree());
for (size_t i = 0; i < theOr->numChildren(); ++i) {
MatchExpression* orChild = theOr->getChild(i);
auto_ptr<CanonicalQuery> orChildCQ(_cqs.front());
_cqs.pop();
// 'solutions' is owned by the SubplanRunner instance until
// it is popped from the queue.
vector<QuerySolution*> solutions = _solutions.front();
_solutions.pop();
// We already checked for zero solutions in planSubqueries(...).
invariant(!solutions.empty());
if (1 == solutions.size()) {
// There is only one solution. Transfer ownership to an auto_ptr.
auto_ptr<QuerySolution> autoSoln(solutions[0]);
// We want a well-formed *indexed* solution.
if (NULL == autoSoln->cacheData.get()) {
// For example, we don't cache things for 2d indices.
QLOG() << "Subplanner: No cache data for subchild " << orChild->toString();
return false;
}
if (SolutionCacheData::USE_INDEX_TAGS_SOLN != autoSoln->cacheData->solnType) {
QLOG() << "Subplanner: No indexed cache data for subchild "
<< orChild->toString();
return false;
}
// Add the index assignments to our original query.
Status tagStatus = QueryPlanner::tagAccordingToCache(
orChild, autoSoln->cacheData->tree.get(), _indexMap);
if (!tagStatus.isOK()) {
QLOG() << "Subplanner: Failed to extract indices from subchild "
<< orChild->toString();
return false;
}
// Add the child's cache data to the cache data we're creating for the main query.
cacheData->children.push_back(autoSoln->cacheData->tree->clone());
}
else {
// N solutions, rank them. Takes ownership of orChildCQ.
// the working set will be shared by the candidate plans and owned by the runner
WorkingSet* sharedWorkingSet = new WorkingSet();
MultiPlanStage* multiPlanStage = new MultiPlanStage(_collection,
orChildCQ.get());
// Dump all the solutions into the MPR.
for (size_t ix = 0; ix < solutions.size(); ++ix) {
PlanStage* nextPlanRoot;
verify(StageBuilder::build(_txn,
_collection,
*solutions[ix],
sharedWorkingSet,
&nextPlanRoot));
// Owns first two arguments
multiPlanStage->addPlan(solutions[ix], nextPlanRoot, sharedWorkingSet);
}
multiPlanStage->pickBestPlan();
if (! multiPlanStage->bestPlanChosen()) {
QLOG() << "Subplanner: Failed to pick best plan for subchild "
<< orChildCQ->toString();
return false;
}
Runner* mpr = new SingleSolutionRunner(_collection,
orChildCQ.release(),
multiPlanStage->bestSolution(),
multiPlanStage,
sharedWorkingSet);
_underlyingRunner.reset(mpr);
if (_killed) {
QLOG() << "Subplanner: Killed while picking best plan for subchild "
<< orChild->toString();
return false;
}
QuerySolution* bestSoln = multiPlanStage->bestSolution();
if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) {
QLOG() << "Subplanner: No indexed cache data for subchild "
<< orChild->toString();
return false;
}
// Add the index assignments to our original query.
Status tagStatus = QueryPlanner::tagAccordingToCache(
orChild, bestSoln->cacheData->tree.get(), _indexMap);
if (!tagStatus.isOK()) {
QLOG() << "Subplanner: Failed to extract indices from subchild "
<< orChild->toString();
return false;
}
cacheData->children.push_back(bestSoln->cacheData->tree->clone());
}
}
// Must do this before using the planner functionality.
sortUsingTags(theOr.get());
// Use the cached index assignments to build solnRoot. Takes ownership of 'theOr'
QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(
*_query, theOr.release(), false, _plannerParams.indices);
if (NULL == solnRoot) {
QLOG() << "Subplanner: Failed to build indexed data path for subplanned query\n";
return false;
}
QLOG() << "Subplanner: fully tagged tree is " << solnRoot->toString();
// Takes ownership of 'solnRoot'
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*_query,
_plannerParams,
solnRoot);
if (NULL == soln) {
QLOG() << "Subplanner: Failed to analyze subplanned query";
return false;
}
// We want our franken-solution to be cached.
SolutionCacheData* scd = new SolutionCacheData();
scd->tree.reset(cacheData.release());
soln->cacheData.reset(scd);
QLOG() << "Subplanner: Composite solution is " << soln->toString() << endl;
// We use one of these even if there is one plan. We do this so that the entry is cached
// with stats obtained in the same fashion as a competitive ranking would have obtained
// them.
MultiPlanStage* multiPlanStage = new MultiPlanStage(_collection, _query.get());
WorkingSet* ws = new WorkingSet();
PlanStage* root;
verify(StageBuilder::build(_txn, _collection, *soln, ws, &root));
multiPlanStage->addPlan(soln, root, ws); // Takes ownership first two arguments.
multiPlanStage->pickBestPlan();
if (! multiPlanStage->bestPlanChosen()) {
QLOG() << "Subplanner: Failed to pick best plan for subchild "
<< _query->toString();
return false;
}
Runner* mpr = new SingleSolutionRunner(_collection,
_query.release(),
multiPlanStage->bestSolution(),
multiPlanStage,
ws);
_underlyingRunner.reset(mpr);
return true;
}
Status SubplanStage::choosePlanForSubqueries(PlanYieldPolicy* yieldPolicy) {
// This is the skeleton of index selections that is inserted into the cache.
std::unique_ptr<PlanCacheIndexTree> cacheData(new PlanCacheIndexTree());
for (size_t i = 0; i < _orExpression->numChildren(); ++i) {
MatchExpression* orChild = _orExpression->getChild(i);
BranchPlanningResult* branchResult = _branchResults[i];
if (branchResult->cachedSolution.get()) {
// We can get the index tags we need out of the cache.
Status tagStatus = tagOrChildAccordingToCache(
cacheData.get(), branchResult->cachedSolution->plannerData[0], orChild, _indexMap);
if (!tagStatus.isOK()) {
return tagStatus;
}
} else if (1 == branchResult->solutions.size()) {
QuerySolution* soln = branchResult->solutions.front();
Status tagStatus = tagOrChildAccordingToCache(
cacheData.get(), soln->cacheData.get(), orChild, _indexMap);
if (!tagStatus.isOK()) {
return tagStatus;
}
} else {
// N solutions, rank them.
// We already checked for zero solutions in planSubqueries(...).
invariant(!branchResult->solutions.empty());
_ws->clear();
// We pass the SometimesCache option to the MPS because the SubplanStage currently does
// not use the CachedPlanStage's eviction mechanism. We therefore are more conservative
// about putting a potentially bad plan into the cache in the subplan path.
// We temporarily add the MPS to _children to ensure that we pass down all
// save/restore/invalidate messages that can be generated if pickBestPlan yields.
invariant(_children.empty());
_children.emplace_back(
stdx::make_unique<MultiPlanStage>(getOpCtx(),
_collection,
branchResult->canonicalQuery.get(),
MultiPlanStage::CachingMode::SometimesCache));
ON_BLOCK_EXIT([&] {
invariant(_children.size() == 1); // Make sure nothing else was added to _children.
_children.pop_back();
});
MultiPlanStage* multiPlanStage = static_cast<MultiPlanStage*>(child().get());
// Dump all the solutions into the MPS.
for (size_t ix = 0; ix < branchResult->solutions.size(); ++ix) {
PlanStage* nextPlanRoot;
invariant(StageBuilder::build(getOpCtx(),
_collection,
*branchResult->canonicalQuery,
*branchResult->solutions[ix],
_ws,
&nextPlanRoot));
// Takes ownership of solution with index 'ix' and 'nextPlanRoot'.
multiPlanStage->addPlan(branchResult->solutions.releaseAt(ix), nextPlanRoot, _ws);
}
Status planSelectStat = multiPlanStage->pickBestPlan(yieldPolicy);
if (!planSelectStat.isOK()) {
return planSelectStat;
}
if (!multiPlanStage->bestPlanChosen()) {
mongoutils::str::stream ss;
ss << "Failed to pick best plan for subchild "
<< branchResult->canonicalQuery->toString();
return Status(ErrorCodes::BadValue, ss);
}
QuerySolution* bestSoln = multiPlanStage->bestSolution();
// Check that we have good cache data. For example, we don't cache things
// for 2d indices.
if (NULL == bestSoln->cacheData.get()) {
mongoutils::str::stream ss;
ss << "No cache data for subchild " << orChild->toString();
return Status(ErrorCodes::BadValue, ss);
}
if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) {
mongoutils::str::stream ss;
ss << "No indexed cache data for subchild " << orChild->toString();
return Status(ErrorCodes::BadValue, ss);
}
// Add the index assignments to our original query.
Status tagStatus = QueryPlanner::tagAccordingToCache(
orChild, bestSoln->cacheData->tree.get(), _indexMap);
if (!tagStatus.isOK()) {
mongoutils::str::stream ss;
ss << "Failed to extract indices from subchild " << orChild->toString();
return Status(ErrorCodes::BadValue, ss);
}
cacheData->children.push_back(bestSoln->cacheData->tree->clone());
}
}
// Must do this before using the planner functionality.
sortUsingTags(_orExpression.get());
// Use the cached index assignments to build solnRoot. Takes ownership of '_orExpression'.
QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(
*_query, _orExpression.release(), false, _plannerParams.indices, _plannerParams);
if (NULL == solnRoot) {
mongoutils::str::stream ss;
ss << "Failed to build indexed data path for subplanned query\n";
return Status(ErrorCodes::BadValue, ss);
}
LOG(5) << "Subplanner: fully tagged tree is " << solnRoot->toString();
// Takes ownership of 'solnRoot'
_compositeSolution.reset(
QueryPlannerAnalysis::analyzeDataAccess(*_query, _plannerParams, solnRoot));
if (NULL == _compositeSolution.get()) {
mongoutils::str::stream ss;
ss << "Failed to analyze subplanned query";
return Status(ErrorCodes::BadValue, ss);
}
LOG(5) << "Subplanner: Composite solution is " << _compositeSolution->toString();
// Use the index tags from planning each branch to construct the composite solution,
// and set that solution as our child stage.
_ws->clear();
PlanStage* root;
invariant(StageBuilder::build(
getOpCtx(), _collection, *_query, *_compositeSolution.get(), _ws, &root));
invariant(_children.empty());
_children.emplace_back(root);
return Status::OK();
}
