void SortKeyGenerator::getBoundsForSort(OperationContext* txn, const BSONObj& queryObj, const BSONObj& sortObj) { QueryPlannerParams params; params.options = QueryPlannerParams::NO_TABLE_SCAN; // We're creating a "virtual index" with key pattern equal to the sort order. IndexEntry sortOrder(sortObj, IndexNames::BTREE, true, MultikeyPaths{}, false, false, "doesnt_matter", NULL, BSONObj()); params.indices.push_back(sortOrder); auto statusWithQueryForSort = CanonicalQuery::canonicalize( txn, NamespaceString("fake.ns"), queryObj, ExtensionsCallbackNoop()); verify(statusWithQueryForSort.isOK()); std::unique_ptr<CanonicalQuery> queryForSort = std::move(statusWithQueryForSort.getValue()); std::vector<QuerySolution*> solns; LOG(5) << "Sort key generation: Planning to obtain bounds for sort."; QueryPlanner::plan(*queryForSort, params, &solns); // TODO: are there ever > 1 solns? If so, do we look for a specific soln? if (1 == solns.size()) { IndexScanNode* ixScan = NULL; QuerySolutionNode* rootNode = solns[0]->root.get(); if (rootNode->getType() == STAGE_FETCH) { FetchNode* fetchNode = static_cast<FetchNode*>(rootNode); if (fetchNode->children[0]->getType() != STAGE_IXSCAN) { delete solns[0]; // No bounds. return; } ixScan = static_cast<IndexScanNode*>(fetchNode->children[0]); } else if (rootNode->getType() == STAGE_IXSCAN) { ixScan = static_cast<IndexScanNode*>(rootNode); } if (ixScan) { _bounds.fields.swap(ixScan->bounds.fields); _hasBounds = true; } } for (size_t i = 0; i < solns.size(); ++i) { delete solns[i]; } }
/** * If possible, turn the provided QuerySolution into a QuerySolution that uses a DistinctNode * to provide results for the distinct command. * * If the provided solution could be mutated successfully, returns true, otherwise returns * false. */ bool turnIxscanIntoDistinctIxscan(QuerySolution* soln, const string& field) { QuerySolutionNode* root = soln->root.get(); // We're looking for a project on top of an ixscan. if (STAGE_PROJECTION == root->getType() && (STAGE_IXSCAN == root->children[0]->getType())) { IndexScanNode* isn = static_cast<IndexScanNode*>(root->children[0]); // An additional filter must be applied to the data in the key, so we can't just skip // all the keys with a given value; we must examine every one to find the one that (may) // pass the filter. if (NULL != isn->filter.get()) { return false; } // We only set this when we have special query modifiers (.max() or .min()) or other // special cases. Don't want to handle the interactions between those and distinct. // Don't think this will ever really be true but if it somehow is, just ignore this // soln. if (isn->bounds.isSimpleRange) { return false; } // Make a new DistinctNode. We swap this for the ixscan in the provided solution. DistinctNode* dn = new DistinctNode(); dn->indexKeyPattern = isn->indexKeyPattern; dn->direction = isn->direction; dn->bounds = isn->bounds; // Figure out which field we're skipping to the next value of. TODO: We currently only // try to distinct-hack when there is an index prefixed by the field we're distinct-ing // over. Consider removing this code if we stick with that policy. dn->fieldNo = 0; BSONObjIterator it(isn->indexKeyPattern); while (it.more()) { if (field == it.next().fieldName()) { break; } dn->fieldNo++; } // Delete the old index scan, set the child of project to the fast distinct scan. delete root->children[0]; root->children[0] = dn; return true; } return false; }
void SortStageKeyGenerator::getBoundsForSort(const BSONObj& queryObj, const BSONObj& sortObj) { QueryPlannerParams params; params.options = QueryPlannerParams::NO_TABLE_SCAN; // We're creating a "virtual index" with key pattern equal to the sort order. IndexEntry sortOrder(sortObj, IndexNames::BTREE, true, false, false, "doesnt_matter", BSONObj()); params.indices.push_back(sortOrder); CanonicalQuery* rawQueryForSort; verify(CanonicalQuery::canonicalize( "fake_ns", queryObj, &rawQueryForSort, WhereCallbackNoop()).isOK()); auto_ptr<CanonicalQuery> queryForSort(rawQueryForSort); vector<QuerySolution*> solns; QLOG() << "Sort stage: Planning to obtain bounds for sort." << endl; QueryPlanner::plan(*queryForSort, params, &solns); // TODO: are there ever > 1 solns? If so, do we look for a specific soln? if (1 == solns.size()) { IndexScanNode* ixScan = NULL; QuerySolutionNode* rootNode = solns[0]->root.get(); if (rootNode->getType() == STAGE_FETCH) { FetchNode* fetchNode = static_cast<FetchNode*>(rootNode); if (fetchNode->children[0]->getType() != STAGE_IXSCAN) { delete solns[0]; // No bounds. return; } ixScan = static_cast<IndexScanNode*>(fetchNode->children[0]); } else if (rootNode->getType() == STAGE_IXSCAN) { ixScan = static_cast<IndexScanNode*>(rootNode); } if (ixScan) { _bounds.fields.swap(ixScan->bounds.fields); _hasBounds = true; } } for (size_t i = 0; i < solns.size(); ++i) { delete solns[i]; } }
//static bool StageBuilder::build(const QuerySolution& solution, PlanStage** rootOut, WorkingSet** wsOut) { QuerySolutionNode* root = solution.root.get(); if (NULL == root) { return false; } if (STAGE_COLLSCAN == root->getType()) { const CollectionScanNode* csn = static_cast<const CollectionScanNode*>(root); CollectionScanParams params; params.ns = csn->name; params.tailable = csn->tailable; params.direction = (csn->direction == 1) ? CollectionScanParams::FORWARD : CollectionScanParams::BACKWARD; *wsOut = new WorkingSet(); *rootOut = new CollectionScan(params, *wsOut, csn->filter); return true; } else { return false; } }
Status SubplanStage::choosePlanForSubqueries(PlanYieldPolicy* yieldPolicy) { // This is what we annotate with the index selections and then turn into a solution. auto_ptr<OrMatchExpression> orExpr( static_cast<OrMatchExpression*>(_query->root()->shallowClone())); // This is the skeleton of index selections that is inserted into the cache. auto_ptr<PlanCacheIndexTree> cacheData(new PlanCacheIndexTree()); for (size_t i = 0; i < orExpr->numChildren(); ++i) { MatchExpression* orChild = orExpr->getChild(i); BranchPlanningResult* branchResult = _branchResults[i]; if (branchResult->cachedSolution.get()) { // We can get the index tags we need out of the cache. Status tagStatus = tagOrChildAccordingToCache( cacheData.get(), branchResult->cachedSolution->plannerData[0], orChild, _indexMap); if (!tagStatus.isOK()) { return tagStatus; } } else if (1 == branchResult->solutions.size()) { QuerySolution* soln = branchResult->solutions.front(); Status tagStatus = tagOrChildAccordingToCache(cacheData.get(), soln->cacheData.get(), orChild, _indexMap); if (!tagStatus.isOK()) { return tagStatus; } } else { // N solutions, rank them. // We already checked for zero solutions in planSubqueries(...). invariant(!branchResult->solutions.empty()); _ws->clear(); _child.reset(new MultiPlanStage(_txn, _collection, branchResult->canonicalQuery.get())); MultiPlanStage* multiPlanStage = static_cast<MultiPlanStage*>(_child.get()); // Dump all the solutions into the MPS. for (size_t ix = 0; ix < branchResult->solutions.size(); ++ix) { PlanStage* nextPlanRoot; invariant(StageBuilder::build(_txn, _collection, *branchResult->solutions[ix], _ws, &nextPlanRoot)); // Takes ownership of solution with index 'ix' and 'nextPlanRoot'. multiPlanStage->addPlan(branchResult->solutions.releaseAt(ix), nextPlanRoot, _ws); } Status planSelectStat = multiPlanStage->pickBestPlan(yieldPolicy); if (!planSelectStat.isOK()) { return planSelectStat; } if (!multiPlanStage->bestPlanChosen()) { mongoutils::str::stream ss; ss << "Failed to pick best plan for subchild " << branchResult->canonicalQuery->toString(); return Status(ErrorCodes::BadValue, ss); } QuerySolution* bestSoln = multiPlanStage->bestSolution(); // Check that we have good cache data. For example, we don't cache things // for 2d indices. if (NULL == bestSoln->cacheData.get()) { mongoutils::str::stream ss; ss << "No cache data for subchild " << orChild->toString(); return Status(ErrorCodes::BadValue, ss); } if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) { mongoutils::str::stream ss; ss << "No indexed cache data for subchild " << orChild->toString(); return Status(ErrorCodes::BadValue, ss); } // Add the index assignments to our original query. Status tagStatus = QueryPlanner::tagAccordingToCache( orChild, bestSoln->cacheData->tree.get(), _indexMap); if (!tagStatus.isOK()) { mongoutils::str::stream ss; ss << "Failed to extract indices from subchild " << orChild->toString(); return Status(ErrorCodes::BadValue, ss); } cacheData->children.push_back(bestSoln->cacheData->tree->clone()); } } // Must do this before using the planner functionality. sortUsingTags(orExpr.get()); // Use the cached index assignments to build solnRoot. Takes ownership of 'orExpr'. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess( *_query, orExpr.release(), false, _plannerParams.indices, _plannerParams); if (NULL == solnRoot) { mongoutils::str::stream ss; ss << "Failed to build indexed data path for subplanned query\n"; return Status(ErrorCodes::BadValue, ss); } QLOG() << "Subplanner: fully tagged tree is " << solnRoot->toString(); // Takes ownership of 'solnRoot' _compositeSolution.reset(QueryPlannerAnalysis::analyzeDataAccess(*_query, _plannerParams, solnRoot)); if (NULL == _compositeSolution.get()) { mongoutils::str::stream ss; ss << "Failed to analyze subplanned query"; return Status(ErrorCodes::BadValue, ss); } QLOG() << "Subplanner: Composite solution is " << _compositeSolution->toString() << endl; // Use the index tags from planning each branch to construct the composite solution, // and set that solution as our child stage. _ws->clear(); PlanStage* root; invariant(StageBuilder::build(_txn, _collection, *_compositeSolution.get(), _ws, &root)); _child.reset(root); return Status::OK(); }
bool SubplanRunner::runSubplans() { // This is what we annotate with the index selections and then turn into a solution. auto_ptr<OrMatchExpression> theOr( static_cast<OrMatchExpression*>(_query->root()->shallowClone())); // This is the skeleton of index selections that is inserted into the cache. auto_ptr<PlanCacheIndexTree> cacheData(new PlanCacheIndexTree()); for (size_t i = 0; i < theOr->numChildren(); ++i) { MatchExpression* orChild = theOr->getChild(i); auto_ptr<CanonicalQuery> orChildCQ(_cqs.front()); _cqs.pop(); // 'solutions' is owned by the SubplanRunner instance until // it is popped from the queue. vector<QuerySolution*> solutions = _solutions.front(); _solutions.pop(); // We already checked for zero solutions in planSubqueries(...). invariant(!solutions.empty()); if (1 == solutions.size()) { // There is only one solution. Transfer ownership to an auto_ptr. auto_ptr<QuerySolution> autoSoln(solutions[0]); // We want a well-formed *indexed* solution. if (NULL == autoSoln->cacheData.get()) { // For example, we don't cache things for 2d indices. QLOG() << "Subplanner: No cache data for subchild " << orChild->toString(); return false; } if (SolutionCacheData::USE_INDEX_TAGS_SOLN != autoSoln->cacheData->solnType) { QLOG() << "Subplanner: No indexed cache data for subchild " << orChild->toString(); return false; } // Add the index assignments to our original query. Status tagStatus = QueryPlanner::tagAccordingToCache( orChild, autoSoln->cacheData->tree.get(), _indexMap); if (!tagStatus.isOK()) { QLOG() << "Subplanner: Failed to extract indices from subchild " << orChild->toString(); return false; } // Add the child's cache data to the cache data we're creating for the main query. cacheData->children.push_back(autoSoln->cacheData->tree->clone()); } else { // N solutions, rank them. Takes ownership of orChildCQ. // the working set will be shared by the candidate plans and owned by the runner WorkingSet* sharedWorkingSet = new WorkingSet(); MultiPlanStage* multiPlanStage = new MultiPlanStage(_collection, orChildCQ.get()); // Dump all the solutions into the MPR. for (size_t ix = 0; ix < solutions.size(); ++ix) { PlanStage* nextPlanRoot; verify(StageBuilder::build(_txn, _collection, *solutions[ix], sharedWorkingSet, &nextPlanRoot)); // Owns first two arguments multiPlanStage->addPlan(solutions[ix], nextPlanRoot, sharedWorkingSet); } multiPlanStage->pickBestPlan(); if (! multiPlanStage->bestPlanChosen()) { QLOG() << "Subplanner: Failed to pick best plan for subchild " << orChildCQ->toString(); return false; } Runner* mpr = new SingleSolutionRunner(_collection, orChildCQ.release(), multiPlanStage->bestSolution(), multiPlanStage, sharedWorkingSet); _underlyingRunner.reset(mpr); if (_killed) { QLOG() << "Subplanner: Killed while picking best plan for subchild " << orChild->toString(); return false; } QuerySolution* bestSoln = multiPlanStage->bestSolution(); if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) { QLOG() << "Subplanner: No indexed cache data for subchild " << orChild->toString(); return false; } // Add the index assignments to our original query. Status tagStatus = QueryPlanner::tagAccordingToCache( orChild, bestSoln->cacheData->tree.get(), _indexMap); if (!tagStatus.isOK()) { QLOG() << "Subplanner: Failed to extract indices from subchild " << orChild->toString(); return false; } cacheData->children.push_back(bestSoln->cacheData->tree->clone()); } } // Must do this before using the planner functionality. sortUsingTags(theOr.get()); // Use the cached index assignments to build solnRoot. Takes ownership of 'theOr' QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess( *_query, theOr.release(), false, _plannerParams.indices); if (NULL == solnRoot) { QLOG() << "Subplanner: Failed to build indexed data path for subplanned query\n"; return false; } QLOG() << "Subplanner: fully tagged tree is " << solnRoot->toString(); // Takes ownership of 'solnRoot' QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*_query, _plannerParams, solnRoot); if (NULL == soln) { QLOG() << "Subplanner: Failed to analyze subplanned query"; return false; } // We want our franken-solution to be cached. SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(cacheData.release()); soln->cacheData.reset(scd); QLOG() << "Subplanner: Composite solution is " << soln->toString() << endl; // We use one of these even if there is one plan. We do this so that the entry is cached // with stats obtained in the same fashion as a competitive ranking would have obtained // them. MultiPlanStage* multiPlanStage = new MultiPlanStage(_collection, _query.get()); WorkingSet* ws = new WorkingSet(); PlanStage* root; verify(StageBuilder::build(_txn, _collection, *soln, ws, &root)); multiPlanStage->addPlan(soln, root, ws); // Takes ownership first two arguments. multiPlanStage->pickBestPlan(); if (! multiPlanStage->bestPlanChosen()) { QLOG() << "Subplanner: Failed to pick best plan for subchild " << _query->toString(); return false; } Runner* mpr = new SingleSolutionRunner(_collection, _query.release(), multiPlanStage->bestSolution(), multiPlanStage, ws); _underlyingRunner.reset(mpr); return true; }
Status SubplanStage::choosePlanForSubqueries(PlanYieldPolicy* yieldPolicy) { // This is the skeleton of index selections that is inserted into the cache. std::unique_ptr<PlanCacheIndexTree> cacheData(new PlanCacheIndexTree()); for (size_t i = 0; i < _orExpression->numChildren(); ++i) { MatchExpression* orChild = _orExpression->getChild(i); BranchPlanningResult* branchResult = _branchResults[i]; if (branchResult->cachedSolution.get()) { // We can get the index tags we need out of the cache. Status tagStatus = tagOrChildAccordingToCache( cacheData.get(), branchResult->cachedSolution->plannerData[0], orChild, _indexMap); if (!tagStatus.isOK()) { return tagStatus; } } else if (1 == branchResult->solutions.size()) { QuerySolution* soln = branchResult->solutions.front(); Status tagStatus = tagOrChildAccordingToCache( cacheData.get(), soln->cacheData.get(), orChild, _indexMap); if (!tagStatus.isOK()) { return tagStatus; } } else { // N solutions, rank them. // We already checked for zero solutions in planSubqueries(...). invariant(!branchResult->solutions.empty()); _ws->clear(); // We pass the SometimesCache option to the MPS because the SubplanStage currently does // not use the CachedPlanStage's eviction mechanism. We therefore are more conservative // about putting a potentially bad plan into the cache in the subplan path. // We temporarily add the MPS to _children to ensure that we pass down all // save/restore/invalidate messages that can be generated if pickBestPlan yields. invariant(_children.empty()); _children.emplace_back( stdx::make_unique<MultiPlanStage>(getOpCtx(), _collection, branchResult->canonicalQuery.get(), MultiPlanStage::CachingMode::SometimesCache)); ON_BLOCK_EXIT([&] { invariant(_children.size() == 1); // Make sure nothing else was added to _children. _children.pop_back(); }); MultiPlanStage* multiPlanStage = static_cast<MultiPlanStage*>(child().get()); // Dump all the solutions into the MPS. for (size_t ix = 0; ix < branchResult->solutions.size(); ++ix) { PlanStage* nextPlanRoot; invariant(StageBuilder::build(getOpCtx(), _collection, *branchResult->canonicalQuery, *branchResult->solutions[ix], _ws, &nextPlanRoot)); // Takes ownership of solution with index 'ix' and 'nextPlanRoot'. multiPlanStage->addPlan(branchResult->solutions.releaseAt(ix), nextPlanRoot, _ws); } Status planSelectStat = multiPlanStage->pickBestPlan(yieldPolicy); if (!planSelectStat.isOK()) { return planSelectStat; } if (!multiPlanStage->bestPlanChosen()) { mongoutils::str::stream ss; ss << "Failed to pick best plan for subchild " << branchResult->canonicalQuery->toString(); return Status(ErrorCodes::BadValue, ss); } QuerySolution* bestSoln = multiPlanStage->bestSolution(); // Check that we have good cache data. For example, we don't cache things // for 2d indices. if (NULL == bestSoln->cacheData.get()) { mongoutils::str::stream ss; ss << "No cache data for subchild " << orChild->toString(); return Status(ErrorCodes::BadValue, ss); } if (SolutionCacheData::USE_INDEX_TAGS_SOLN != bestSoln->cacheData->solnType) { mongoutils::str::stream ss; ss << "No indexed cache data for subchild " << orChild->toString(); return Status(ErrorCodes::BadValue, ss); } // Add the index assignments to our original query. Status tagStatus = QueryPlanner::tagAccordingToCache( orChild, bestSoln->cacheData->tree.get(), _indexMap); if (!tagStatus.isOK()) { mongoutils::str::stream ss; ss << "Failed to extract indices from subchild " << orChild->toString(); return Status(ErrorCodes::BadValue, ss); } cacheData->children.push_back(bestSoln->cacheData->tree->clone()); } } // Must do this before using the planner functionality. sortUsingTags(_orExpression.get()); // Use the cached index assignments to build solnRoot. Takes ownership of '_orExpression'. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess( *_query, _orExpression.release(), false, _plannerParams.indices, _plannerParams); if (NULL == solnRoot) { mongoutils::str::stream ss; ss << "Failed to build indexed data path for subplanned query\n"; return Status(ErrorCodes::BadValue, ss); } LOG(5) << "Subplanner: fully tagged tree is " << solnRoot->toString(); // Takes ownership of 'solnRoot' _compositeSolution.reset( QueryPlannerAnalysis::analyzeDataAccess(*_query, _plannerParams, solnRoot)); if (NULL == _compositeSolution.get()) { mongoutils::str::stream ss; ss << "Failed to analyze subplanned query"; return Status(ErrorCodes::BadValue, ss); } LOG(5) << "Subplanner: Composite solution is " << _compositeSolution->toString(); // Use the index tags from planning each branch to construct the composite solution, // and set that solution as our child stage. _ws->clear(); PlanStage* root; invariant(StageBuilder::build( getOpCtx(), _collection, *_query, *_compositeSolution.get(), _ws, &root)); invariant(_children.empty()); _children.emplace_back(root); return Status::OK(); }