// static void QueryPlanner::plan(const CanonicalQuery& query, const QueryPlannerParams& params, vector<QuerySolution*>* out) { QLOG() << "=============================\n" << "Beginning planning, options = " << optionString(params.options) << endl << "Canonical query:\n" << query.toString() << endl << "=============================" << endl; // The shortcut formerly known as IDHACK. See if it's a simple _id query. If so we might // just make an ixscan over the _id index and bypass the rest of planning entirely. if (!query.getParsed().isExplain() && !query.getParsed().showDiskLoc() && isSimpleIdQuery(query.getParsed().getFilter()) && !query.getParsed().hasOption(QueryOption_CursorTailable)) { // See if we can find an _id index. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { const IndexEntry& index = params.indices[i]; QLOG() << "IDHACK using index " << index.toString() << endl; // If so, we make a simple scan to find the doc. IndexScanNode* isn = new IndexScanNode(); isn->indexKeyPattern = index.keyPattern; isn->indexIsMultiKey = index.multikey; isn->direction = 1; isn->bounds.isSimpleRange = true; BSONObj key = getKeyFromQuery(index.keyPattern, query.getParsed().getFilter()); isn->bounds.startKey = isn->bounds.endKey = key; isn->bounds.endKeyInclusive = true; isn->computeProperties(); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, isn); if (NULL != soln) { out->push_back(soln); QLOG() << "IDHACK solution is:\n" << (*out)[0]->toString() << endl; // And that's it. return; } } } } for (size_t i = 0; i < params.indices.size(); ++i) { QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl; } bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().hasOption(QueryOption_CursorTailable)) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { QuerySolution* soln = buildCollscanSoln(query, true, params); if (NULL != soln) { out->push_back(soln); } } return; } // The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of // saying "table scan for two, please." if (!query.getParsed().getHint().isEmpty()) { BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural"); if (!natural.eoo()) { QLOG() << "forcing a table scan due to hinted $natural\n"; if (canTableScan) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return; } } // NOR and NOT we can't handle well with indices. If we see them here, they weren't // rewritten to remove the negation. Just output a collscan for those. if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOT) || QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOR)) { // If there's a near predicate, we can't handle this. // TODO: Should canonicalized query detect this? if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)) { warning() << "Can't handle NOT/NOR with GEO_NEAR"; return; } QLOG() << "NOT/NOR in plan, just outtping a collscan\n"; if (canTableScan) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return; } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { QLOG() << "predicate over field " << *it << endl; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. BSONObj hintIndex = query.getParsed().getHint(); // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real // plan. If that fails, just scan the _id index. if (query.getParsed().isSnapshot()) { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } size_t hintIndexNumber = numeric_limits<size_t>::max(); if (!hintIndex.isEmpty()) { // Sigh. If the hint is specified it might be using the index name. BSONElement firstHintElt = hintIndex.firstElement(); if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { string hintName = firstHintElt.String(); for (size_t i = 0; i < params.indices.size(); ++i) { if (params.indices[i].name == hintName) { QLOG() << "hint by name specified, restricting indices to " << params.indices[i].keyPattern.toString() << endl; relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); hintIndexNumber = i; hintIndex = params.indices[i].keyPattern; break; } } } else { for (size_t i = 0; i < params.indices.size(); ++i) { if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); QLOG() << "hint specified, restricting indices to " << hintIndex.toString() << endl; hintIndexNumber = i; break; } } } if (hintIndexNumber == numeric_limits<size_t>::max()) { // This is supposed to be an error. warning() << "Can't find hint for " << hintIndex.toString(); return; } } else { QLOG() << "Finding relevant indices\n"; QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); } for (size_t i = 0; i < relevantIndices.size(); ++i) { QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl; } // Figure out how useful each index is to each predicate. // query.root() is now annotated with RelevantTag(s). QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); QLOG() << "rated tree" << endl; QLOG() << query.root()->toString() << endl; // If there is a GEO_NEAR it must have an index it can use directly. // XXX: move into data access? MatchExpression* gnNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return; } GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode); vector<size_t> newFirst; // 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed // the full query tree inside it as a matcher. for (size_t i = 0; i < tag->first.size(); ++i) { // GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning. if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) { newFirst.push_back(i); continue; } // If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the // entire tree as a filter, if possible. GeoNear2DNode* solnRoot = new GeoNear2DNode(); solnRoot->nq = gnme->getData(); if (MatchExpression::GEO_NEAR != query.root()->matchType()) { // root is an AND, clone and delete the GEO_NEAR child. MatchExpression* filterTree = query.root()->shallowClone(); verify(MatchExpression::AND == filterTree->matchType()); bool foundChild = false; for (size_t i = 0; i < filterTree->numChildren(); ++i) { if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) { foundChild = true; filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i); break; } } verify(foundChild); solnRoot->filter.reset(filterTree); } solnRoot->numWanted = query.getParsed().getNumToReturn(); if (0 == solnRoot->numWanted) { solnRoot->numWanted = 100; } solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern; // Remove the 2d index. 2d can only be the first field, and we know there is // only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll // only be first for gnNode. tag->first.erase(tag->first.begin() + i); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } } // Continue planning w/non-2d indices tagged for this pred. tag->first.swap(newFirst); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return; } } // Likewise, if there is a TEXT it must have an index it can use directly. MatchExpression* textNode; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return; } } // If we have any relevant indices, we try to create indexed plans. if (0 < relevantIndices.size()) { // The enumerator spits out trees tagged with IndexTag(s). PlanEnumerator isp(query.root(), &relevantIndices); isp.init(); MatchExpression* rawTree; while (isp.getNext(&rawTree)) { QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString() << endl; // This can fail if enumeration makes a mistake. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices); if (NULL == solnRoot) { continue; } QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { QLOG() << "Planner: adding solution:\n" << soln->toString() << endl; out->push_back(soln); } } } QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n"; // An index was hinted. If there are any solutions, they use the hinted index. If not, we // scan the entire index to provide results and output that as our plan. This is the // desired behavior when an index is hinted that is not relevant to the query. if (!hintIndex.isEmpty() && (0 == out->size())) { QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); if (NULL != soln) { QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl; out->push_back(soln); } return; } // If a sort order is requested, there may be an index that provides it, even if that // index is not over any predicates in the query. // // XXX XXX: Can we do this even if the index is sparse? Might we miss things? if (!query.getParsed().getSort().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { // See if we have a sort provided from an index already. bool usingIndexToSort = false; for (size_t i = 0; i < out->size(); ++i) { QuerySolution* soln = (*out)[i]; if (!soln->hasSortStage) { usingIndexToSort = true; break; } } if (!usingIndexToSort) { for (size_t i = 0; i < params.indices.size(); ++i) { const BSONObj& kp = params.indices[i].keyPattern; if (providesSort(query, kp)) { QLOG() << "Planner: outputting soln that uses index to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); if (NULL != soln) { out->push_back(soln); break; } } if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { QLOG() << "Planner: outputting soln that uses (reverse) index " << "to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); if (NULL != soln) { out->push_back(soln); break; } } } } } // TODO: Do we always want to offer a collscan solution? // XXX: currently disabling the always-use-a-collscan in order to find more planner bugs. if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan))) { QuerySolution* collscan = buildCollscanSoln(query, false, params); if (NULL != collscan) { out->push_back(collscan); QLOG() << "Planner: outputting a collscan:\n"; QLOG() << collscan->toString() << endl; } } }
// static Status QueryPlanner::plan(const CanonicalQuery& query, const QueryPlannerParams& params, std::vector<QuerySolution*>* out) { QLOG() << "=============================\n" << "Beginning planning, options = " << optionString(params.options) << endl << "Canonical query:\n" << query.toString() << endl << "=============================" << endl; for (size_t i = 0; i < params.indices.size(); ++i) { QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl; } bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().hasOption(QueryOption_CursorTailable)) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { QuerySolution* soln = buildCollscanSoln(query, true, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } // The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of // saying "table scan for two, please." if (!query.getParsed().getHint().isEmpty()) { BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural"); if (!natural.eoo()) { QLOG() << "forcing a table scan due to hinted $natural\n"; // min/max are incompatible with $natural. if (canTableScan && query.getParsed().getMin().isEmpty() && query.getParsed().getMax().isEmpty()) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { QLOG() << "predicate over field " << *it << endl; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. BSONObj hintIndex = query.getParsed().getHint(); // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real // plan. If that fails, just scan the _id index. if (query.getParsed().isSnapshot()) { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } size_t hintIndexNumber = numeric_limits<size_t>::max(); if (hintIndex.isEmpty()) { QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); } else { // Sigh. If the hint is specified it might be using the index name. BSONElement firstHintElt = hintIndex.firstElement(); if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { string hintName = firstHintElt.String(); for (size_t i = 0; i < params.indices.size(); ++i) { if (params.indices[i].name == hintName) { QLOG() << "hint by name specified, restricting indices to " << params.indices[i].keyPattern.toString() << endl; relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); hintIndexNumber = i; hintIndex = params.indices[i].keyPattern; break; } } } else { for (size_t i = 0; i < params.indices.size(); ++i) { if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); QLOG() << "hint specified, restricting indices to " << hintIndex.toString() << endl; hintIndexNumber = i; break; } } } if (hintIndexNumber == numeric_limits<size_t>::max()) { return Status(ErrorCodes::BadValue, "bad hint"); } } // Deal with the .min() and .max() query options. If either exist we can only use an index // that matches the object inside. if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) { BSONObj minObj = query.getParsed().getMin(); BSONObj maxObj = query.getParsed().getMax(); // This is the index into params.indices[...] that we use. size_t idxNo = numeric_limits<size_t>::max(); // If there's an index hinted we need to be able to use it. if (!hintIndex.isEmpty()) { if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) { QLOG() << "minobj doesnt work w hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with min query"); } if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) { QLOG() << "maxobj doesnt work w hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with max query"); } idxNo = hintIndexNumber; } else { // No hinted index, look for one that is compatible (has same field names and // ordering thereof). for (size_t i = 0; i < params.indices.size(); ++i) { const BSONObj& kp = params.indices[i].keyPattern; BSONObj toUse = minObj.isEmpty() ? maxObj : minObj; if (indexCompatibleMaxMin(toUse, kp)) { idxNo = i; break; } } } if (idxNo == numeric_limits<size_t>::max()) { QLOG() << "Can't find relevant index to use for max/min query"; // Can't find an index to use, bail out. return Status(ErrorCodes::BadValue, "unable to find relevant index for max/min query"); } // maxObj can be empty; the index scan just goes until the end. minObj can't be empty // though, so if it is, we make a minKey object. if (minObj.isEmpty()) { BSONObjBuilder bob; bob.appendMinKey(""); minObj = bob.obj(); } else { // Must strip off the field names to make an index key. minObj = stripFieldNames(minObj); } if (!maxObj.isEmpty()) { // Must strip off the field names to make an index key. maxObj = stripFieldNames(maxObj); } QLOG() << "max/min query using index " << params.indices[idxNo].toString() << endl; // Make our scan and output. QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(params.indices[idxNo], query, params, minObj, maxObj); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } return Status::OK(); } for (size_t i = 0; i < relevantIndices.size(); ++i) { QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl; } // Figure out how useful each index is to each predicate. // query.root() is now annotated with RelevantTag(s). QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); QLOG() << "rated tree" << endl; QLOG() << query.root()->toString() << endl; // If there is a GEO_NEAR it must have an index it can use directly. // XXX: move into data access? MatchExpression* gnNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { QLOG() << "unable to find index for $geoNear query" << endl; return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query"); } GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode); vector<size_t> newFirst; // 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed // the full query tree inside it as a matcher. for (size_t i = 0; i < tag->first.size(); ++i) { // GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning. if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) { newFirst.push_back(i); continue; } // If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the // entire tree as a filter, if possible. GeoNear2DNode* solnRoot = new GeoNear2DNode(); solnRoot->nq = gnme->getData(); if (NULL != query.getProj()) { solnRoot->addPointMeta = query.getProj()->wantGeoNearPoint(); solnRoot->addDistMeta = query.getProj()->wantGeoNearDistance(); } if (MatchExpression::GEO_NEAR != query.root()->matchType()) { // root is an AND, clone and delete the GEO_NEAR child. MatchExpression* filterTree = query.root()->shallowClone(); verify(MatchExpression::AND == filterTree->matchType()); bool foundChild = false; for (size_t i = 0; i < filterTree->numChildren(); ++i) { if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) { foundChild = true; filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i); break; } } verify(foundChild); solnRoot->filter.reset(filterTree); } solnRoot->numWanted = query.getParsed().getNumToReturn(); if (0 == solnRoot->numWanted) { solnRoot->numWanted = 100; } solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern; // Remove the 2d index. 2d can only be the first field, and we know there is // only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll // only be first for gnNode. tag->first.erase(tag->first.begin() + i); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } } // Continue planning w/non-2d indices tagged for this pred. tag->first.swap(newFirst); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return Status::OK(); } } // Likewise, if there is a TEXT it must have an index it can use directly. MatchExpression* textNode; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return Status::OK(); } } // If we have any relevant indices, we try to create indexed plans. if (0 < relevantIndices.size()) { // The enumerator spits out trees tagged with IndexTag(s). PlanEnumeratorParams enumParams; enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION; enumParams.root = query.root(); enumParams.indices = &relevantIndices; PlanEnumerator isp(enumParams); isp.init(); MatchExpression* rawTree; // XXX: have limit on # of indexed solns we'll consider. We could have a perverse // query and index that could make n^2 very unpleasant. while (isp.getNext(&rawTree)) { QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString() << endl; // This can fail if enumeration makes a mistake. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices); if (NULL == solnRoot) { continue; } QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { QLOG() << "Planner: adding solution:\n" << soln->toString() << endl; out->push_back(soln); } } } QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n"; // An index was hinted. If there are any solutions, they use the hinted index. If not, we // scan the entire index to provide results and output that as our plan. This is the // desired behavior when an index is hinted that is not relevant to the query. if (!hintIndex.isEmpty()) { if (0 == out->size()) { QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); verify(NULL != soln); QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl; out->push_back(soln); } return Status::OK(); } // If a sort order is requested, there may be an index that provides it, even if that // index is not over any predicates in the query. // if (!query.getParsed().getSort().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { // See if we have a sort provided from an index already. bool usingIndexToSort = false; for (size_t i = 0; i < out->size(); ++i) { QuerySolution* soln = (*out)[i]; if (!soln->hasSortStage) { usingIndexToSort = true; break; } } if (!usingIndexToSort) { for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& index = params.indices[i]; if (index.sparse) { continue; } const BSONObj kp = LiteParsedQuery::normalizeSortOrder(index.keyPattern); if (providesSort(query, kp)) { QLOG() << "Planner: outputting soln that uses index to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); if (NULL != soln) { out->push_back(soln); break; } } if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { QLOG() << "Planner: outputting soln that uses (reverse) index " << "to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); if (NULL != soln) { out->push_back(soln); break; } } } } } // TODO: Do we always want to offer a collscan solution? // XXX: currently disabling the always-use-a-collscan in order to find more planner bugs. if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && hintIndex.isEmpty() && ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan))) { QuerySolution* collscan = buildCollscanSoln(query, false, params); if (NULL != collscan) { out->push_back(collscan); QLOG() << "Planner: outputting a collscan:\n"; QLOG() << collscan->toString() << endl; } } return Status::OK(); }
// static Status QueryPlanner::plan(const CanonicalQuery& query, const QueryPlannerParams& params, std::vector<QuerySolution*>* out) { QLOG() << "Beginning planning..." << endl << "=============================" << endl << "Options = " << optionString(params.options) << endl << "Canonical query:" << endl << query.toString() << "=============================" << endl; for (size_t i = 0; i < params.indices.size(); ++i) { QLOG() << "Index " << i << " is " << params.indices[i].toString() << endl; } bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().hasOption(QueryOption_CursorTailable)) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { QuerySolution* soln = buildCollscanSoln(query, true, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } // The hint or sort can be $natural: 1. If this happens, output a collscan. If both // a $natural hint and a $natural sort are specified, then the direction of the collscan // is determined by the sign of the sort (not the sign of the hint). if (!query.getParsed().getHint().isEmpty() || !query.getParsed().getSort().isEmpty()) { BSONObj hintObj = query.getParsed().getHint(); BSONObj sortObj = query.getParsed().getSort(); BSONElement naturalHint = hintObj.getFieldDotted("$natural"); BSONElement naturalSort = sortObj.getFieldDotted("$natural"); // A hint overrides a $natural sort. This means that we don't force a table // scan if there is a $natural sort with a non-$natural hint. if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) { QLOG() << "Forcing a table scan due to hinted $natural\n"; // min/max are incompatible with $natural. if (canTableScan && query.getParsed().getMin().isEmpty() && query.getParsed().getMax().isEmpty()) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { QLOG() << "Predicate over field '" << *it << "'" << endl; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. // If index filters in the query settings were used to override // the allowed indices for planning, we should not use the hinted index // requested in the query. BSONObj hintIndex; if (!params.indexFiltersApplied) { hintIndex = query.getParsed().getHint(); } // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real // plan. If that fails, just scan the _id index. if (query.getParsed().isSnapshot()) { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } size_t hintIndexNumber = numeric_limits<size_t>::max(); if (hintIndex.isEmpty()) { QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); } else { // Sigh. If the hint is specified it might be using the index name. BSONElement firstHintElt = hintIndex.firstElement(); if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { string hintName = firstHintElt.String(); for (size_t i = 0; i < params.indices.size(); ++i) { if (params.indices[i].name == hintName) { QLOG() << "Hint by name specified, restricting indices to " << params.indices[i].keyPattern.toString() << endl; relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); hintIndexNumber = i; hintIndex = params.indices[i].keyPattern; break; } } } else { for (size_t i = 0; i < params.indices.size(); ++i) { if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); QLOG() << "Hint specified, restricting indices to " << hintIndex.toString() << endl; hintIndexNumber = i; break; } } } if (hintIndexNumber == numeric_limits<size_t>::max()) { return Status(ErrorCodes::BadValue, "bad hint"); } } // Deal with the .min() and .max() query options. If either exist we can only use an index // that matches the object inside. if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) { BSONObj minObj = query.getParsed().getMin(); BSONObj maxObj = query.getParsed().getMax(); // This is the index into params.indices[...] that we use. size_t idxNo = numeric_limits<size_t>::max(); // If there's an index hinted we need to be able to use it. if (!hintIndex.isEmpty()) { if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) { QLOG() << "Minobj doesn't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with min query"); } if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) { QLOG() << "Maxobj doesn't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with max query"); } idxNo = hintIndexNumber; } else { // No hinted index, look for one that is compatible (has same field names and // ordering thereof). for (size_t i = 0; i < params.indices.size(); ++i) { const BSONObj& kp = params.indices[i].keyPattern; BSONObj toUse = minObj.isEmpty() ? maxObj : minObj; if (indexCompatibleMaxMin(toUse, kp)) { idxNo = i; break; } } } if (idxNo == numeric_limits<size_t>::max()) { QLOG() << "Can't find relevant index to use for max/min query"; // Can't find an index to use, bail out. return Status(ErrorCodes::BadValue, "unable to find relevant index for max/min query"); } // maxObj can be empty; the index scan just goes until the end. minObj can't be empty // though, so if it is, we make a minKey object. if (minObj.isEmpty()) { BSONObjBuilder bob; bob.appendMinKey(""); minObj = bob.obj(); } else { // Must strip off the field names to make an index key. minObj = stripFieldNames(minObj); } if (!maxObj.isEmpty()) { // Must strip off the field names to make an index key. maxObj = stripFieldNames(maxObj); } QLOG() << "Max/min query using index " << params.indices[idxNo].toString() << endl; // Make our scan and output. QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(params.indices[idxNo], query, params, minObj, maxObj); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } return Status::OK(); } for (size_t i = 0; i < relevantIndices.size(); ++i) { QLOG() << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl; LOG(2) << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl; } // Figure out how useful each index is to each predicate. QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); QueryPlannerIXSelect::stripInvalidAssignments(query.root(), relevantIndices); // query.root() is now annotated with RelevantTag(s). QLOG() << "Rated tree:" << endl << query.root()->toString(); // If there is a GEO_NEAR it must have an index it can use directly. MatchExpression* gnNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { QLOG() << "Unable to find index for $geoNear query." << endl; // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query"); } QLOG() << "Rated tree after geonear processing:" << query.root()->toString(); } // Likewise, if there is a TEXT it must have an index it can use directly. MatchExpression* textNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag()); // Exactly one text index required for TEXT. We need to check this explicitly because // the text stage can't be built if no text index exists or there is an ambiguity as to // which one to use. size_t textIndexCount = 0; for (size_t i = 0; i < params.indices.size(); i++) { if (INDEX_TEXT == params.indices[i].type) { textIndexCount++; } } if (textIndexCount != 1) { // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "need exactly one text index for $text query"); } // Error if the text node is tagged with zero indices. if (0 == tag->first.size() && 0 == tag->notFirst.size()) { // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "failed to use text index to satisfy $text query (if text index is " "compound, are equality predicates given for all prefix fields?)"); } // At this point, we know that there is only one text index and that the TEXT node is // assigned to it. invariant(1 == tag->first.size() + tag->notFirst.size()); QLOG() << "Rated tree after text processing:" << query.root()->toString(); } // If we have any relevant indices, we try to create indexed plans. if (0 < relevantIndices.size()) { // The enumerator spits out trees tagged with IndexTag(s). PlanEnumeratorParams enumParams; enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION; enumParams.root = query.root(); enumParams.indices = &relevantIndices; PlanEnumerator isp(enumParams); isp.init(); MatchExpression* rawTree; while (isp.getNext(&rawTree) && (out->size() < params.maxIndexedSolutions)) { QLOG() << "About to build solntree from tagged tree:" << endl << rawTree->toString(); // The tagged tree produced by the plan enumerator is not guaranteed // to be canonically sorted. In order to be compatible with the cached // data, sort the tagged tree according to CanonicalQuery ordering. boost::scoped_ptr<MatchExpression> clone(rawTree->shallowClone()); CanonicalQuery::sortTree(clone.get()); PlanCacheIndexTree* cacheData; Status indexTreeStatus = cacheDataFromTaggedTree(clone.get(), relevantIndices, &cacheData); if (!indexTreeStatus.isOK()) { QLOG() << "Query is not cachable: " << indexTreeStatus.reason() << endl; } auto_ptr<PlanCacheIndexTree> autoData(cacheData); // This can fail if enumeration makes a mistake. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices); if (NULL == solnRoot) { continue; } QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { QLOG() << "Planner: adding solution:" << endl << soln->toString(); if (indexTreeStatus.isOK()) { SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(autoData.release()); soln->cacheData.reset(scd); } out->push_back(soln); } } } // Don't leave tags on query tree. query.root()->resetTag(); QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n"; // Produce legible error message for failed OR planning with a TEXT child. // TODO: support collection scan for non-TEXT children of OR. if (out->size() == 0 && textNode != NULL && MatchExpression::OR == query.root()->matchType()) { MatchExpression* root = query.root(); for (size_t i = 0; i < root->numChildren(); ++i) { if (textNode == root->getChild(i)) { return Status(ErrorCodes::BadValue, "Failed to produce a solution for TEXT under OR - " "other non-TEXT clauses under OR have to be indexed as well."); } } } // An index was hinted. If there are any solutions, they use the hinted index. If not, we // scan the entire index to provide results and output that as our plan. This is the // desired behavior when an index is hinted that is not relevant to the query. if (!hintIndex.isEmpty()) { if (0 == out->size()) { QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); verify(NULL != soln); QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl; out->push_back(soln); } return Status::OK(); } // If a sort order is requested, there may be an index that provides it, even if that // index is not over any predicates in the query. // if (!query.getParsed().getSort().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { // See if we have a sort provided from an index already. // This is implied by the presence of a non-blocking solution. bool usingIndexToSort = false; for (size_t i = 0; i < out->size(); ++i) { QuerySolution* soln = (*out)[i]; if (!soln->hasBlockingStage) { usingIndexToSort = true; break; } } if (!usingIndexToSort) { for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& index = params.indices[i]; // Only regular (non-plugin) indexes can be used to provide a sort. if (index.type != INDEX_BTREE) { continue; } // Only non-sparse indexes can be used to provide a sort. if (index.sparse) { continue; } // TODO: Sparse indexes can't normally provide a sort, because non-indexed // documents could potentially be missing from the result set. However, if the // query predicate can be used to guarantee that all documents to be returned // are indexed, then the index should be able to provide the sort. // // For example: // - Sparse index {a: 1, b: 1} should be able to provide a sort for // find({b: 1}).sort({a: 1}). SERVER-13908. // - Index {a: 1, b: "2dsphere"} (which is "geo-sparse", if // 2dsphereIndexVersion=2) should be able to provide a sort for // find({b: GEO}).sort({a:1}). SERVER-10801. const BSONObj kp = LiteParsedQuery::normalizeSortOrder(index.keyPattern); if (providesSort(query, kp)) { QLOG() << "Planner: outputting soln that uses index to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); if (NULL != soln) { PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); indexTree->setIndexEntry(params.indices[i]); SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(indexTree); scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; scd->wholeIXSolnDir = 1; soln->cacheData.reset(scd); out->push_back(soln); break; } } if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { QLOG() << "Planner: outputting soln that uses (reverse) index " << "to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); if (NULL != soln) { PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); indexTree->setIndexEntry(params.indices[i]); SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(indexTree); scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; scd->wholeIXSolnDir = -1; soln->cacheData.reset(scd); out->push_back(soln); break; } } } } } // geoNear and text queries *require* an index. // Also, if a hint is specified it indicates that we MUST use it. bool possibleToCollscan = !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && hintIndex.isEmpty(); // The caller can explicitly ask for a collscan. bool collscanRequested = (params.options & QueryPlannerParams::INCLUDE_COLLSCAN); // No indexed plans? We must provide a collscan if possible or else we can't run the query. bool collscanNeeded = (0 == out->size() && canTableScan); if (possibleToCollscan && (collscanRequested || collscanNeeded)) { QuerySolution* collscan = buildCollscanSoln(query, false, params); if (NULL != collscan) { SolutionCacheData* scd = new SolutionCacheData(); scd->solnType = SolutionCacheData::COLLSCAN_SOLN; collscan->cacheData.reset(scd); out->push_back(collscan); QLOG() << "Planner: outputting a collscan:" << endl << collscan->toString(); } } return Status::OK(); }
StatusWith<std::unique_ptr<QuerySolution>> QueryPlanner::planFromCache( const CanonicalQuery& query, const QueryPlannerParams& params, const CachedSolution& cachedSoln) { invariant(!cachedSoln.plannerData.empty()); // A query not suitable for caching should not have made its way into the cache. invariant(PlanCache::shouldCacheQuery(query)); // Look up winning solution in cached solution's array. const SolutionCacheData& winnerCacheData = *cachedSoln.plannerData[0]; if (SolutionCacheData::WHOLE_IXSCAN_SOLN == winnerCacheData.solnType) { // The solution can be constructed by a scan over the entire index. auto soln = buildWholeIXSoln( *winnerCacheData.tree->entry, query, params, winnerCacheData.wholeIXSolnDir); if (!soln) { return Status(ErrorCodes::BadValue, "plan cache error: soln that uses index to provide sort"); } else { return {std::move(soln)}; } } else if (SolutionCacheData::COLLSCAN_SOLN == winnerCacheData.solnType) { // The cached solution is a collection scan. We don't cache collscans // with tailable==true, hence the false below. auto soln = buildCollscanSoln(query, false, params); if (!soln) { return Status(ErrorCodes::BadValue, "plan cache error: collection scan soln"); } else { return {std::move(soln)}; } } // SolutionCacheData::USE_TAGS_SOLN == cacheData->solnType // If we're here then this is neither the whole index scan or collection scan // cases, and we proceed by using the PlanCacheIndexTree to tag the query tree. // Create a copy of the expression tree. We use cachedSoln to annotate this with indices. unique_ptr<MatchExpression> clone = query.root()->shallowClone(); LOG(5) << "Tagging the match expression according to cache data: " << endl << "Filter:" << endl << redact(clone->toString()) << "Cache data:" << endl << redact(winnerCacheData.toString()); // Map from index name to index number. // TODO: can we assume that the index numbering has the same lifetime // as the cache state? map<StringData, size_t> indexMap; for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& ie = params.indices[i]; indexMap[ie.name] = i; LOG(5) << "Index " << i << ": " << ie.name; } Status s = tagAccordingToCache(clone.get(), winnerCacheData.tree.get(), indexMap); if (!s.isOK()) { return s; } // The MatchExpression tree is in canonical order. We must order the nodes for access planning. prepareForAccessPlanning(clone.get()); LOG(5) << "Tagged tree:" << endl << redact(clone->toString()); // Use the cached index assignments to build solnRoot. std::unique_ptr<QuerySolutionNode> solnRoot(QueryPlannerAccess::buildIndexedDataAccess( query, clone.release(), false, params.indices, params)); if (!solnRoot) { return Status(ErrorCodes::BadValue, str::stream() << "Failed to create data access plan from cache. Query: " << query.toStringShort()); } auto soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, std::move(solnRoot)); if (!soln) { return Status(ErrorCodes::BadValue, str::stream() << "Failed to analyze plan from cache. Query: " << query.toStringShort()); } LOG(5) << "Planner: solution constructed from the cache:\n" << redact(soln->toString()); return {std::move(soln)}; }
// static Status QueryPlanner::planFromCache(const CanonicalQuery& query, const QueryPlannerParams& params, const SolutionCacheData& cacheData, QuerySolution** out) { if (SolutionCacheData::WHOLE_IXSCAN_SOLN == cacheData.solnType) { // The solution can be constructed by a scan over the entire index. QuerySolution* soln = buildWholeIXSoln(*cacheData.tree->entry, query, params, cacheData.wholeIXSolnDir); if (soln == NULL) { return Status(ErrorCodes::BadValue, "plan cache error: soln that uses index to provide sort"); } else { *out = soln; return Status::OK(); } } else if (SolutionCacheData::COLLSCAN_SOLN == cacheData.solnType) { // The cached solution is a collection scan. We don't cache collscans // with tailable==true, hence the false below. QuerySolution* soln = buildCollscanSoln(query, false, params); if (soln == NULL) { return Status(ErrorCodes::BadValue, "plan cache error: collection scan soln"); } else { *out = soln; return Status::OK(); } } // SolutionCacheData::USE_TAGS_SOLN == cacheData->solnType // If we're here then this is neither the whole index scan or collection scan // cases, and we proceed by using the PlanCacheIndexTree to tag the query tree. // Create a copy of the expression tree. We use cachedSoln to annotate this with indices. MatchExpression* clone = query.root()->shallowClone(); QLOG() << "Tagging the match expression according to cache data: " << endl << "Filter:" << endl << clone->toString() << "Cache data:" << endl << cacheData.toString(); // Map from index name to index number. // TODO: can we assume that the index numbering has the same lifetime // as the cache state? map<BSONObj, size_t> indexMap; for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& ie = params.indices[i]; indexMap[ie.keyPattern] = i; QLOG() << "Index " << i << ": " << ie.keyPattern.toString() << endl; } Status s = tagAccordingToCache(clone, cacheData.tree.get(), indexMap); if (!s.isOK()) { return s; } // The planner requires a defined sort order. sortUsingTags(clone); QLOG() << "Tagged tree:" << endl << clone->toString(); // Use the cached index assignments to build solnRoot. Takes ownership of clone. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, clone, false, params.indices); if (NULL != solnRoot) { // Takes ownership of 'solnRoot'. QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { QLOG() << "Planner: solution constructed from the cache:\n" << soln->toString() << endl; *out = soln; return Status::OK(); } } return Status(ErrorCodes::BadValue, "couldn't plan from cache"); }