Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query, const vector<FieldRef*>* immutablePathsPtr, mutablebson::Document& doc) const { EqualityMatches equalities; Status status = Status::OK(); if (isDocReplacement()) { FieldRefSet pathsToExtract; // TODO: Refactor update logic, make _id just another immutable field static const FieldRef idPath("_id"); static const vector<FieldRef*> emptyImmutablePaths; const vector<FieldRef*>& immutablePaths = immutablePathsPtr ? *immutablePathsPtr : emptyImmutablePaths; pathsToExtract.fillFrom(immutablePaths); pathsToExtract.insert(&idPath); // Extract only immutable fields from replacement-style status = pathsupport::extractFullEqualityMatches(*query.root(), pathsToExtract, &equalities); } else { // Extract all fields from op-style status = pathsupport::extractEqualityMatches(*query.root(), &equalities); } if (!status.isOK()) return status; status = pathsupport::addEqualitiesToDoc(equalities, &doc); return status; }
bool SubplanStage::canUseSubplanning(const CanonicalQuery& query) { const QueryRequest& qr = query.getQueryRequest(); const MatchExpression* expr = query.root(); // Hint provided if (!qr.getHint().isEmpty()) { return false; } // Min provided // Min queries are a special case of hinted queries. if (!qr.getMin().isEmpty()) { return false; } // Max provided // Similar to min, max queries are a special case of hinted queries. if (!qr.getMax().isEmpty()) { return false; } // Tailable cursors won't get cached, just turn into collscans. if (query.getQueryRequest().isTailable()) { return false; } // We can only subplan rooted $or queries, and only if they have at least one clause. return MatchExpression::OR == expr->matchType() && expr->numChildren() > 0; }
bool PlanCache::shouldCacheQuery(const CanonicalQuery& query) { const LiteParsedQuery& lpq = query.getParsed(); const MatchExpression* expr = query.root(); // Collection scan // No sort order requested if (lpq.getSort().isEmpty() && expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) { return false; } // Hint provided if (!lpq.getHint().isEmpty()) { return false; } // Min provided // Min queries are a special case of hinted queries. if (!lpq.getMin().isEmpty()) { return false; } // Max provided // Similar to min, max queries are a special case of hinted queries. if (!lpq.getMax().isEmpty()) { return false; } return true; }
bool SubplanStage::canUseSubplanning(const CanonicalQuery& query) { const LiteParsedQuery& lpq = query.getParsed(); const MatchExpression* expr = query.root(); // Hint provided if (!lpq.getHint().isEmpty()) { return false; } // Min provided // Min queries are a special case of hinted queries. if (!lpq.getMin().isEmpty()) { return false; } // Max provided // Similar to min, max queries are a special case of hinted queries. if (!lpq.getMax().isEmpty()) { return false; } // Tailable cursors won't get cached, just turn into collscans. if (query.getParsed().isTailable()) { return false; } // Snapshot is really a hint. if (query.getParsed().isSnapshot()) { return false; } // TODO: For now we only allow rooted OR. We should consider also allowing contained OR that // does not have a TEXT or GEO_NEAR node. return MatchExpression::OR == expr->matchType(); }
// static Status QueryPlanner::planFromCache(const CanonicalQuery& query, const QueryPlannerParams& params, CachedSolution* cachedSoln, QuerySolution** out) { // Create a copy of the expression tree. We use cachedSoln to annotate this with indices. MatchExpression* clone = query.root()->shallowClone(); // XXX: Use data in cachedSoln to tag 'clone' with the indices used. The tags use an index // ID which is an index into some vector of IndexEntry(s). How do we maintain this across // calls to plan? Do we want to store in the soln the keypatterns of the indices and just // map those to an index into params.indices? Might be easiest thing to do, and certainly // most intelligible for debugging. // Use the cached index assignments to build solnRoot. Takes ownership of clone. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, clone, false, params.indices); // XXX: are the NULL cases an error/when does this happen / can this happen? if (NULL != solnRoot) { QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { QLOG() << "Planner: adding cached solution:\n" << soln->toString() << endl; *out = soln; } } // XXX: if any NULLs return error status? return Status::OK(); }
PlanCacheKey PlanCache::computeKey(const CanonicalQuery& cq) const { str::stream ss; encodePlanCacheKeyTree(cq.root(), &ss); encodePlanCacheKeySort(cq.getParsed().getSort(), &ss); encodePlanCacheKeyProj(cq.getParsed().getProj(), &ss); return ss; }
PlanCacheKey PlanCache::computeKey(const CanonicalQuery& cq) const { StringBuilder keyBuilder; encodeKeyForMatch(cq.root(), &keyBuilder); encodeKeyForSort(cq.getQueryRequest().getSort(), &keyBuilder); encodeKeyForProj(cq.getQueryRequest().getProj(), &keyBuilder); return keyBuilder.str(); }
/** * Cache key is a string-ified combination of the query and sort obfuscated * for minimal user comprehension. */ PlanCacheKey PlanCache::getPlanCacheKey(const CanonicalQuery& query) { stringstream ss; encodePlanCacheKeyTree(query.root(), &ss); encodePlanCacheKeySort(query.getParsed().getSort(), &ss); encodePlanCacheKeyProj(query.getParsed().getProj(), &ss); PlanCacheKey key(ss.str()); return key; }
Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query, const FieldRefSet& immutablePaths, mutablebson::Document& doc) const { EqualityMatches equalities; Status status = Status::OK(); if (_updateType == UpdateType::kReplacement) { // Extract only immutable fields. status = pathsupport::extractFullEqualityMatches(*query.root(), immutablePaths, &equalities); } else { // Extract all fields from op-style update. status = pathsupport::extractEqualityMatches(*query.root(), &equalities); } if (!status.isOK()) return status; status = pathsupport::addEqualitiesToDoc(equalities, &doc); return status; }
// static void Explain::generatePlannerInfo(PlanExecutor* exec, PlanStageStats* winnerStats, const vector<PlanStageStats*>& rejectedStats, BSONObjBuilder* out) { CanonicalQuery* query = exec->getCanonicalQuery(); BSONObjBuilder plannerBob(out->subobjStart("queryPlanner")); ; plannerBob.append("plannerVersion", QueryPlanner::kPlannerVersion); plannerBob.append("namespace", exec->ns()); // Find whether there is an index filter set for the query shape. The 'indexFilterSet' // field will always be false in the case of EOF or idhack plans. bool indexFilterSet = false; if (exec->collection() && exec->getCanonicalQuery()) { const CollectionInfoCache* infoCache = exec->collection()->infoCache(); const QuerySettings* querySettings = infoCache->getQuerySettings(); PlanCacheKey planCacheKey = infoCache->getPlanCache()->computeKey(*exec->getCanonicalQuery()); AllowedIndices* allowedIndicesRaw; if (querySettings->getAllowedIndices(planCacheKey, &allowedIndicesRaw)) { // Found an index filter set on the query shape. std::unique_ptr<AllowedIndices> allowedIndices(allowedIndicesRaw); indexFilterSet = true; } } plannerBob.append("indexFilterSet", indexFilterSet); // In general we should have a canonical query, but sometimes we may avoid // creating a canonical query as an optimization (specifically, the update system // does not canonicalize for idhack updates). In these cases, 'query' is NULL. if (NULL != query) { BSONObjBuilder parsedQueryBob(plannerBob.subobjStart("parsedQuery")); query->root()->toBSON(&parsedQueryBob); parsedQueryBob.doneFast(); } BSONObjBuilder winningPlanBob(plannerBob.subobjStart("winningPlan")); statsToBSON(*winnerStats, &winningPlanBob, ExplainCommon::QUERY_PLANNER); winningPlanBob.doneFast(); // Genenerate array of rejected plans. BSONArrayBuilder allPlansBob(plannerBob.subarrayStart("rejectedPlans")); for (size_t i = 0; i < rejectedStats.size(); i++) { BSONObjBuilder childBob(allPlansBob.subobjStart()); statsToBSON(*rejectedStats[i], &childBob, ExplainCommon::QUERY_PLANNER); } allPlansBob.doneFast(); plannerBob.doneFast(); }
void ChunkManager::getShardIdsForQuery(set<ShardId>& shardIds, const BSONObj& query) const { CanonicalQuery* canonicalQuery = NULL; Status status = CanonicalQuery::canonicalize( _ns, query, &canonicalQuery, WhereCallbackNoop()); boost::scoped_ptr<CanonicalQuery> canonicalQueryPtr(canonicalQuery); uassert(status.code(), status.reason(), status.isOK()); // Query validation if (QueryPlannerCommon::hasNode(canonicalQuery->root(), MatchExpression::GEO_NEAR)) { uassert(13501, "use geoNear command rather than $near query", false); } // Transforms query into bounds for each field in the shard key // for example : // Key { a: 1, b: 1 }, // Query { a : { $gte : 1, $lt : 2 }, // b : { $gte : 3, $lt : 4 } } // => Bounds { a : [1, 2), b : [3, 4) } IndexBounds bounds = getIndexBoundsForQuery(_keyPattern.toBSON(), canonicalQuery); // Transforms bounds for each shard key field into full shard key ranges // for example : // Key { a : 1, b : 1 } // Bounds { a : [1, 2), b : [3, 4) } // => Ranges { a : 1, b : 3 } => { a : 2, b : 4 } BoundList ranges = _keyPattern.flattenBounds(bounds); for (BoundList::const_iterator it = ranges.begin(); it != ranges.end(); ++it) { getShardIdsForRange(shardIds, it->first /*min*/, it->second /*max*/); // once we know we need to visit all shards no need to keep looping if(shardIds.size() == _shardIds.size()) break; } // SERVER-4914 Some clients of getShardIdsForQuery() assume at least one shard will be // returned. For now, we satisfy that assumption by adding a shard with no matches rather // than return an empty set of shards. if (shardIds.empty()) { massert( 16068, "no chunk ranges available", !_chunkRanges.ranges().empty() ); shardIds.insert(_chunkRanges.ranges().begin()->second->getShardId()); } }
IndexBounds ChunkManager::getIndexBoundsForQuery(const BSONObj& key, const CanonicalQuery& canonicalQuery) { // $text is not allowed in planning since we don't have text index on mongos. // // TODO: Treat $text query as a no-op in planning. So with shard key {a: 1}, // the query { a: 2, $text: { ... } } will only target to {a: 2}. if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::TEXT)) { IndexBounds bounds; IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey] return bounds; } // Consider shard key as an index string accessMethod = IndexNames::findPluginName(key); dassert(accessMethod == IndexNames::BTREE || accessMethod == IndexNames::HASHED); // Use query framework to generate index bounds QueryPlannerParams plannerParams; // Must use "shard key" index plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN; IndexEntry indexEntry(key, accessMethod, false /* multiKey */, false /* sparse */, false /* unique */, "shardkey", NULL /* filterExpr */, BSONObj()); plannerParams.indices.push_back(indexEntry); OwnedPointerVector<QuerySolution> solutions; Status status = QueryPlanner::plan(canonicalQuery, plannerParams, &solutions.mutableVector()); uassert(status.code(), status.reason(), status.isOK()); IndexBounds bounds; for (vector<QuerySolution*>::const_iterator it = solutions.begin(); bounds.size() == 0 && it != solutions.end(); it++) { // Try next solution if we failed to generate index bounds, i.e. bounds.size() == 0 bounds = collapseQuerySolution((*it)->root.get()); } if (bounds.size() == 0) { // We cannot plan the query without collection scan, so target to all shards. IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey] } return bounds; }
bool PlanCache::shouldCacheQuery(const CanonicalQuery& query) { const LiteParsedQuery& lpq = query.getParsed(); const MatchExpression* expr = query.root(); // Collection scan // No sort order requested if (lpq.getSort().isEmpty() && expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) { return false; } // Hint provided if (!lpq.getHint().isEmpty()) { return false; } // Min provided // Min queries are a special case of hinted queries. if (!lpq.getMin().isEmpty()) { return false; } // Max provided // Similar to min, max queries are a special case of hinted queries. if (!lpq.getMax().isEmpty()) { return false; } // Explain queries are not-cacheable. This is primarily because of // the need to generate current and accurate information in allPlans. // If the explain report is generated by the cached plan runner using // stale information from the cache for the losing plans, allPlans would // simply be wrong. if (lpq.isExplain()) { return false; } // Tailable cursors won't get cached, just turn into collscans. if (query.getParsed().isTailable()) { return false; } // Snapshot is really a hint. if (query.getParsed().isSnapshot()) { return false; } return true; }
bool PlanCache::shouldCacheQuery(const CanonicalQuery& query) { const QueryRequest& qr = query.getQueryRequest(); const MatchExpression* expr = query.root(); // Collection scan // No sort order requested if (qr.getSort().isEmpty() && expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) { return false; } // Hint provided if (!qr.getHint().isEmpty()) { return false; } // Min provided // Min queries are a special case of hinted queries. if (!qr.getMin().isEmpty()) { return false; } // Max provided // Similar to min, max queries are a special case of hinted queries. if (!qr.getMax().isEmpty()) { return false; } // We don't read or write from the plan cache for explain. This ensures // that explain queries don't affect cache state, and it also makes // sure that we can always generate information regarding rejected plans // and/or trial period execution of candidate plans. if (qr.isExplain()) { return false; } // Tailable cursors won't get cached, just turn into collscans. if (query.getQueryRequest().isTailable()) { return false; } // Snapshot is really a hint. if (query.getQueryRequest().isSnapshot()) { return false; } return true; }
StatusWith<BSONObj> ShardKeyPattern::extractShardKeyFromQuery(const CanonicalQuery& query) const { if (!isValid()) return StatusWith<BSONObj>(BSONObj()); // Extract equalities from query. EqualityMatches equalities; // TODO: Build the path set initially? FieldRefSet keyPatternPathSet(_keyPatternPaths.vector()); // We only care about extracting the full key pattern paths - if they don't exist (or are // conflicting), we don't contain the shard key. Status eqStatus = pathsupport::extractFullEqualityMatches(*query.root(), keyPatternPathSet, &equalities); // NOTE: Failure to extract equality matches just means we return no shard key - it's not // an error we propagate if (!eqStatus.isOK()) return StatusWith<BSONObj>(BSONObj()); // Extract key from equalities // NOTE: The method below is equivalent to constructing a BSONObj and running // extractShardKeyFromMatchable, but doesn't require creating the doc. BSONObjBuilder keyBuilder; // Iterate the parsed paths to avoid re-parsing for (OwnedPointerVector<FieldRef>::const_iterator it = _keyPatternPaths.begin(); it != _keyPatternPaths.end(); ++it) { const FieldRef& patternPath = **it; BSONElement equalEl = findEqualityElement(equalities, patternPath); if (!isShardKeyElement(equalEl, false)) return StatusWith<BSONObj>(BSONObj()); if (isHashedPattern()) { keyBuilder.append( patternPath.dottedField(), BSONElementHasher::hash64(equalEl, BSONElementHasher::DEFAULT_HASH_SEED)); } else { // NOTE: The equal element may *not* have the same field name as the path - // nested $and, $eq, for example keyBuilder.appendAs(equalEl, patternPath.dottedField()); } } dassert(isShardKey(keyBuilder.asTempObj())); return StatusWith<BSONObj>(keyBuilder.obj()); }
// static bool SubplanRunner::canUseSubplanRunner(const CanonicalQuery& query) { const LiteParsedQuery& lpq = query.getParsed(); const MatchExpression* expr = query.root(); // Only rooted ORs work with the subplan scheme. if (MatchExpression::OR != expr->matchType()) { return false; } // Collection scan // No sort order requested if (lpq.getSort().isEmpty() && expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) { return false; } // Hint provided if (!lpq.getHint().isEmpty()) { return false; } // Min provided // Min queries are a special case of hinted queries. if (!lpq.getMin().isEmpty()) { return false; } // Max provided // Similar to min, max queries are a special case of hinted queries. if (!lpq.getMax().isEmpty()) { return false; } // Tailable cursors won't get cached, just turn into collscans. if (query.getParsed().hasOption(QueryOption_CursorTailable)) { return false; } // Snapshot is really a hint. if (query.getParsed().isSnapshot()) { return false; } return true; }
// static bool SubplanStage::canUseSubplanning(const CanonicalQuery& query) { const LiteParsedQuery& lpq = query.getParsed(); const MatchExpression* expr = query.root(); // Only rooted ORs work with the subplan scheme. if (MatchExpression::OR != expr->matchType()) { return false; } // Hint provided if (!lpq.getHint().isEmpty()) { return false; } // Min provided // Min queries are a special case of hinted queries. if (!lpq.getMin().isEmpty()) { return false; } // Max provided // Similar to min, max queries are a special case of hinted queries. if (!lpq.getMax().isEmpty()) { return false; } // Tailable cursors won't get cached, just turn into collscans. if (query.getParsed().getOptions().tailable) { return false; } // Snapshot is really a hint. if (query.getParsed().isSnapshot()) { return false; } return true; }
Status getRunnerDistinct(Collection* collection, const BSONObj& query, const string& field, Runner** out) { // This should'a been checked by the distinct command. verify(collection); // TODO: check for idhack here? // When can we do a fast distinct hack? // 1. There is a plan with just one leaf and that leaf is an ixscan. // 2. The ixscan indexes the field we're interested in. // 2a: We are correct if the index contains the field but for now we look for prefix. // 3. The query is covered/no fetch. // // We go through normal planning (with limited parameters) to see if we can produce // a soln with the above properties. QueryPlannerParams plannerParams; plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN; IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false); while (ii.more()) { const IndexDescriptor* desc = ii.next(); // The distinct hack can work if any field is in the index but it's not always clear // if it's a win unless it's the first field. if (desc->keyPattern().firstElement().fieldName() == field) { plannerParams.indices.push_back(IndexEntry(desc->keyPattern(), desc->isMultikey(), desc->isSparse(), desc->indexName(), desc->infoObj())); } } // We only care about the field that we're projecting over. Have to drop the _id field // explicitly because those are .find() semantics. // // Applying a projection allows the planner to try to give us covered plans. BSONObj projection; if ("_id" == field) { projection = BSON("_id" << 1); } else { projection = BSON("_id" << 0 << field << 1); } // Apply a projection of the key. Empty BSONObj() is for the sort. CanonicalQuery* cq; Status status = CanonicalQuery::canonicalize(collection->ns().ns(), query, BSONObj(), projection, &cq); if (!status.isOK()) { return status; } // No index has the field we're looking for. Punt to normal planning. if (plannerParams.indices.empty()) { // Takes ownership of cq. return getRunner(cq, out); } // If we're here, we have an index prefixed by the field we're distinct-ing over. // If there's no query, we can just distinct-scan one of the indices. if (query.isEmpty()) { DistinctNode* dn = new DistinctNode(); dn->indexKeyPattern = plannerParams.indices[0].keyPattern; dn->direction = 1; IndexBoundsBuilder::allValuesBounds(dn->indexKeyPattern, &dn->bounds); dn->fieldNo = 0; QueryPlannerParams params; // Takes ownership of 'dn'. QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*cq, params, dn); verify(soln); WorkingSet* ws; PlanStage* root; verify(StageBuilder::build(*soln, &root, &ws)); *out = new SingleSolutionRunner(collection, cq, soln, root, ws); return Status::OK(); } // See if we can answer the query in a fast-distinct compatible fashion. vector<QuerySolution*> solutions; status = QueryPlanner::plan(*cq, plannerParams, &solutions); if (!status.isOK()) { return getRunner(cq, out); } // XXX: why do we need to do this? planner should prob do this internally cq->root()->resetTag(); // We look for a solution that has an ixscan we can turn into a distinctixscan for (size_t i = 0; i < solutions.size(); ++i) { if (turnIxscanIntoDistinctIxscan(solutions[i], field)) { // Great, we can use solutions[i]. Clean up the other QuerySolution(s). for (size_t j = 0; j < solutions.size(); ++j) { if (j != i) { delete solutions[j]; } } // Build and return the SSR over solutions[i]. WorkingSet* ws; PlanStage* root; verify(StageBuilder::build(*solutions[i], &root, &ws)); *out = new SingleSolutionRunner(collection, cq, solutions[i], root, ws); return Status::OK(); } } // If we're here, the planner made a soln with the restricted index set but we couldn't // translate any of them into a distinct-compatible soln. So, delete the solutions and just // go through normal planning. for (size_t i = 0; i < solutions.size(); ++i) { delete solutions[i]; } return getRunner(cq, out); }
// static StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan( const CanonicalQuery& query, const QueryPlannerParams& params) { LOG(5) << "Beginning planning..." << endl << "=============================" << endl << "Options = " << optionString(params.options) << endl << "Canonical query:" << endl << redact(query.toString()) << "============================="; std::vector<std::unique_ptr<QuerySolution>> out; for (size_t i = 0; i < params.indices.size(); ++i) { LOG(5) << "Index " << i << " is " << params.indices[i].toString(); } const bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); const bool isTailable = query.getQueryRequest().isTailable(); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. if (isTailable) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { auto soln = buildCollscanSoln(query, isTailable, params); if (soln) { out.push_back(std::move(soln)); } } return {std::move(out)}; } // The hint or sort can be $natural: 1. If this happens, output a collscan. If both // a $natural hint and a $natural sort are specified, then the direction of the collscan // is determined by the sign of the sort (not the sign of the hint). if (!query.getQueryRequest().getHint().isEmpty() || !query.getQueryRequest().getSort().isEmpty()) { BSONObj hintObj = query.getQueryRequest().getHint(); BSONObj sortObj = query.getQueryRequest().getSort(); BSONElement naturalHint = dps::extractElementAtPath(hintObj, "$natural"); BSONElement naturalSort = dps::extractElementAtPath(sortObj, "$natural"); // A hint overrides a $natural sort. This means that we don't force a table // scan if there is a $natural sort with a non-$natural hint. if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) { LOG(5) << "Forcing a table scan due to hinted $natural"; // min/max are incompatible with $natural. if (canTableScan && query.getQueryRequest().getMin().isEmpty() && query.getQueryRequest().getMax().isEmpty()) { auto soln = buildCollscanSoln(query, isTailable, params); if (soln) { out.push_back(std::move(soln)); } } return {std::move(out)}; } } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { LOG(5) << "Predicate over field '" << *it << "'"; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. // If index filters in the query settings were used to override // the allowed indices for planning, we should not use the hinted index // requested in the query. BSONObj hintIndex; if (!params.indexFiltersApplied) { hintIndex = query.getQueryRequest().getHint(); } // If snapshot is set, default to collscanning. If the query param SNAPSHOT_USE_ID is set, // snapshot is a form of a hint, so try to use _id index to make a real plan. If that fails, // just scan the _id index. // // Don't do this if the query is a geonear or text as as text search queries must be answered // using full text indices and geoNear queries must be answered using geospatial indices. if (query.getQueryRequest().isSnapshot()) { RARELY { warning() << "The snapshot option is deprecated. See " "http://dochub.mongodb.org/core/snapshot-deprecation"; } if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { const bool useIXScan = params.options & QueryPlannerParams::SNAPSHOT_USE_ID; if (!useIXScan) { auto soln = buildCollscanSoln(query, isTailable, params); if (soln) { out.push_back(std::move(soln)); } return {std::move(out)}; } else { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } } }
// static Status QueryPlanner::plan(const CanonicalQuery& query, const QueryPlannerParams& params, std::vector<QuerySolution*>* out) { QLOG() << "=============================\n" << "Beginning planning, options = " << optionString(params.options) << endl << "Canonical query:\n" << query.toString() << endl << "=============================" << endl; for (size_t i = 0; i < params.indices.size(); ++i) { QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl; } bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().hasOption(QueryOption_CursorTailable)) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { QuerySolution* soln = buildCollscanSoln(query, true, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } // The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of // saying "table scan for two, please." if (!query.getParsed().getHint().isEmpty()) { BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural"); if (!natural.eoo()) { QLOG() << "forcing a table scan due to hinted $natural\n"; // min/max are incompatible with $natural. if (canTableScan && query.getParsed().getMin().isEmpty() && query.getParsed().getMax().isEmpty()) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { QLOG() << "predicate over field " << *it << endl; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. BSONObj hintIndex = query.getParsed().getHint(); // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real // plan. If that fails, just scan the _id index. if (query.getParsed().isSnapshot()) { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } size_t hintIndexNumber = numeric_limits<size_t>::max(); if (hintIndex.isEmpty()) { QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); } else { // Sigh. If the hint is specified it might be using the index name. BSONElement firstHintElt = hintIndex.firstElement(); if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { string hintName = firstHintElt.String(); for (size_t i = 0; i < params.indices.size(); ++i) { if (params.indices[i].name == hintName) { QLOG() << "hint by name specified, restricting indices to " << params.indices[i].keyPattern.toString() << endl; relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); hintIndexNumber = i; hintIndex = params.indices[i].keyPattern; break; } } } else { for (size_t i = 0; i < params.indices.size(); ++i) { if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); QLOG() << "hint specified, restricting indices to " << hintIndex.toString() << endl; hintIndexNumber = i; break; } } } if (hintIndexNumber == numeric_limits<size_t>::max()) { return Status(ErrorCodes::BadValue, "bad hint"); } } // Deal with the .min() and .max() query options. If either exist we can only use an index // that matches the object inside. if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) { BSONObj minObj = query.getParsed().getMin(); BSONObj maxObj = query.getParsed().getMax(); // This is the index into params.indices[...] that we use. size_t idxNo = numeric_limits<size_t>::max(); // If there's an index hinted we need to be able to use it. if (!hintIndex.isEmpty()) { if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) { QLOG() << "minobj doesnt work w hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with min query"); } if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) { QLOG() << "maxobj doesnt work w hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with max query"); } idxNo = hintIndexNumber; } else { // No hinted index, look for one that is compatible (has same field names and // ordering thereof). for (size_t i = 0; i < params.indices.size(); ++i) { const BSONObj& kp = params.indices[i].keyPattern; BSONObj toUse = minObj.isEmpty() ? maxObj : minObj; if (indexCompatibleMaxMin(toUse, kp)) { idxNo = i; break; } } } if (idxNo == numeric_limits<size_t>::max()) { QLOG() << "Can't find relevant index to use for max/min query"; // Can't find an index to use, bail out. return Status(ErrorCodes::BadValue, "unable to find relevant index for max/min query"); } // maxObj can be empty; the index scan just goes until the end. minObj can't be empty // though, so if it is, we make a minKey object. if (minObj.isEmpty()) { BSONObjBuilder bob; bob.appendMinKey(""); minObj = bob.obj(); } else { // Must strip off the field names to make an index key. minObj = stripFieldNames(minObj); } if (!maxObj.isEmpty()) { // Must strip off the field names to make an index key. maxObj = stripFieldNames(maxObj); } QLOG() << "max/min query using index " << params.indices[idxNo].toString() << endl; // Make our scan and output. QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(params.indices[idxNo], query, params, minObj, maxObj); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } return Status::OK(); } for (size_t i = 0; i < relevantIndices.size(); ++i) { QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl; } // Figure out how useful each index is to each predicate. // query.root() is now annotated with RelevantTag(s). QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); QLOG() << "rated tree" << endl; QLOG() << query.root()->toString() << endl; // If there is a GEO_NEAR it must have an index it can use directly. // XXX: move into data access? MatchExpression* gnNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { QLOG() << "unable to find index for $geoNear query" << endl; return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query"); } GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode); vector<size_t> newFirst; // 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed // the full query tree inside it as a matcher. for (size_t i = 0; i < tag->first.size(); ++i) { // GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning. if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) { newFirst.push_back(i); continue; } // If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the // entire tree as a filter, if possible. GeoNear2DNode* solnRoot = new GeoNear2DNode(); solnRoot->nq = gnme->getData(); if (NULL != query.getProj()) { solnRoot->addPointMeta = query.getProj()->wantGeoNearPoint(); solnRoot->addDistMeta = query.getProj()->wantGeoNearDistance(); } if (MatchExpression::GEO_NEAR != query.root()->matchType()) { // root is an AND, clone and delete the GEO_NEAR child. MatchExpression* filterTree = query.root()->shallowClone(); verify(MatchExpression::AND == filterTree->matchType()); bool foundChild = false; for (size_t i = 0; i < filterTree->numChildren(); ++i) { if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) { foundChild = true; filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i); break; } } verify(foundChild); solnRoot->filter.reset(filterTree); } solnRoot->numWanted = query.getParsed().getNumToReturn(); if (0 == solnRoot->numWanted) { solnRoot->numWanted = 100; } solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern; // Remove the 2d index. 2d can only be the first field, and we know there is // only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll // only be first for gnNode. tag->first.erase(tag->first.begin() + i); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } } // Continue planning w/non-2d indices tagged for this pred. tag->first.swap(newFirst); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return Status::OK(); } } // Likewise, if there is a TEXT it must have an index it can use directly. MatchExpression* textNode; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return Status::OK(); } } // If we have any relevant indices, we try to create indexed plans. if (0 < relevantIndices.size()) { // The enumerator spits out trees tagged with IndexTag(s). PlanEnumeratorParams enumParams; enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION; enumParams.root = query.root(); enumParams.indices = &relevantIndices; PlanEnumerator isp(enumParams); isp.init(); MatchExpression* rawTree; // XXX: have limit on # of indexed solns we'll consider. We could have a perverse // query and index that could make n^2 very unpleasant. while (isp.getNext(&rawTree)) { QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString() << endl; // This can fail if enumeration makes a mistake. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices); if (NULL == solnRoot) { continue; } QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { QLOG() << "Planner: adding solution:\n" << soln->toString() << endl; out->push_back(soln); } } } QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n"; // An index was hinted. If there are any solutions, they use the hinted index. If not, we // scan the entire index to provide results and output that as our plan. This is the // desired behavior when an index is hinted that is not relevant to the query. if (!hintIndex.isEmpty()) { if (0 == out->size()) { QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); verify(NULL != soln); QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl; out->push_back(soln); } return Status::OK(); } // If a sort order is requested, there may be an index that provides it, even if that // index is not over any predicates in the query. // if (!query.getParsed().getSort().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { // See if we have a sort provided from an index already. bool usingIndexToSort = false; for (size_t i = 0; i < out->size(); ++i) { QuerySolution* soln = (*out)[i]; if (!soln->hasSortStage) { usingIndexToSort = true; break; } } if (!usingIndexToSort) { for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& index = params.indices[i]; if (index.sparse) { continue; } const BSONObj kp = LiteParsedQuery::normalizeSortOrder(index.keyPattern); if (providesSort(query, kp)) { QLOG() << "Planner: outputting soln that uses index to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); if (NULL != soln) { out->push_back(soln); break; } } if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { QLOG() << "Planner: outputting soln that uses (reverse) index " << "to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); if (NULL != soln) { out->push_back(soln); break; } } } } } // TODO: Do we always want to offer a collscan solution? // XXX: currently disabling the always-use-a-collscan in order to find more planner bugs. if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && hintIndex.isEmpty() && ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan))) { QuerySolution* collscan = buildCollscanSoln(query, false, params); if (NULL != collscan) { out->push_back(collscan); QLOG() << "Planner: outputting a collscan:\n"; QLOG() << collscan->toString() << endl; } } return Status::OK(); }
// static void QueryPlanner::plan(const CanonicalQuery& query, const vector<BSONObj>& indexKeyPatterns, vector<QuerySolution*>* out) { // XXX: If pq.hasOption(QueryOption_OplogReplay) use FindingStartCursor equivalent which // must be translated into stages. // // Planner Section 1: Calculate predicate/index data. // // Get all the predicates (and their fields). PredicateMap predicates; makePredicateMap(query.root(), &predicates); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().hasOption(QueryOption_CursorTailable)) { if (!hasPredicate(predicates, MatchExpression::GEO_NEAR)) { out->push_back(makeCollectionScan(query, true)); } return; } // NOR and NOT we can't handle well with indices. If we see them here, they weren't // rewritten. Just output a collscan for those. if (hasPredicate(predicates, MatchExpression::NOT) || hasPredicate(predicates, MatchExpression::NOR)) { // If there's a near predicate, we can't handle this. // TODO: Should canonicalized query detect this? if (hasPredicate(predicates, MatchExpression::GEO_NEAR)) { warning() << "Can't handle NOT/NOR with GEO_NEAR"; return; } out->push_back(makeCollectionScan(query, false)); return; } // Filter our indices so we only look at indices that are over our predicates. vector<BSONObj> relevantIndices; findRelevantIndices(predicates, indexKeyPatterns, &relevantIndices); // No indices, no work to do. if (0 == relevantIndices.size()) { return; } // Figure out how useful each index is to each predicate. rateIndices(relevantIndices, &predicates); // // Planner Section 2: Use predicate/index data to output sets of indices that we can use. // PlanEnumerator isp(&query, &predicates, &relevantIndices); MatchExpression* rawTree; while (isp.getNext(&rawTree)) { QuerySolutionNode* solutionRoot = NULL; // // Planner Section 3: Logical Rewrite. Use the index selection and the tree structure // to try to rewrite the tree. TODO: Do this for real. We treat the tree as static. // // // Planner Section 4: Covering. If we're projecting, See if we get any covering from // this plan. If not, add a fetch. // if (!query.getParsed().getProj().isEmpty()) { warning() << "Can't deal with proj yet" << endl; } else { // Note that we need a fetch, possibly tack on to end? } // // Planner Section 5: Sort. If we're sorting, see if the plan gives us a sort for free. // If not, add a sort. // if (!query.getParsed().getSort().isEmpty()) { } else { // Note that we need a sort, possibly tack on to end? may want to see if sort is // covered and then tack fetch on after the covered sort... } // // Planner Section 6: Final check. Make sure that we build a valid solution. // TODO: Validate. // if (NULL != solutionRoot) { QuerySolution* qs = new QuerySolution(); qs->root.reset(solutionRoot); out->push_back(qs); } } // TODO: Do we always want to offer a collscan solution? if (!hasPredicate(predicates, MatchExpression::GEO_NEAR)) { out->push_back(makeCollectionScan(query, false)); } }
void UpdateStage::transformAndUpdate(BSONObj& oldObj, DiskLoc& loc) { const UpdateRequest* request = _params.request; UpdateDriver* driver = _params.driver; CanonicalQuery* cq = _params.canonicalQuery; UpdateLifecycle* lifecycle = request->getLifecycle(); // Ask the driver to apply the mods. It may be that the driver can apply those "in // place", that is, some values of the old document just get adjusted without any // change to the binary layout on the bson layer. It may be that a whole new // document is needed to accomodate the new bson layout of the resulting document. _doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled); BSONObj logObj; FieldRefSet updatedFields; Status status = Status::OK(); if (!driver->needMatchDetails()) { // If we don't need match details, avoid doing the rematch status = driver->update(StringData(), &_doc, &logObj, &updatedFields); } else { // If there was a matched field, obtain it. MatchDetails matchDetails; matchDetails.requestElemMatchKey(); dassert(cq); verify(cq->root()->matchesBSON(oldObj, &matchDetails)); string matchedField; if (matchDetails.hasElemMatchKey()) matchedField = matchDetails.elemMatchKey(); // TODO: Right now, each mod checks in 'prepare' that if it needs positional // data, that a non-empty StringData() was provided. In principle, we could do // that check here in an else clause to the above conditional and remove the // checks from the mods. status = driver->update(matchedField, &_doc, &logObj, &updatedFields); } if (!status.isOK()) { uasserted(16837, status.reason()); } // Ensure _id exists and is first uassertStatusOK(ensureIdAndFirst(_doc)); // If the driver applied the mods in place, we can ask the mutable for what // changed. We call those changes "damages". :) We use the damages to inform the // journal what was changed, and then apply them to the original document // ourselves. If, however, the driver applied the mods out of place, we ask it to // generate a new, modified document for us. In that case, the file manager will // take care of the journaling details for us. // // This code flow is admittedly odd. But, right now, journaling is baked in the file // manager. And if we aren't using the file manager, we have to do jounaling // ourselves. bool docWasModified = false; BSONObj newObj; const char* source = NULL; bool inPlace = _doc.getInPlaceUpdates(&_damages, &source); // If something changed in the document, verify that no immutable fields were changed // and data is valid for storage. if ((!inPlace || !_damages.empty()) ) { if (!(request->isFromReplication() || request->isFromMigration())) { const std::vector<FieldRef*>* immutableFields = NULL; if (lifecycle) immutableFields = lifecycle->getImmutableFields(); uassertStatusOK(validate(oldObj, updatedFields, _doc, immutableFields, driver->modOptions()) ); } } // Save state before making changes saveState(); { WriteUnitOfWork wunit(request->getOpCtx()); if (inPlace && !driver->modsAffectIndices()) { // If a set of modifiers were all no-ops, we are still 'in place', but there // is no work to do, in which case we want to consider the object unchanged. if (!_damages.empty() ) { // Don't actually do the write if this is an explain. if (!request->isExplain()) { invariant(_collection); const RecordData oldRec(oldObj.objdata(), oldObj.objsize()); _collection->updateDocumentWithDamages(request->getOpCtx(), loc, oldRec, source, _damages); } docWasModified = true; _specificStats.fastmod = true; } newObj = oldObj; } else { // The updates were not in place. Apply them through the file manager. newObj = _doc.getObject(); uassert(17419, str::stream() << "Resulting document after update is larger than " << BSONObjMaxUserSize, newObj.objsize() <= BSONObjMaxUserSize); docWasModified = true; // Don't actually do the write if this is an explain. if (!request->isExplain()) { invariant(_collection); StatusWith<DiskLoc> res = _collection->updateDocument(request->getOpCtx(), loc, newObj, true, _params.opDebug); uassertStatusOK(res.getStatus()); DiskLoc newLoc = res.getValue(); // If the document moved, we might see it again in a collection scan (maybe it's // a document after our current document). // // If the document is indexed and the mod changes an indexed value, we might see // it again. For an example, see the comment above near declaration of // updatedLocs. if (_updatedLocs && (newLoc != loc || driver->modsAffectIndices())) { _updatedLocs->insert(newLoc); } } } // Call logOp if requested, and we're not an explain. if (request->shouldCallLogOp() && !logObj.isEmpty() && !request->isExplain()) { BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request->isMulti()); repl::logOp(request->getOpCtx(), "u", request->getNamespaceString().ns().c_str(), logObj, &idQuery, NULL, request->isFromMigration()); } wunit.commit(); } // Restore state after modification // As restoreState may restore (recreate) cursors, make sure to restore the // state outside of the WritUnitOfWork. restoreState(request->getOpCtx()); // Only record doc modifications if they wrote (exclude no-ops). Explains get // recorded as if they wrote. if (docWasModified) { _specificStats.nModified++; } }
/* ns: namespace, e.g. <database>.<collection> pattern: the "where" clause / criteria justOne: stop after 1 match god: allow access to system namespaces, and don't yield */ long long deleteObjects(const StringData& ns, BSONObj pattern, bool justOne, bool logop, bool god) { if (!god) { if (ns.find( ".system.") != string::npos) { // note a delete from system.indexes would corrupt the db if done here, as there are // pointers into those objects in NamespaceDetails. uassert(12050, "cannot delete from system namespace", legalClientSystemNS( ns, true ) ); } if (ns.find('$') != string::npos) { log() << "cannot delete from collection with reserved $ in name: " << ns << endl; uasserted( 10100, "cannot delete from collection with reserved $ in name" ); } } Collection* collection = currentClient.get()->database()->getCollection(ns); if (NULL == collection) { return 0; } uassert(10101, str::stream() << "can't remove from a capped collection: " << ns, !collection->isCapped()); string nsForLogOp = ns.toString(); // XXX-ERH long long nDeleted = 0; CanonicalQuery* cq; if (!CanonicalQuery::canonicalize(ns.toString(), pattern, &cq).isOK()) { uasserted(17218, "Can't canonicalize query " + pattern.toString()); return 0; } bool canYield = !god && !QueryPlannerCommon::hasNode(cq->root(), MatchExpression::ATOMIC); Runner* rawRunner; if (!getRunner(cq, &rawRunner).isOK()) { uasserted(17219, "Can't get runner for query " + pattern.toString()); return 0; } auto_ptr<Runner> runner(rawRunner); auto_ptr<ScopedRunnerRegistration> safety; if (canYield) { safety.reset(new ScopedRunnerRegistration(runner.get())); runner->setYieldPolicy(Runner::YIELD_AUTO); } DiskLoc rloc; Runner::RunnerState state; while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &rloc))) { BSONObj toDelete; // XXX: do we want to buffer docs and delete them in a group rather than // saving/restoring state repeatedly? runner->saveState(); collection->deleteDocument(rloc, false, false, logop ? &toDelete : NULL ); runner->restoreState(); nDeleted++; if (logop) { if ( toDelete.isEmpty() ) { problem() << "deleted object without id, not logging" << endl; } else { bool replJustOne = true; logOp("d", nsForLogOp.c_str(), toDelete, 0, &replJustOne); } } if (justOne) { break; } if (!god) { getDur().commitIfNeeded(); } if (debug && god && nDeleted == 100) { log() << "warning high number of deletes with god=true " << " which could use significant memory b/c we don't commit journal"; } } return nDeleted; }
// static Status QueryPlanner::plan(const CanonicalQuery& query, const QueryPlannerParams& params, std::vector<QuerySolution*>* out) { LOG(5) << "Beginning planning..." << endl << "=============================" << endl << "Options = " << optionString(params.options) << endl << "Canonical query:" << endl << query.toString() << "=============================" << endl; for (size_t i = 0; i < params.indices.size(); ++i) { LOG(5) << "Index " << i << " is " << params.indices[i].toString() << endl; } bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().isTailable()) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { QuerySolution* soln = buildCollscanSoln(query, true, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } // The hint or sort can be $natural: 1. If this happens, output a collscan. If both // a $natural hint and a $natural sort are specified, then the direction of the collscan // is determined by the sign of the sort (not the sign of the hint). if (!query.getParsed().getHint().isEmpty() || !query.getParsed().getSort().isEmpty()) { BSONObj hintObj = query.getParsed().getHint(); BSONObj sortObj = query.getParsed().getSort(); BSONElement naturalHint = hintObj.getFieldDotted("$natural"); BSONElement naturalSort = sortObj.getFieldDotted("$natural"); // A hint overrides a $natural sort. This means that we don't force a table // scan if there is a $natural sort with a non-$natural hint. if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) { LOG(5) << "Forcing a table scan due to hinted $natural\n"; // min/max are incompatible with $natural. if (canTableScan && query.getParsed().getMin().isEmpty() && query.getParsed().getMax().isEmpty()) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return Status::OK(); } } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { LOG(5) << "Predicate over field '" << *it << "'" << endl; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. // If index filters in the query settings were used to override // the allowed indices for planning, we should not use the hinted index // requested in the query. BSONObj hintIndex; if (!params.indexFiltersApplied) { hintIndex = query.getParsed().getHint(); } // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real // plan. If that fails, just scan the _id index. if (query.getParsed().isSnapshot()) { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } size_t hintIndexNumber = numeric_limits<size_t>::max(); if (hintIndex.isEmpty()) { QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); } else { // Sigh. If the hint is specified it might be using the index name. BSONElement firstHintElt = hintIndex.firstElement(); if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { string hintName = firstHintElt.String(); for (size_t i = 0; i < params.indices.size(); ++i) { if (params.indices[i].name == hintName) { LOG(5) << "Hint by name specified, restricting indices to " << params.indices[i].keyPattern.toString() << endl; relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); hintIndexNumber = i; hintIndex = params.indices[i].keyPattern; break; } } } else { for (size_t i = 0; i < params.indices.size(); ++i) { if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); LOG(5) << "Hint specified, restricting indices to " << hintIndex.toString() << endl; hintIndexNumber = i; break; } } } if (hintIndexNumber == numeric_limits<size_t>::max()) { return Status(ErrorCodes::BadValue, "bad hint"); } } // Deal with the .min() and .max() query options. If either exist we can only use an index // that matches the object inside. if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) { BSONObj minObj = query.getParsed().getMin(); BSONObj maxObj = query.getParsed().getMax(); // The unfinished siblings of these objects may not be proper index keys because they // may be empty objects or have field names. When an index is picked to use for the // min/max query, these "finished" objects will always be valid index keys for the // index's key pattern. BSONObj finishedMinObj; BSONObj finishedMaxObj; // This is the index into params.indices[...] that we use. size_t idxNo = numeric_limits<size_t>::max(); // If there's an index hinted we need to be able to use it. if (!hintIndex.isEmpty()) { if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) { LOG(5) << "Minobj doesn't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with min query"); } if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) { LOG(5) << "Maxobj doesn't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with max query"); } const BSONObj& kp = params.indices[hintIndexNumber].keyPattern; finishedMinObj = finishMinObj(kp, minObj, maxObj); finishedMaxObj = finishMaxObj(kp, minObj, maxObj); // The min must be less than the max for the hinted index ordering. if (0 <= finishedMinObj.woCompare(finishedMaxObj, kp, false)) { LOG(5) << "Minobj/Maxobj don't work with hint"; return Status(ErrorCodes::BadValue, "hint provided does not work with min/max query"); } idxNo = hintIndexNumber; } else { // No hinted index, look for one that is compatible (has same field names and // ordering thereof). for (size_t i = 0; i < params.indices.size(); ++i) { const BSONObj& kp = params.indices[i].keyPattern; BSONObj toUse = minObj.isEmpty() ? maxObj : minObj; if (indexCompatibleMaxMin(toUse, kp)) { // In order to be fully compatible, the min has to be less than the max // according to the index key pattern ordering. The first step in verifying // this is "finish" the min and max by replacing empty objects and stripping // field names. finishedMinObj = finishMinObj(kp, minObj, maxObj); finishedMaxObj = finishMaxObj(kp, minObj, maxObj); // Now we have the final min and max. This index is only relevant for // the min/max query if min < max. if (0 >= finishedMinObj.woCompare(finishedMaxObj, kp, false)) { // Found a relevant index. idxNo = i; break; } // This index is not relevant; move on to the next. } } } if (idxNo == numeric_limits<size_t>::max()) { LOG(5) << "Can't find relevant index to use for max/min query"; // Can't find an index to use, bail out. return Status(ErrorCodes::BadValue, "unable to find relevant index for max/min query"); } LOG(5) << "Max/min query using index " << params.indices[idxNo].toString() << endl; // Make our scan and output. QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan( params.indices[idxNo], query, params, finishedMinObj, finishedMaxObj); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } return Status::OK(); } for (size_t i = 0; i < relevantIndices.size(); ++i) { LOG(2) << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl; } // Figure out how useful each index is to each predicate. QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); QueryPlannerIXSelect::stripInvalidAssignments(query.root(), relevantIndices); // Unless we have GEO_NEAR, TEXT, or a projection, we may be able to apply an optimization // in which we strip unnecessary index assignments. // // Disallowed with projection because assignment to a non-unique index can allow the plan // to be covered. // // TEXT and GEO_NEAR are special because they require the use of a text/geo index in order // to be evaluated correctly. Stripping these "mandatory assignments" is therefore invalid. if (query.getParsed().getProj().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { QueryPlannerIXSelect::stripUnneededAssignments(query.root(), relevantIndices); } // query.root() is now annotated with RelevantTag(s). LOG(5) << "Rated tree:" << endl << query.root()->toString(); // If there is a GEO_NEAR it must have an index it can use directly. MatchExpression* gnNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { LOG(5) << "Unable to find index for $geoNear query." << endl; // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query"); } LOG(5) << "Rated tree after geonear processing:" << query.root()->toString(); } // Likewise, if there is a TEXT it must have an index it can use directly. MatchExpression* textNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag()); // Exactly one text index required for TEXT. We need to check this explicitly because // the text stage can't be built if no text index exists or there is an ambiguity as to // which one to use. size_t textIndexCount = 0; for (size_t i = 0; i < params.indices.size(); i++) { if (INDEX_TEXT == params.indices[i].type) { textIndexCount++; } } if (textIndexCount != 1) { // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "need exactly one text index for $text query"); } // Error if the text node is tagged with zero indices. if (0 == tag->first.size() && 0 == tag->notFirst.size()) { // Don't leave tags on query tree. query.root()->resetTag(); return Status(ErrorCodes::BadValue, "failed to use text index to satisfy $text query (if text index is " "compound, are equality predicates given for all prefix fields?)"); } // At this point, we know that there is only one text index and that the TEXT node is // assigned to it. invariant(1 == tag->first.size() + tag->notFirst.size()); LOG(5) << "Rated tree after text processing:" << query.root()->toString(); } // If we have any relevant indices, we try to create indexed plans. if (0 < relevantIndices.size()) { // The enumerator spits out trees tagged with IndexTag(s). PlanEnumeratorParams enumParams; enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION; enumParams.root = query.root(); enumParams.indices = &relevantIndices; PlanEnumerator isp(enumParams); isp.init(); MatchExpression* rawTree; while (isp.getNext(&rawTree) && (out->size() < params.maxIndexedSolutions)) { LOG(5) << "About to build solntree from tagged tree:" << endl << rawTree->toString(); // The tagged tree produced by the plan enumerator is not guaranteed // to be canonically sorted. In order to be compatible with the cached // data, sort the tagged tree according to CanonicalQuery ordering. std::unique_ptr<MatchExpression> clone(rawTree->shallowClone()); CanonicalQuery::sortTree(clone.get()); PlanCacheIndexTree* cacheData; Status indexTreeStatus = cacheDataFromTaggedTree(clone.get(), relevantIndices, &cacheData); if (!indexTreeStatus.isOK()) { LOG(5) << "Query is not cachable: " << indexTreeStatus.reason() << endl; } unique_ptr<PlanCacheIndexTree> autoData(cacheData); // This can fail if enumeration makes a mistake. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess( query, rawTree, false, relevantIndices, params); if (NULL == solnRoot) { continue; } QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { LOG(5) << "Planner: adding solution:" << endl << soln->toString(); if (indexTreeStatus.isOK()) { SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(autoData.release()); soln->cacheData.reset(scd); } out->push_back(soln); } } } // Don't leave tags on query tree. query.root()->resetTag(); LOG(5) << "Planner: outputted " << out->size() << " indexed solutions.\n"; // Produce legible error message for failed OR planning with a TEXT child. // TODO: support collection scan for non-TEXT children of OR. if (out->size() == 0 && textNode != NULL && MatchExpression::OR == query.root()->matchType()) { MatchExpression* root = query.root(); for (size_t i = 0; i < root->numChildren(); ++i) { if (textNode == root->getChild(i)) { return Status(ErrorCodes::BadValue, "Failed to produce a solution for TEXT under OR - " "other non-TEXT clauses under OR have to be indexed as well."); } } } // An index was hinted. If there are any solutions, they use the hinted index. If not, we // scan the entire index to provide results and output that as our plan. This is the // desired behavior when an index is hinted that is not relevant to the query. if (!hintIndex.isEmpty()) { if (0 == out->size()) { QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); verify(NULL != soln); LOG(5) << "Planner: outputting soln that uses hinted index as scan." << endl; out->push_back(soln); } return Status::OK(); } // If a sort order is requested, there may be an index that provides it, even if that // index is not over any predicates in the query. // if (!query.getParsed().getSort().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { // See if we have a sort provided from an index already. // This is implied by the presence of a non-blocking solution. bool usingIndexToSort = false; for (size_t i = 0; i < out->size(); ++i) { QuerySolution* soln = (*out)[i]; if (!soln->hasBlockingStage) { usingIndexToSort = true; break; } } if (!usingIndexToSort) { for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& index = params.indices[i]; // Only regular (non-plugin) indexes can be used to provide a sort, and only // non-sparse indexes can be used to provide a sort. // // TODO: Sparse indexes can't normally provide a sort, because non-indexed // documents could potentially be missing from the result set. However, if the // query predicate can be used to guarantee that all documents to be returned // are indexed, then the index should be able to provide the sort. // // For example: // - Sparse index {a: 1, b: 1} should be able to provide a sort for // find({b: 1}).sort({a: 1}). SERVER-13908. // - Index {a: 1, b: "2dsphere"} (which is "geo-sparse", if // 2dsphereIndexVersion=2) should be able to provide a sort for // find({b: GEO}).sort({a:1}). SERVER-10801. if (index.type != INDEX_BTREE) { continue; } if (index.sparse) { continue; } // Partial indexes can only be used to provide a sort only if the query predicate is // compatible. if (index.filterExpr && !expression::isSubsetOf(query.root(), index.filterExpr)) { continue; } const BSONObj kp = QueryPlannerAnalysis::getSortPattern(index.keyPattern); if (providesSort(query, kp)) { LOG(5) << "Planner: outputting soln that uses index to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); if (NULL != soln) { PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); indexTree->setIndexEntry(params.indices[i]); SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(indexTree); scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; scd->wholeIXSolnDir = 1; soln->cacheData.reset(scd); out->push_back(soln); break; } } if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { LOG(5) << "Planner: outputting soln that uses (reverse) index " << "to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); if (NULL != soln) { PlanCacheIndexTree* indexTree = new PlanCacheIndexTree(); indexTree->setIndexEntry(params.indices[i]); SolutionCacheData* scd = new SolutionCacheData(); scd->tree.reset(indexTree); scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN; scd->wholeIXSolnDir = -1; soln->cacheData.reset(scd); out->push_back(soln); break; } } } } } // geoNear and text queries *require* an index. // Also, if a hint is specified it indicates that we MUST use it. bool possibleToCollscan = !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && hintIndex.isEmpty(); // The caller can explicitly ask for a collscan. bool collscanRequested = (params.options & QueryPlannerParams::INCLUDE_COLLSCAN); // No indexed plans? We must provide a collscan if possible or else we can't run the query. bool collscanNeeded = (0 == out->size() && canTableScan); if (possibleToCollscan && (collscanRequested || collscanNeeded)) { QuerySolution* collscan = buildCollscanSoln(query, false, params); if (NULL != collscan) { SolutionCacheData* scd = new SolutionCacheData(); scd->solnType = SolutionCacheData::COLLSCAN_SOLN; collscan->cacheData.reset(scd); out->push_back(collscan); LOG(5) << "Planner: outputting a collscan:" << endl << collscan->toString(); } } return Status::OK(); }
// static Status QueryPlanner::planFromCache(const CanonicalQuery& query, const QueryPlannerParams& params, const CachedSolution& cachedSoln, QuerySolution** out) { invariant(!cachedSoln.plannerData.empty()); invariant(out); // A query not suitable for caching should not have made its way into the cache. invariant(PlanCache::shouldCacheQuery(query)); // Look up winning solution in cached solution's array. const SolutionCacheData& winnerCacheData = *cachedSoln.plannerData[0]; if (SolutionCacheData::WHOLE_IXSCAN_SOLN == winnerCacheData.solnType) { // The solution can be constructed by a scan over the entire index. QuerySolution* soln = buildWholeIXSoln( *winnerCacheData.tree->entry, query, params, winnerCacheData.wholeIXSolnDir); if (soln == NULL) { return Status(ErrorCodes::BadValue, "plan cache error: soln that uses index to provide sort"); } else { *out = soln; return Status::OK(); } } else if (SolutionCacheData::COLLSCAN_SOLN == winnerCacheData.solnType) { // The cached solution is a collection scan. We don't cache collscans // with tailable==true, hence the false below. QuerySolution* soln = buildCollscanSoln(query, false, params); if (soln == NULL) { return Status(ErrorCodes::BadValue, "plan cache error: collection scan soln"); } else { *out = soln; return Status::OK(); } } // SolutionCacheData::USE_TAGS_SOLN == cacheData->solnType // If we're here then this is neither the whole index scan or collection scan // cases, and we proceed by using the PlanCacheIndexTree to tag the query tree. // Create a copy of the expression tree. We use cachedSoln to annotate this with indices. unique_ptr<MatchExpression> clone = std::move(query.root()->shallowClone()); LOG(5) << "Tagging the match expression according to cache data: " << endl << "Filter:" << endl << clone->toString() << "Cache data:" << endl << winnerCacheData.toString(); // Map from index name to index number. // TODO: can we assume that the index numbering has the same lifetime // as the cache state? map<BSONObj, size_t> indexMap; for (size_t i = 0; i < params.indices.size(); ++i) { const IndexEntry& ie = params.indices[i]; indexMap[ie.keyPattern] = i; LOG(5) << "Index " << i << ": " << ie.keyPattern.toString() << endl; } Status s = tagAccordingToCache(clone.get(), winnerCacheData.tree.get(), indexMap); if (!s.isOK()) { return s; } // The planner requires a defined sort order. sortUsingTags(clone.get()); LOG(5) << "Tagged tree:" << endl << clone->toString(); // Use the cached index assignments to build solnRoot. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess( query, clone.release(), false, params.indices, params); if (!solnRoot) { return Status(ErrorCodes::BadValue, str::stream() << "Failed to create data access plan from cache. Query: " << query.toStringShort()); } // Takes ownership of 'solnRoot'. QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (!soln) { return Status(ErrorCodes::BadValue, str::stream() << "Failed to analyze plan from cache. Query: " << query.toStringShort()); } LOG(5) << "Planner: solution constructed from the cache:\n" << soln->toString(); *out = soln; return Status::OK(); }
// static void QueryPlanner::plan(const CanonicalQuery& query, const QueryPlannerParams& params, vector<QuerySolution*>* out) { QLOG() << "=============================\n" << "Beginning planning, options = " << optionString(params.options) << endl << "Canonical query:\n" << query.toString() << endl << "=============================" << endl; // The shortcut formerly known as IDHACK. See if it's a simple _id query. If so we might // just make an ixscan over the _id index and bypass the rest of planning entirely. if (!query.getParsed().isExplain() && !query.getParsed().showDiskLoc() && isSimpleIdQuery(query.getParsed().getFilter()) && !query.getParsed().hasOption(QueryOption_CursorTailable)) { // See if we can find an _id index. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { const IndexEntry& index = params.indices[i]; QLOG() << "IDHACK using index " << index.toString() << endl; // If so, we make a simple scan to find the doc. IndexScanNode* isn = new IndexScanNode(); isn->indexKeyPattern = index.keyPattern; isn->indexIsMultiKey = index.multikey; isn->direction = 1; isn->bounds.isSimpleRange = true; BSONObj key = getKeyFromQuery(index.keyPattern, query.getParsed().getFilter()); isn->bounds.startKey = isn->bounds.endKey = key; isn->bounds.endKeyInclusive = true; isn->computeProperties(); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, isn); if (NULL != soln) { out->push_back(soln); QLOG() << "IDHACK solution is:\n" << (*out)[0]->toString() << endl; // And that's it. return; } } } } for (size_t i = 0; i < params.indices.size(); ++i) { QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl; } bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN); // If the query requests a tailable cursor, the only solution is a collscan + filter with // tailable set on the collscan. TODO: This is a policy departure. Previously I think you // could ask for a tailable cursor and it just tried to give you one. Now, we fail if we // can't provide one. Is this what we want? if (query.getParsed().hasOption(QueryOption_CursorTailable)) { if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) { QuerySolution* soln = buildCollscanSoln(query, true, params); if (NULL != soln) { out->push_back(soln); } } return; } // The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of // saying "table scan for two, please." if (!query.getParsed().getHint().isEmpty()) { BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural"); if (!natural.eoo()) { QLOG() << "forcing a table scan due to hinted $natural\n"; if (canTableScan) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return; } } // NOR and NOT we can't handle well with indices. If we see them here, they weren't // rewritten to remove the negation. Just output a collscan for those. if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOT) || QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOR)) { // If there's a near predicate, we can't handle this. // TODO: Should canonicalized query detect this? if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)) { warning() << "Can't handle NOT/NOR with GEO_NEAR"; return; } QLOG() << "NOT/NOR in plan, just outtping a collscan\n"; if (canTableScan) { QuerySolution* soln = buildCollscanSoln(query, false, params); if (NULL != soln) { out->push_back(soln); } } return; } // Figure out what fields we care about. unordered_set<string> fields; QueryPlannerIXSelect::getFields(query.root(), "", &fields); for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) { QLOG() << "predicate over field " << *it << endl; } // Filter our indices so we only look at indices that are over our predicates. vector<IndexEntry> relevantIndices; // Hints require us to only consider the hinted index. BSONObj hintIndex = query.getParsed().getHint(); // Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real // plan. If that fails, just scan the _id index. if (query.getParsed().isSnapshot()) { // Find the ID index in indexKeyPatterns. It's our hint. for (size_t i = 0; i < params.indices.size(); ++i) { if (isIdIndex(params.indices[i].keyPattern)) { hintIndex = params.indices[i].keyPattern; break; } } } size_t hintIndexNumber = numeric_limits<size_t>::max(); if (!hintIndex.isEmpty()) { // Sigh. If the hint is specified it might be using the index name. BSONElement firstHintElt = hintIndex.firstElement(); if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) { string hintName = firstHintElt.String(); for (size_t i = 0; i < params.indices.size(); ++i) { if (params.indices[i].name == hintName) { QLOG() << "hint by name specified, restricting indices to " << params.indices[i].keyPattern.toString() << endl; relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); hintIndexNumber = i; hintIndex = params.indices[i].keyPattern; break; } } } else { for (size_t i = 0; i < params.indices.size(); ++i) { if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) { relevantIndices.clear(); relevantIndices.push_back(params.indices[i]); QLOG() << "hint specified, restricting indices to " << hintIndex.toString() << endl; hintIndexNumber = i; break; } } } if (hintIndexNumber == numeric_limits<size_t>::max()) { // This is supposed to be an error. warning() << "Can't find hint for " << hintIndex.toString(); return; } } else { QLOG() << "Finding relevant indices\n"; QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices); } for (size_t i = 0; i < relevantIndices.size(); ++i) { QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl; } // Figure out how useful each index is to each predicate. // query.root() is now annotated with RelevantTag(s). QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices); QLOG() << "rated tree" << endl; QLOG() << query.root()->toString() << endl; // If there is a GEO_NEAR it must have an index it can use directly. // XXX: move into data access? MatchExpression* gnNode = NULL; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return; } GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode); vector<size_t> newFirst; // 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed // the full query tree inside it as a matcher. for (size_t i = 0; i < tag->first.size(); ++i) { // GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning. if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) { newFirst.push_back(i); continue; } // If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the // entire tree as a filter, if possible. GeoNear2DNode* solnRoot = new GeoNear2DNode(); solnRoot->nq = gnme->getData(); if (MatchExpression::GEO_NEAR != query.root()->matchType()) { // root is an AND, clone and delete the GEO_NEAR child. MatchExpression* filterTree = query.root()->shallowClone(); verify(MatchExpression::AND == filterTree->matchType()); bool foundChild = false; for (size_t i = 0; i < filterTree->numChildren(); ++i) { if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) { foundChild = true; filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i); break; } } verify(foundChild); solnRoot->filter.reset(filterTree); } solnRoot->numWanted = query.getParsed().getNumToReturn(); if (0 == solnRoot->numWanted) { solnRoot->numWanted = 100; } solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern; // Remove the 2d index. 2d can only be the first field, and we know there is // only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll // only be first for gnNode. tag->first.erase(tag->first.begin() + i); QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { out->push_back(soln); } } // Continue planning w/non-2d indices tagged for this pred. tag->first.swap(newFirst); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return; } } // Likewise, if there is a TEXT it must have an index it can use directly. MatchExpression* textNode; if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) { RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag()); if (0 == tag->first.size() && 0 == tag->notFirst.size()) { return; } } // If we have any relevant indices, we try to create indexed plans. if (0 < relevantIndices.size()) { // The enumerator spits out trees tagged with IndexTag(s). PlanEnumerator isp(query.root(), &relevantIndices); isp.init(); MatchExpression* rawTree; while (isp.getNext(&rawTree)) { QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString() << endl; // This can fail if enumeration makes a mistake. QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices); if (NULL == solnRoot) { continue; } QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot); if (NULL != soln) { QLOG() << "Planner: adding solution:\n" << soln->toString() << endl; out->push_back(soln); } } } QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n"; // An index was hinted. If there are any solutions, they use the hinted index. If not, we // scan the entire index to provide results and output that as our plan. This is the // desired behavior when an index is hinted that is not relevant to the query. if (!hintIndex.isEmpty() && (0 == out->size())) { QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params); if (NULL != soln) { QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl; out->push_back(soln); } return; } // If a sort order is requested, there may be an index that provides it, even if that // index is not over any predicates in the query. // // XXX XXX: Can we do this even if the index is sparse? Might we miss things? if (!query.getParsed().getSort().isEmpty() && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) { // See if we have a sort provided from an index already. bool usingIndexToSort = false; for (size_t i = 0; i < out->size(); ++i) { QuerySolution* soln = (*out)[i]; if (!soln->hasSortStage) { usingIndexToSort = true; break; } } if (!usingIndexToSort) { for (size_t i = 0; i < params.indices.size(); ++i) { const BSONObj& kp = params.indices[i].keyPattern; if (providesSort(query, kp)) { QLOG() << "Planner: outputting soln that uses index to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params); if (NULL != soln) { out->push_back(soln); break; } } if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) { QLOG() << "Planner: outputting soln that uses (reverse) index " << "to provide sort." << endl; QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1); if (NULL != soln) { out->push_back(soln); break; } } } } } // TODO: Do we always want to offer a collscan solution? // XXX: currently disabling the always-use-a-collscan in order to find more planner bugs. if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan))) { QuerySolution* collscan = buildCollscanSoln(query, false, params); if (NULL != collscan) { out->push_back(collscan); QLOG() << "Planner: outputting a collscan:\n"; QLOG() << collscan->toString() << endl; } } }
BSONObj UpdateStage::transformAndUpdate(const Snapshotted<BSONObj>& oldObj, RecordId& recordId) { const UpdateRequest* request = _params.request; UpdateDriver* driver = _params.driver; CanonicalQuery* cq = _params.canonicalQuery; UpdateLifecycle* lifecycle = request->getLifecycle(); // If asked to return new doc, default to the oldObj, in case nothing changes. BSONObj newObj = oldObj.value(); // Ask the driver to apply the mods. It may be that the driver can apply those "in // place", that is, some values of the old document just get adjusted without any // change to the binary layout on the bson layer. It may be that a whole new document // is needed to accomodate the new bson layout of the resulting document. In any event, // only enable in-place mutations if the underlying storage engine offers support for // writing damage events. _doc.reset(oldObj.value(), (_collection->updateWithDamagesSupported() ? mutablebson::Document::kInPlaceEnabled : mutablebson::Document::kInPlaceDisabled)); BSONObj logObj; bool docWasModified = false; Status status = Status::OK(); const bool validateForStorage = getOpCtx()->writesAreReplicated() && _enforceOkForStorage; FieldRefSet immutablePaths; if (getOpCtx()->writesAreReplicated() && !request->isFromMigration()) { if (lifecycle) { auto immutablePathsVector = getImmutableFields(getOpCtx(), request->getNamespaceString()); if (immutablePathsVector) { immutablePaths.fillFrom( transitional_tools_do_not_use::unspool_vector(*immutablePathsVector)); } } immutablePaths.keepShortest(&idFieldRef); } if (!driver->needMatchDetails()) { // If we don't need match details, avoid doing the rematch status = driver->update( StringData(), &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified); } else { // If there was a matched field, obtain it. MatchDetails matchDetails; matchDetails.requestElemMatchKey(); dassert(cq); verify(cq->root()->matchesBSON(oldObj.value(), &matchDetails)); string matchedField; if (matchDetails.hasElemMatchKey()) matchedField = matchDetails.elemMatchKey(); status = driver->update( matchedField, &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified); } if (!status.isOK()) { uasserted(16837, status.reason()); } // Skip adding _id field if the collection is capped (since capped collection documents can // neither grow nor shrink). const auto createIdField = !_collection->isCapped(); // Ensure if _id exists it is first status = ensureIdFieldIsFirst(&_doc); if (status.code() == ErrorCodes::InvalidIdField) { // Create ObjectId _id field if we are doing that if (createIdField) { addObjectIDIdField(&_doc); } } else { uassertStatusOK(status); } // See if the changes were applied in place const char* source = NULL; const bool inPlace = _doc.getInPlaceUpdates(&_damages, &source); if (inPlace && _damages.empty()) { // An interesting edge case. A modifier didn't notice that it was really a no-op // during its 'prepare' phase. That represents a missed optimization, but we still // shouldn't do any real work. Toggle 'docWasModified' to 'false'. // // Currently, an example of this is '{ $push : { x : {$each: [], $sort: 1} } }' when the 'x' // array exists and is already sorted. docWasModified = false; } if (docWasModified) { // Prepare to write back the modified document WriteUnitOfWork wunit(getOpCtx()); RecordId newRecordId; OplogUpdateEntryArgs args; if (!request->isExplain()) { invariant(_collection); auto* css = CollectionShardingState::get(getOpCtx(), _collection->ns()); args.nss = _collection->ns(); args.uuid = _collection->uuid(); args.stmtId = request->getStmtId(); args.update = logObj; args.criteria = css->getMetadata().extractDocumentKey(newObj); uassert(16980, "Multi-update operations require all documents to have an '_id' field", !request->isMulti() || args.criteria.hasField("_id"_sd)); args.fromMigrate = request->isFromMigration(); args.storeDocOption = getStoreDocMode(*request); if (args.storeDocOption == OplogUpdateEntryArgs::StoreDocOption::PreImage) { args.preImageDoc = oldObj.value().getOwned(); } } if (inPlace) { if (!request->isExplain()) { newObj = oldObj.value(); const RecordData oldRec(oldObj.value().objdata(), oldObj.value().objsize()); Snapshotted<RecordData> snap(oldObj.snapshotId(), oldRec); StatusWith<RecordData> newRecStatus = _collection->updateDocumentWithDamages( getOpCtx(), recordId, std::move(snap), source, _damages, &args); newObj = uassertStatusOK(std::move(newRecStatus)).releaseToBson(); } newRecordId = recordId; } else { // The updates were not in place. Apply them through the file manager. newObj = _doc.getObject(); uassert(17419, str::stream() << "Resulting document after update is larger than " << BSONObjMaxUserSize, newObj.objsize() <= BSONObjMaxUserSize); if (!request->isExplain()) { newRecordId = _collection->updateDocument(getOpCtx(), recordId, oldObj, newObj, true, driver->modsAffectIndices(), _params.opDebug, &args); } } invariant(oldObj.snapshotId() == getOpCtx()->recoveryUnit()->getSnapshotId()); wunit.commit(); // If the document moved, we might see it again in a collection scan (maybe it's // a document after our current document). // // If the document is indexed and the mod changes an indexed value, we might see // it again. For an example, see the comment above near declaration of // updatedRecordIds. // // This must be done after the wunit commits so we are sure we won't be rolling back. if (_updatedRecordIds && (newRecordId != recordId || driver->modsAffectIndices())) { _updatedRecordIds->insert(newRecordId); } } // Only record doc modifications if they wrote (exclude no-ops). Explains get // recorded as if they wrote. if (docWasModified || request->isExplain()) { _specificStats.nModified++; } return newObj; }