Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query,
const vector<FieldRef*>* immutablePathsPtr,
mutablebson::Document& doc) const {
EqualityMatches equalities;
Status status = Status::OK();
if (isDocReplacement()) {
FieldRefSet pathsToExtract;
// TODO: Refactor update logic, make _id just another immutable field
static const FieldRef idPath("_id");
static const vector<FieldRef*> emptyImmutablePaths;
const vector<FieldRef*>& immutablePaths =
immutablePathsPtr ? *immutablePathsPtr : emptyImmutablePaths;
pathsToExtract.fillFrom(immutablePaths);
pathsToExtract.insert(&idPath);
// Extract only immutable fields from replacement-style
status =
pathsupport::extractFullEqualityMatches(*query.root(), pathsToExtract, &equalities);
} else {
// Extract all fields from op-style
status = pathsupport::extractEqualityMatches(*query.root(), &equalities);
}
if (!status.isOK())
return status;
status = pathsupport::addEqualitiesToDoc(equalities, &doc);
return status;
}
bool SubplanStage::canUseSubplanning(const CanonicalQuery& query) {
const QueryRequest& qr = query.getQueryRequest();
const MatchExpression* expr = query.root();
// Hint provided
if (!qr.getHint().isEmpty()) {
return false;
}
// Min provided
// Min queries are a special case of hinted queries.
if (!qr.getMin().isEmpty()) {
return false;
}
// Max provided
// Similar to min, max queries are a special case of hinted queries.
if (!qr.getMax().isEmpty()) {
return false;
}
// Tailable cursors won't get cached, just turn into collscans.
if (query.getQueryRequest().isTailable()) {
return false;
}
// We can only subplan rooted $or queries, and only if they have at least one clause.
return MatchExpression::OR == expr->matchType() && expr->numChildren() > 0;
}
bool PlanCache::shouldCacheQuery(const CanonicalQuery& query) {
const LiteParsedQuery& lpq = query.getParsed();
const MatchExpression* expr = query.root();
// Collection scan
// No sort order requested
if (lpq.getSort().isEmpty() &&
expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) {
return false;
}
// Hint provided
if (!lpq.getHint().isEmpty()) {
return false;
}
// Min provided
// Min queries are a special case of hinted queries.
if (!lpq.getMin().isEmpty()) {
return false;
}
// Max provided
// Similar to min, max queries are a special case of hinted queries.
if (!lpq.getMax().isEmpty()) {
return false;
}
return true;
}
bool SubplanStage::canUseSubplanning(const CanonicalQuery& query) {
const LiteParsedQuery& lpq = query.getParsed();
const MatchExpression* expr = query.root();
// Hint provided
if (!lpq.getHint().isEmpty()) {
return false;
}
// Min provided
// Min queries are a special case of hinted queries.
if (!lpq.getMin().isEmpty()) {
return false;
}
// Max provided
// Similar to min, max queries are a special case of hinted queries.
if (!lpq.getMax().isEmpty()) {
return false;
}
// Tailable cursors won't get cached, just turn into collscans.
if (query.getParsed().isTailable()) {
return false;
}
// Snapshot is really a hint.
if (query.getParsed().isSnapshot()) {
return false;
}
// TODO: For now we only allow rooted OR. We should consider also allowing contained OR that
// does not have a TEXT or GEO_NEAR node.
return MatchExpression::OR == expr->matchType();
}
// static
Status QueryPlanner::planFromCache(const CanonicalQuery& query,
const QueryPlannerParams& params,
CachedSolution* cachedSoln,
QuerySolution** out) {
// Create a copy of the expression tree. We use cachedSoln to annotate this with indices.
MatchExpression* clone = query.root()->shallowClone();
// XXX: Use data in cachedSoln to tag 'clone' with the indices used. The tags use an index
// ID which is an index into some vector of IndexEntry(s). How do we maintain this across
// calls to plan? Do we want to store in the soln the keypatterns of the indices and just
// map those to an index into params.indices? Might be easiest thing to do, and certainly
// most intelligible for debugging.
// Use the cached index assignments to build solnRoot. Takes ownership of clone.
QuerySolutionNode* solnRoot =
QueryPlannerAccess::buildIndexedDataAccess(query, clone, false, params.indices);
// XXX: are the NULL cases an error/when does this happen / can this happen?
if (NULL != solnRoot) {
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
QLOG() << "Planner: adding cached solution:\n" << soln->toString() << endl;
*out = soln;
}
}
// XXX: if any NULLs return error status?
return Status::OK();
}
PlanCacheKey PlanCache::computeKey(const CanonicalQuery& cq) const {
str::stream ss;
encodePlanCacheKeyTree(cq.root(), &ss);
encodePlanCacheKeySort(cq.getParsed().getSort(), &ss);
encodePlanCacheKeyProj(cq.getParsed().getProj(), &ss);
return ss;
}
PlanCacheKey PlanCache::computeKey(const CanonicalQuery& cq) const {
StringBuilder keyBuilder;
encodeKeyForMatch(cq.root(), &keyBuilder);
encodeKeyForSort(cq.getQueryRequest().getSort(), &keyBuilder);
encodeKeyForProj(cq.getQueryRequest().getProj(), &keyBuilder);
return keyBuilder.str();
}
/**
* Cache key is a string-ified combination of the query and sort obfuscated
* for minimal user comprehension.
*/
PlanCacheKey PlanCache::getPlanCacheKey(const CanonicalQuery& query) {
stringstream ss;
encodePlanCacheKeyTree(query.root(), &ss);
encodePlanCacheKeySort(query.getParsed().getSort(), &ss);
encodePlanCacheKeyProj(query.getParsed().getProj(), &ss);
PlanCacheKey key(ss.str());
return key;
}
Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query,
const FieldRefSet& immutablePaths,
mutablebson::Document& doc) const {
EqualityMatches equalities;
Status status = Status::OK();
if (_updateType == UpdateType::kReplacement) {
// Extract only immutable fields.
status =
pathsupport::extractFullEqualityMatches(*query.root(), immutablePaths, &equalities);
} else {
// Extract all fields from op-style update.
status = pathsupport::extractEqualityMatches(*query.root(), &equalities);
}
if (!status.isOK())
return status;
status = pathsupport::addEqualitiesToDoc(equalities, &doc);
return status;
}
// static
void Explain::generatePlannerInfo(PlanExecutor* exec,
PlanStageStats* winnerStats,
const vector<PlanStageStats*>& rejectedStats,
BSONObjBuilder* out) {
CanonicalQuery* query = exec->getCanonicalQuery();
BSONObjBuilder plannerBob(out->subobjStart("queryPlanner"));
;
plannerBob.append("plannerVersion", QueryPlanner::kPlannerVersion);
plannerBob.append("namespace", exec->ns());
// Find whether there is an index filter set for the query shape. The 'indexFilterSet'
// field will always be false in the case of EOF or idhack plans.
bool indexFilterSet = false;
if (exec->collection() && exec->getCanonicalQuery()) {
const CollectionInfoCache* infoCache = exec->collection()->infoCache();
const QuerySettings* querySettings = infoCache->getQuerySettings();
PlanCacheKey planCacheKey =
infoCache->getPlanCache()->computeKey(*exec->getCanonicalQuery());
AllowedIndices* allowedIndicesRaw;
if (querySettings->getAllowedIndices(planCacheKey, &allowedIndicesRaw)) {
// Found an index filter set on the query shape.
std::unique_ptr<AllowedIndices> allowedIndices(allowedIndicesRaw);
indexFilterSet = true;
}
}
plannerBob.append("indexFilterSet", indexFilterSet);
// In general we should have a canonical query, but sometimes we may avoid
// creating a canonical query as an optimization (specifically, the update system
// does not canonicalize for idhack updates). In these cases, 'query' is NULL.
if (NULL != query) {
BSONObjBuilder parsedQueryBob(plannerBob.subobjStart("parsedQuery"));
query->root()->toBSON(&parsedQueryBob);
parsedQueryBob.doneFast();
}
BSONObjBuilder winningPlanBob(plannerBob.subobjStart("winningPlan"));
statsToBSON(*winnerStats, &winningPlanBob, ExplainCommon::QUERY_PLANNER);
winningPlanBob.doneFast();
// Genenerate array of rejected plans.
BSONArrayBuilder allPlansBob(plannerBob.subarrayStart("rejectedPlans"));
for (size_t i = 0; i < rejectedStats.size(); i++) {
BSONObjBuilder childBob(allPlansBob.subobjStart());
statsToBSON(*rejectedStats[i], &childBob, ExplainCommon::QUERY_PLANNER);
}
allPlansBob.doneFast();
plannerBob.doneFast();
}
void ChunkManager::getShardIdsForQuery(set<ShardId>& shardIds, const BSONObj& query) const {
CanonicalQuery* canonicalQuery = NULL;
Status status = CanonicalQuery::canonicalize(
_ns,
query,
&canonicalQuery,
WhereCallbackNoop());
boost::scoped_ptr<CanonicalQuery> canonicalQueryPtr(canonicalQuery);
uassert(status.code(), status.reason(), status.isOK());
// Query validation
if (QueryPlannerCommon::hasNode(canonicalQuery->root(), MatchExpression::GEO_NEAR)) {
uassert(13501, "use geoNear command rather than $near query", false);
}
// Transforms query into bounds for each field in the shard key
// for example :
// Key { a: 1, b: 1 },
// Query { a : { $gte : 1, $lt : 2 },
// b : { $gte : 3, $lt : 4 } }
// => Bounds { a : [1, 2), b : [3, 4) }
IndexBounds bounds = getIndexBoundsForQuery(_keyPattern.toBSON(), canonicalQuery);
// Transforms bounds for each shard key field into full shard key ranges
// for example :
// Key { a : 1, b : 1 }
// Bounds { a : [1, 2), b : [3, 4) }
// => Ranges { a : 1, b : 3 } => { a : 2, b : 4 }
BoundList ranges = _keyPattern.flattenBounds(bounds);
for (BoundList::const_iterator it = ranges.begin(); it != ranges.end();
++it) {
getShardIdsForRange(shardIds, it->first /*min*/, it->second /*max*/);
// once we know we need to visit all shards no need to keep looping
if(shardIds.size() == _shardIds.size()) break;
}
// SERVER-4914 Some clients of getShardIdsForQuery() assume at least one shard will be
// returned. For now, we satisfy that assumption by adding a shard with no matches rather
// than return an empty set of shards.
if (shardIds.empty()) {
massert( 16068, "no chunk ranges available", !_chunkRanges.ranges().empty() );
shardIds.insert(_chunkRanges.ranges().begin()->second->getShardId());
}
}
IndexBounds ChunkManager::getIndexBoundsForQuery(const BSONObj& key,
const CanonicalQuery& canonicalQuery) {
// $text is not allowed in planning since we don't have text index on mongos.
//
// TODO: Treat $text query as a no-op in planning. So with shard key {a: 1},
// the query { a: 2, $text: { ... } } will only target to {a: 2}.
if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::TEXT)) {
IndexBounds bounds;
IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey]
return bounds;
}
// Consider shard key as an index
string accessMethod = IndexNames::findPluginName(key);
dassert(accessMethod == IndexNames::BTREE || accessMethod == IndexNames::HASHED);
// Use query framework to generate index bounds
QueryPlannerParams plannerParams;
// Must use "shard key" index
plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN;
IndexEntry indexEntry(key,
accessMethod,
false /* multiKey */,
false /* sparse */,
false /* unique */,
"shardkey",
NULL /* filterExpr */,
BSONObj());
plannerParams.indices.push_back(indexEntry);
OwnedPointerVector<QuerySolution> solutions;
Status status = QueryPlanner::plan(canonicalQuery, plannerParams, &solutions.mutableVector());
uassert(status.code(), status.reason(), status.isOK());
IndexBounds bounds;
for (vector<QuerySolution*>::const_iterator it = solutions.begin();
bounds.size() == 0 && it != solutions.end();
it++) {
// Try next solution if we failed to generate index bounds, i.e. bounds.size() == 0
bounds = collapseQuerySolution((*it)->root.get());
}
if (bounds.size() == 0) {
// We cannot plan the query without collection scan, so target to all shards.
IndexBoundsBuilder::allValuesBounds(key, &bounds); // [minKey, maxKey]
}
return bounds;
}
bool PlanCache::shouldCacheQuery(const CanonicalQuery& query) {
const LiteParsedQuery& lpq = query.getParsed();
const MatchExpression* expr = query.root();
// Collection scan
// No sort order requested
if (lpq.getSort().isEmpty() &&
expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) {
return false;
}
// Hint provided
if (!lpq.getHint().isEmpty()) {
return false;
}
// Min provided
// Min queries are a special case of hinted queries.
if (!lpq.getMin().isEmpty()) {
return false;
}
// Max provided
// Similar to min, max queries are a special case of hinted queries.
if (!lpq.getMax().isEmpty()) {
return false;
}
// Explain queries are not-cacheable. This is primarily because of
// the need to generate current and accurate information in allPlans.
// If the explain report is generated by the cached plan runner using
// stale information from the cache for the losing plans, allPlans would
// simply be wrong.
if (lpq.isExplain()) {
return false;
}
// Tailable cursors won't get cached, just turn into collscans.
if (query.getParsed().isTailable()) {
return false;
}
// Snapshot is really a hint.
if (query.getParsed().isSnapshot()) {
return false;
}
return true;
}
bool PlanCache::shouldCacheQuery(const CanonicalQuery& query) {
const QueryRequest& qr = query.getQueryRequest();
const MatchExpression* expr = query.root();
// Collection scan
// No sort order requested
if (qr.getSort().isEmpty() && expr->matchType() == MatchExpression::AND &&
expr->numChildren() == 0) {
return false;
}
// Hint provided
if (!qr.getHint().isEmpty()) {
return false;
}
// Min provided
// Min queries are a special case of hinted queries.
if (!qr.getMin().isEmpty()) {
return false;
}
// Max provided
// Similar to min, max queries are a special case of hinted queries.
if (!qr.getMax().isEmpty()) {
return false;
}
// We don't read or write from the plan cache for explain. This ensures
// that explain queries don't affect cache state, and it also makes
// sure that we can always generate information regarding rejected plans
// and/or trial period execution of candidate plans.
if (qr.isExplain()) {
return false;
}
// Tailable cursors won't get cached, just turn into collscans.
if (query.getQueryRequest().isTailable()) {
return false;
}
// Snapshot is really a hint.
if (query.getQueryRequest().isSnapshot()) {
return false;
}
return true;
}
StatusWith<BSONObj> ShardKeyPattern::extractShardKeyFromQuery(const CanonicalQuery& query) const {
if (!isValid())
return StatusWith<BSONObj>(BSONObj());
// Extract equalities from query.
EqualityMatches equalities;
// TODO: Build the path set initially?
FieldRefSet keyPatternPathSet(_keyPatternPaths.vector());
// We only care about extracting the full key pattern paths - if they don't exist (or are
// conflicting), we don't contain the shard key.
Status eqStatus =
pathsupport::extractFullEqualityMatches(*query.root(), keyPatternPathSet, &equalities);
// NOTE: Failure to extract equality matches just means we return no shard key - it's not
// an error we propagate
if (!eqStatus.isOK())
return StatusWith<BSONObj>(BSONObj());
// Extract key from equalities
// NOTE: The method below is equivalent to constructing a BSONObj and running
// extractShardKeyFromMatchable, but doesn't require creating the doc.
BSONObjBuilder keyBuilder;
// Iterate the parsed paths to avoid re-parsing
for (OwnedPointerVector<FieldRef>::const_iterator it = _keyPatternPaths.begin();
it != _keyPatternPaths.end();
++it) {
const FieldRef& patternPath = **it;
BSONElement equalEl = findEqualityElement(equalities, patternPath);
if (!isShardKeyElement(equalEl, false))
return StatusWith<BSONObj>(BSONObj());
if (isHashedPattern()) {
keyBuilder.append(
patternPath.dottedField(),
BSONElementHasher::hash64(equalEl, BSONElementHasher::DEFAULT_HASH_SEED));
} else {
// NOTE: The equal element may *not* have the same field name as the path -
// nested $and, $eq, for example
keyBuilder.appendAs(equalEl, patternPath.dottedField());
}
}
dassert(isShardKey(keyBuilder.asTempObj()));
return StatusWith<BSONObj>(keyBuilder.obj());
}
// static
bool SubplanRunner::canUseSubplanRunner(const CanonicalQuery& query) {
const LiteParsedQuery& lpq = query.getParsed();
const MatchExpression* expr = query.root();
// Only rooted ORs work with the subplan scheme.
if (MatchExpression::OR != expr->matchType()) {
return false;
}
// Collection scan
// No sort order requested
if (lpq.getSort().isEmpty() &&
expr->matchType() == MatchExpression::AND && expr->numChildren() == 0) {
return false;
}
// Hint provided
if (!lpq.getHint().isEmpty()) {
return false;
}
// Min provided
// Min queries are a special case of hinted queries.
if (!lpq.getMin().isEmpty()) {
return false;
}
// Max provided
// Similar to min, max queries are a special case of hinted queries.
if (!lpq.getMax().isEmpty()) {
return false;
}
// Tailable cursors won't get cached, just turn into collscans.
if (query.getParsed().hasOption(QueryOption_CursorTailable)) {
return false;
}
// Snapshot is really a hint.
if (query.getParsed().isSnapshot()) {
return false;
}
return true;
}
// static
bool SubplanStage::canUseSubplanning(const CanonicalQuery& query) {
const LiteParsedQuery& lpq = query.getParsed();
const MatchExpression* expr = query.root();
// Only rooted ORs work with the subplan scheme.
if (MatchExpression::OR != expr->matchType()) {
return false;
}
// Hint provided
if (!lpq.getHint().isEmpty()) {
return false;
}
// Min provided
// Min queries are a special case of hinted queries.
if (!lpq.getMin().isEmpty()) {
return false;
}
// Max provided
// Similar to min, max queries are a special case of hinted queries.
if (!lpq.getMax().isEmpty()) {
return false;
}
// Tailable cursors won't get cached, just turn into collscans.
if (query.getParsed().getOptions().tailable) {
return false;
}
// Snapshot is really a hint.
if (query.getParsed().isSnapshot()) {
return false;
}
return true;
}
Status getRunnerDistinct(Collection* collection,
const BSONObj& query,
const string& field,
Runner** out) {
// This should'a been checked by the distinct command.
verify(collection);
// TODO: check for idhack here?
// When can we do a fast distinct hack?
// 1. There is a plan with just one leaf and that leaf is an ixscan.
// 2. The ixscan indexes the field we're interested in.
// 2a: We are correct if the index contains the field but for now we look for prefix.
// 3. The query is covered/no fetch.
//
// We go through normal planning (with limited parameters) to see if we can produce
// a soln with the above properties.
QueryPlannerParams plannerParams;
plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN;
IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false);
while (ii.more()) {
const IndexDescriptor* desc = ii.next();
// The distinct hack can work if any field is in the index but it's not always clear
// if it's a win unless it's the first field.
if (desc->keyPattern().firstElement().fieldName() == field) {
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName(),
desc->infoObj()));
}
}
// We only care about the field that we're projecting over. Have to drop the _id field
// explicitly because those are .find() semantics.
//
// Applying a projection allows the planner to try to give us covered plans.
BSONObj projection;
if ("_id" == field) {
projection = BSON("_id" << 1);
}
else {
projection = BSON("_id" << 0 << field << 1);
}
// Apply a projection of the key. Empty BSONObj() is for the sort.
CanonicalQuery* cq;
Status status = CanonicalQuery::canonicalize(collection->ns().ns(), query, BSONObj(), projection, &cq);
if (!status.isOK()) {
return status;
}
// No index has the field we're looking for. Punt to normal planning.
if (plannerParams.indices.empty()) {
// Takes ownership of cq.
return getRunner(cq, out);
}
// If we're here, we have an index prefixed by the field we're distinct-ing over.
// If there's no query, we can just distinct-scan one of the indices.
if (query.isEmpty()) {
DistinctNode* dn = new DistinctNode();
dn->indexKeyPattern = plannerParams.indices[0].keyPattern;
dn->direction = 1;
IndexBoundsBuilder::allValuesBounds(dn->indexKeyPattern, &dn->bounds);
dn->fieldNo = 0;
QueryPlannerParams params;
// Takes ownership of 'dn'.
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*cq, params, dn);
verify(soln);
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*soln, &root, &ws));
*out = new SingleSolutionRunner(collection, cq, soln, root, ws);
return Status::OK();
}
// See if we can answer the query in a fast-distinct compatible fashion.
vector<QuerySolution*> solutions;
status = QueryPlanner::plan(*cq, plannerParams, &solutions);
if (!status.isOK()) {
return getRunner(cq, out);
}
// XXX: why do we need to do this? planner should prob do this internally
cq->root()->resetTag();
// We look for a solution that has an ixscan we can turn into a distinctixscan
for (size_t i = 0; i < solutions.size(); ++i) {
if (turnIxscanIntoDistinctIxscan(solutions[i], field)) {
// Great, we can use solutions[i]. Clean up the other QuerySolution(s).
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
// Build and return the SSR over solutions[i].
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
*out = new SingleSolutionRunner(collection, cq, solutions[i], root, ws);
return Status::OK();
}
}
// If we're here, the planner made a soln with the restricted index set but we couldn't
// translate any of them into a distinct-compatible soln. So, delete the solutions and just
// go through normal planning.
for (size_t i = 0; i < solutions.size(); ++i) {
delete solutions[i];
}
return getRunner(cq, out);
}
// static
StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
const CanonicalQuery& query, const QueryPlannerParams& params) {
LOG(5) << "Beginning planning..." << endl
<< "=============================" << endl
<< "Options = " << optionString(params.options) << endl
<< "Canonical query:" << endl
<< redact(query.toString()) << "=============================";
std::vector<std::unique_ptr<QuerySolution>> out;
for (size_t i = 0; i < params.indices.size(); ++i) {
LOG(5) << "Index " << i << " is " << params.indices[i].toString();
}
const bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
const bool isTailable = query.getQueryRequest().isTailable();
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan.
if (isTailable) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) {
auto soln = buildCollscanSoln(query, isTailable, params);
if (soln) {
out.push_back(std::move(soln));
}
}
return {std::move(out)};
}
// The hint or sort can be $natural: 1. If this happens, output a collscan. If both
// a $natural hint and a $natural sort are specified, then the direction of the collscan
// is determined by the sign of the sort (not the sign of the hint).
if (!query.getQueryRequest().getHint().isEmpty() ||
!query.getQueryRequest().getSort().isEmpty()) {
BSONObj hintObj = query.getQueryRequest().getHint();
BSONObj sortObj = query.getQueryRequest().getSort();
BSONElement naturalHint = dps::extractElementAtPath(hintObj, "$natural");
BSONElement naturalSort = dps::extractElementAtPath(sortObj, "$natural");
// A hint overrides a $natural sort. This means that we don't force a table
// scan if there is a $natural sort with a non-$natural hint.
if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) {
LOG(5) << "Forcing a table scan due to hinted $natural";
// min/max are incompatible with $natural.
if (canTableScan && query.getQueryRequest().getMin().isEmpty() &&
query.getQueryRequest().getMax().isEmpty()) {
auto soln = buildCollscanSoln(query, isTailable, params);
if (soln) {
out.push_back(std::move(soln));
}
}
return {std::move(out)};
}
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
LOG(5) << "Predicate over field '" << *it << "'";
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
// If index filters in the query settings were used to override
// the allowed indices for planning, we should not use the hinted index
// requested in the query.
BSONObj hintIndex;
if (!params.indexFiltersApplied) {
hintIndex = query.getQueryRequest().getHint();
}
// If snapshot is set, default to collscanning. If the query param SNAPSHOT_USE_ID is set,
// snapshot is a form of a hint, so try to use _id index to make a real plan. If that fails,
// just scan the _id index.
//
// Don't do this if the query is a geonear or text as as text search queries must be answered
// using full text indices and geoNear queries must be answered using geospatial indices.
if (query.getQueryRequest().isSnapshot()) {
RARELY {
warning() << "The snapshot option is deprecated. See "
"http://dochub.mongodb.org/core/snapshot-deprecation";
}
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) &&
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
const bool useIXScan = params.options & QueryPlannerParams::SNAPSHOT_USE_ID;
if (!useIXScan) {
auto soln = buildCollscanSoln(query, isTailable, params);
if (soln) {
out.push_back(std::move(soln));
}
return {std::move(out)};
} else {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
}
}
// static
Status QueryPlanner::plan(const CanonicalQuery& query,
const QueryPlannerParams& params,
std::vector<QuerySolution*>* out) {
QLOG() << "=============================\n"
<< "Beginning planning, options = " << optionString(params.options) << endl
<< "Canonical query:\n" << query.toString() << endl
<< "============================="
<< endl;
for (size_t i = 0; i < params.indices.size(); ++i) {
QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl;
}
bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan. TODO: This is a policy departure. Previously I think you
// could ask for a tailable cursor and it just tried to give you one. Now, we fail if we
// can't provide one. Is this what we want?
if (query.getParsed().hasOption(QueryOption_CursorTailable)) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, true, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
// The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of
// saying "table scan for two, please."
if (!query.getParsed().getHint().isEmpty()) {
BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural");
if (!natural.eoo()) {
QLOG() << "forcing a table scan due to hinted $natural\n";
// min/max are incompatible with $natural.
if (canTableScan && query.getParsed().getMin().isEmpty()
&& query.getParsed().getMax().isEmpty()) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
QLOG() << "predicate over field " << *it << endl;
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
BSONObj hintIndex = query.getParsed().getHint();
// Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real
// plan. If that fails, just scan the _id index.
if (query.getParsed().isSnapshot()) {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
size_t hintIndexNumber = numeric_limits<size_t>::max();
if (hintIndex.isEmpty()) {
QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices);
}
else {
// Sigh. If the hint is specified it might be using the index name.
BSONElement firstHintElt = hintIndex.firstElement();
if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) {
string hintName = firstHintElt.String();
for (size_t i = 0; i < params.indices.size(); ++i) {
if (params.indices[i].name == hintName) {
QLOG() << "hint by name specified, restricting indices to "
<< params.indices[i].keyPattern.toString() << endl;
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
hintIndexNumber = i;
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
else {
for (size_t i = 0; i < params.indices.size(); ++i) {
if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) {
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
QLOG() << "hint specified, restricting indices to " << hintIndex.toString()
<< endl;
hintIndexNumber = i;
break;
}
}
}
if (hintIndexNumber == numeric_limits<size_t>::max()) {
return Status(ErrorCodes::BadValue, "bad hint");
}
}
// Deal with the .min() and .max() query options. If either exist we can only use an index
// that matches the object inside.
if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) {
BSONObj minObj = query.getParsed().getMin();
BSONObj maxObj = query.getParsed().getMax();
// This is the index into params.indices[...] that we use.
size_t idxNo = numeric_limits<size_t>::max();
// If there's an index hinted we need to be able to use it.
if (!hintIndex.isEmpty()) {
if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) {
QLOG() << "minobj doesnt work w hint";
return Status(ErrorCodes::BadValue,
"hint provided does not work with min query");
}
if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) {
QLOG() << "maxobj doesnt work w hint";
return Status(ErrorCodes::BadValue,
"hint provided does not work with max query");
}
idxNo = hintIndexNumber;
}
else {
// No hinted index, look for one that is compatible (has same field names and
// ordering thereof).
for (size_t i = 0; i < params.indices.size(); ++i) {
const BSONObj& kp = params.indices[i].keyPattern;
BSONObj toUse = minObj.isEmpty() ? maxObj : minObj;
if (indexCompatibleMaxMin(toUse, kp)) {
idxNo = i;
break;
}
}
}
if (idxNo == numeric_limits<size_t>::max()) {
QLOG() << "Can't find relevant index to use for max/min query";
// Can't find an index to use, bail out.
return Status(ErrorCodes::BadValue,
"unable to find relevant index for max/min query");
}
// maxObj can be empty; the index scan just goes until the end. minObj can't be empty
// though, so if it is, we make a minKey object.
if (minObj.isEmpty()) {
BSONObjBuilder bob;
bob.appendMinKey("");
minObj = bob.obj();
}
else {
// Must strip off the field names to make an index key.
minObj = stripFieldNames(minObj);
}
if (!maxObj.isEmpty()) {
// Must strip off the field names to make an index key.
maxObj = stripFieldNames(maxObj);
}
QLOG() << "max/min query using index " << params.indices[idxNo].toString() << endl;
// Make our scan and output.
QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(params.indices[idxNo],
query,
params,
minObj,
maxObj);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
return Status::OK();
}
for (size_t i = 0; i < relevantIndices.size(); ++i) {
QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl;
}
// Figure out how useful each index is to each predicate.
// query.root() is now annotated with RelevantTag(s).
QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices);
QLOG() << "rated tree" << endl;
QLOG() << query.root()->toString() << endl;
// If there is a GEO_NEAR it must have an index it can use directly.
// XXX: move into data access?
MatchExpression* gnNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) {
// No index for GEO_NEAR? No query.
RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
QLOG() << "unable to find index for $geoNear query" << endl;
return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query");
}
GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode);
vector<size_t> newFirst;
// 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed
// the full query tree inside it as a matcher.
for (size_t i = 0; i < tag->first.size(); ++i) {
// GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning.
if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) {
newFirst.push_back(i);
continue;
}
// If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the
// entire tree as a filter, if possible.
GeoNear2DNode* solnRoot = new GeoNear2DNode();
solnRoot->nq = gnme->getData();
if (NULL != query.getProj()) {
solnRoot->addPointMeta = query.getProj()->wantGeoNearPoint();
solnRoot->addDistMeta = query.getProj()->wantGeoNearDistance();
}
if (MatchExpression::GEO_NEAR != query.root()->matchType()) {
// root is an AND, clone and delete the GEO_NEAR child.
MatchExpression* filterTree = query.root()->shallowClone();
verify(MatchExpression::AND == filterTree->matchType());
bool foundChild = false;
for (size_t i = 0; i < filterTree->numChildren(); ++i) {
if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) {
foundChild = true;
filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i);
break;
}
}
verify(foundChild);
solnRoot->filter.reset(filterTree);
}
solnRoot->numWanted = query.getParsed().getNumToReturn();
if (0 == solnRoot->numWanted) {
solnRoot->numWanted = 100;
}
solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern;
// Remove the 2d index. 2d can only be the first field, and we know there is
// only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll
// only be first for gnNode.
tag->first.erase(tag->first.begin() + i);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
}
// Continue planning w/non-2d indices tagged for this pred.
tag->first.swap(newFirst);
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return Status::OK();
}
}
// Likewise, if there is a TEXT it must have an index it can use directly.
MatchExpression* textNode;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) {
RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return Status::OK();
}
}
// If we have any relevant indices, we try to create indexed plans.
if (0 < relevantIndices.size()) {
// The enumerator spits out trees tagged with IndexTag(s).
PlanEnumeratorParams enumParams;
enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION;
enumParams.root = query.root();
enumParams.indices = &relevantIndices;
PlanEnumerator isp(enumParams);
isp.init();
MatchExpression* rawTree;
// XXX: have limit on # of indexed solns we'll consider. We could have a perverse
// query and index that could make n^2 very unpleasant.
while (isp.getNext(&rawTree)) {
QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString()
<< endl;
// This can fail if enumeration makes a mistake.
QuerySolutionNode* solnRoot =
QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices);
if (NULL == solnRoot) { continue; }
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
QLOG() << "Planner: adding solution:\n" << soln->toString() << endl;
out->push_back(soln);
}
}
}
QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n";
// An index was hinted. If there are any solutions, they use the hinted index. If not, we
// scan the entire index to provide results and output that as our plan. This is the
// desired behavior when an index is hinted that is not relevant to the query.
if (!hintIndex.isEmpty()) {
if (0 == out->size()) {
QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params);
verify(NULL != soln);
QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl;
out->push_back(soln);
}
return Status::OK();
}
// If a sort order is requested, there may be an index that provides it, even if that
// index is not over any predicates in the query.
//
if (!query.getParsed().getSort().isEmpty()
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
// See if we have a sort provided from an index already.
bool usingIndexToSort = false;
for (size_t i = 0; i < out->size(); ++i) {
QuerySolution* soln = (*out)[i];
if (!soln->hasSortStage) {
usingIndexToSort = true;
break;
}
}
if (!usingIndexToSort) {
for (size_t i = 0; i < params.indices.size(); ++i) {
const IndexEntry& index = params.indices[i];
if (index.sparse) {
continue;
}
const BSONObj kp = LiteParsedQuery::normalizeSortOrder(index.keyPattern);
if (providesSort(query, kp)) {
QLOG() << "Planner: outputting soln that uses index to provide sort."
<< endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
QLOG() << "Planner: outputting soln that uses (reverse) index "
<< "to provide sort." << endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
}
}
}
// TODO: Do we always want to offer a collscan solution?
// XXX: currently disabling the always-use-a-collscan in order to find more planner bugs.
if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)
&& hintIndex.isEmpty()
&& ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan)))
{
QuerySolution* collscan = buildCollscanSoln(query, false, params);
if (NULL != collscan) {
out->push_back(collscan);
QLOG() << "Planner: outputting a collscan:\n";
QLOG() << collscan->toString() << endl;
}
}
return Status::OK();
}
// static
void QueryPlanner::plan(const CanonicalQuery& query, const vector<BSONObj>& indexKeyPatterns,
vector<QuerySolution*>* out) {
// XXX: If pq.hasOption(QueryOption_OplogReplay) use FindingStartCursor equivalent which
// must be translated into stages.
//
// Planner Section 1: Calculate predicate/index data.
//
// Get all the predicates (and their fields).
PredicateMap predicates;
makePredicateMap(query.root(), &predicates);
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan. TODO: This is a policy departure. Previously I think you
// could ask for a tailable cursor and it just tried to give you one. Now, we fail if we
// can't provide one. Is this what we want?
if (query.getParsed().hasOption(QueryOption_CursorTailable)) {
if (!hasPredicate(predicates, MatchExpression::GEO_NEAR)) {
out->push_back(makeCollectionScan(query, true));
}
return;
}
// NOR and NOT we can't handle well with indices. If we see them here, they weren't
// rewritten. Just output a collscan for those.
if (hasPredicate(predicates, MatchExpression::NOT)
|| hasPredicate(predicates, MatchExpression::NOR)) {
// If there's a near predicate, we can't handle this.
// TODO: Should canonicalized query detect this?
if (hasPredicate(predicates, MatchExpression::GEO_NEAR)) {
warning() << "Can't handle NOT/NOR with GEO_NEAR";
return;
}
out->push_back(makeCollectionScan(query, false));
return;
}
// Filter our indices so we only look at indices that are over our predicates.
vector<BSONObj> relevantIndices;
findRelevantIndices(predicates, indexKeyPatterns, &relevantIndices);
// No indices, no work to do.
if (0 == relevantIndices.size()) { return; }
// Figure out how useful each index is to each predicate.
rateIndices(relevantIndices, &predicates);
//
// Planner Section 2: Use predicate/index data to output sets of indices that we can use.
//
PlanEnumerator isp(&query, &predicates, &relevantIndices);
MatchExpression* rawTree;
while (isp.getNext(&rawTree)) {
QuerySolutionNode* solutionRoot = NULL;
//
// Planner Section 3: Logical Rewrite. Use the index selection and the tree structure
// to try to rewrite the tree. TODO: Do this for real. We treat the tree as static.
//
//
// Planner Section 4: Covering. If we're projecting, See if we get any covering from
// this plan. If not, add a fetch.
//
if (!query.getParsed().getProj().isEmpty()) {
warning() << "Can't deal with proj yet" << endl;
}
else {
// Note that we need a fetch, possibly tack on to end?
}
//
// Planner Section 5: Sort. If we're sorting, see if the plan gives us a sort for free.
// If not, add a sort.
//
if (!query.getParsed().getSort().isEmpty()) {
}
else {
// Note that we need a sort, possibly tack on to end? may want to see if sort is
// covered and then tack fetch on after the covered sort...
}
//
// Planner Section 6: Final check. Make sure that we build a valid solution.
// TODO: Validate.
//
if (NULL != solutionRoot) {
QuerySolution* qs = new QuerySolution();
qs->root.reset(solutionRoot);
out->push_back(qs);
}
}
// TODO: Do we always want to offer a collscan solution?
if (!hasPredicate(predicates, MatchExpression::GEO_NEAR)) {
out->push_back(makeCollectionScan(query, false));
}
}
void UpdateStage::transformAndUpdate(BSONObj& oldObj, DiskLoc& loc) {
const UpdateRequest* request = _params.request;
UpdateDriver* driver = _params.driver;
CanonicalQuery* cq = _params.canonicalQuery;
UpdateLifecycle* lifecycle = request->getLifecycle();
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new
// document is needed to accomodate the new bson layout of the resulting document.
_doc.reset(oldObj, mutablebson::Document::kInPlaceEnabled);
BSONObj logObj;
FieldRefSet updatedFields;
Status status = Status::OK();
if (!driver->needMatchDetails()) {
// If we don't need match details, avoid doing the rematch
status = driver->update(StringData(), &_doc, &logObj, &updatedFields);
}
else {
// If there was a matched field, obtain it.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
dassert(cq);
verify(cq->root()->matchesBSON(oldObj, &matchDetails));
string matchedField;
if (matchDetails.hasElemMatchKey())
matchedField = matchDetails.elemMatchKey();
// TODO: Right now, each mod checks in 'prepare' that if it needs positional
// data, that a non-empty StringData() was provided. In principle, we could do
// that check here in an else clause to the above conditional and remove the
// checks from the mods.
status = driver->update(matchedField, &_doc, &logObj, &updatedFields);
}
if (!status.isOK()) {
uasserted(16837, status.reason());
}
// Ensure _id exists and is first
uassertStatusOK(ensureIdAndFirst(_doc));
// If the driver applied the mods in place, we can ask the mutable for what
// changed. We call those changes "damages". :) We use the damages to inform the
// journal what was changed, and then apply them to the original document
// ourselves. If, however, the driver applied the mods out of place, we ask it to
// generate a new, modified document for us. In that case, the file manager will
// take care of the journaling details for us.
//
// This code flow is admittedly odd. But, right now, journaling is baked in the file
// manager. And if we aren't using the file manager, we have to do jounaling
// ourselves.
bool docWasModified = false;
BSONObj newObj;
const char* source = NULL;
bool inPlace = _doc.getInPlaceUpdates(&_damages, &source);
// If something changed in the document, verify that no immutable fields were changed
// and data is valid for storage.
if ((!inPlace || !_damages.empty()) ) {
if (!(request->isFromReplication() || request->isFromMigration())) {
const std::vector<FieldRef*>* immutableFields = NULL;
if (lifecycle)
immutableFields = lifecycle->getImmutableFields();
uassertStatusOK(validate(oldObj,
updatedFields,
_doc,
immutableFields,
driver->modOptions()) );
}
}
// Save state before making changes
saveState();
{
WriteUnitOfWork wunit(request->getOpCtx());
if (inPlace && !driver->modsAffectIndices()) {
// If a set of modifiers were all no-ops, we are still 'in place', but there
// is no work to do, in which case we want to consider the object unchanged.
if (!_damages.empty() ) {
// Don't actually do the write if this is an explain.
if (!request->isExplain()) {
invariant(_collection);
const RecordData oldRec(oldObj.objdata(), oldObj.objsize());
_collection->updateDocumentWithDamages(request->getOpCtx(), loc,
oldRec, source, _damages);
}
docWasModified = true;
_specificStats.fastmod = true;
}
newObj = oldObj;
}
else {
// The updates were not in place. Apply them through the file manager.
newObj = _doc.getObject();
uassert(17419,
str::stream() << "Resulting document after update is larger than "
<< BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
docWasModified = true;
// Don't actually do the write if this is an explain.
if (!request->isExplain()) {
invariant(_collection);
StatusWith<DiskLoc> res = _collection->updateDocument(request->getOpCtx(),
loc,
newObj,
true,
_params.opDebug);
uassertStatusOK(res.getStatus());
DiskLoc newLoc = res.getValue();
// If the document moved, we might see it again in a collection scan (maybe it's
// a document after our current document).
//
// If the document is indexed and the mod changes an indexed value, we might see
// it again. For an example, see the comment above near declaration of
// updatedLocs.
if (_updatedLocs && (newLoc != loc || driver->modsAffectIndices())) {
_updatedLocs->insert(newLoc);
}
}
}
// Call logOp if requested, and we're not an explain.
if (request->shouldCallLogOp() && !logObj.isEmpty() && !request->isExplain()) {
BSONObj idQuery = driver->makeOplogEntryQuery(newObj, request->isMulti());
repl::logOp(request->getOpCtx(),
"u",
request->getNamespaceString().ns().c_str(),
logObj,
&idQuery,
NULL,
request->isFromMigration());
}
wunit.commit();
}
// Restore state after modification
// As restoreState may restore (recreate) cursors, make sure to restore the
// state outside of the WritUnitOfWork.
restoreState(request->getOpCtx());
// Only record doc modifications if they wrote (exclude no-ops). Explains get
// recorded as if they wrote.
if (docWasModified) {
_specificStats.nModified++;
}
}
/* ns: namespace, e.g. <database>.<collection>
pattern: the "where" clause / criteria
justOne: stop after 1 match
god: allow access to system namespaces, and don't yield
*/
long long deleteObjects(const StringData& ns, BSONObj pattern, bool justOne, bool logop, bool god) {
if (!god) {
if (ns.find( ".system.") != string::npos) {
// note a delete from system.indexes would corrupt the db if done here, as there are
// pointers into those objects in NamespaceDetails.
uassert(12050, "cannot delete from system namespace", legalClientSystemNS( ns, true ) );
}
if (ns.find('$') != string::npos) {
log() << "cannot delete from collection with reserved $ in name: " << ns << endl;
uasserted( 10100, "cannot delete from collection with reserved $ in name" );
}
}
Collection* collection = currentClient.get()->database()->getCollection(ns);
if (NULL == collection) {
return 0;
}
uassert(10101,
str::stream() << "can't remove from a capped collection: " << ns,
!collection->isCapped());
string nsForLogOp = ns.toString(); // XXX-ERH
long long nDeleted = 0;
CanonicalQuery* cq;
if (!CanonicalQuery::canonicalize(ns.toString(), pattern, &cq).isOK()) {
uasserted(17218, "Can't canonicalize query " + pattern.toString());
return 0;
}
bool canYield = !god && !QueryPlannerCommon::hasNode(cq->root(), MatchExpression::ATOMIC);
Runner* rawRunner;
if (!getRunner(cq, &rawRunner).isOK()) {
uasserted(17219, "Can't get runner for query " + pattern.toString());
return 0;
}
auto_ptr<Runner> runner(rawRunner);
auto_ptr<ScopedRunnerRegistration> safety;
if (canYield) {
safety.reset(new ScopedRunnerRegistration(runner.get()));
runner->setYieldPolicy(Runner::YIELD_AUTO);
}
DiskLoc rloc;
Runner::RunnerState state;
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(NULL, &rloc))) {
BSONObj toDelete;
// XXX: do we want to buffer docs and delete them in a group rather than
// saving/restoring state repeatedly?
runner->saveState();
collection->deleteDocument(rloc, false, false, logop ? &toDelete : NULL );
runner->restoreState();
nDeleted++;
if (logop) {
if ( toDelete.isEmpty() ) {
problem() << "deleted object without id, not logging" << endl;
}
else {
bool replJustOne = true;
logOp("d", nsForLogOp.c_str(), toDelete, 0, &replJustOne);
}
}
if (justOne) {
break;
}
if (!god) {
getDur().commitIfNeeded();
}
if (debug && god && nDeleted == 100) {
log() << "warning high number of deletes with god=true "
<< " which could use significant memory b/c we don't commit journal";
}
}
return nDeleted;
}
// static
Status QueryPlanner::plan(const CanonicalQuery& query,
const QueryPlannerParams& params,
std::vector<QuerySolution*>* out) {
LOG(5) << "Beginning planning..." << endl
<< "=============================" << endl
<< "Options = " << optionString(params.options) << endl
<< "Canonical query:" << endl
<< query.toString() << "=============================" << endl;
for (size_t i = 0; i < params.indices.size(); ++i) {
LOG(5) << "Index " << i << " is " << params.indices[i].toString() << endl;
}
bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan. TODO: This is a policy departure. Previously I think you
// could ask for a tailable cursor and it just tried to give you one. Now, we fail if we
// can't provide one. Is this what we want?
if (query.getParsed().isTailable()) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, true, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
// The hint or sort can be $natural: 1. If this happens, output a collscan. If both
// a $natural hint and a $natural sort are specified, then the direction of the collscan
// is determined by the sign of the sort (not the sign of the hint).
if (!query.getParsed().getHint().isEmpty() || !query.getParsed().getSort().isEmpty()) {
BSONObj hintObj = query.getParsed().getHint();
BSONObj sortObj = query.getParsed().getSort();
BSONElement naturalHint = hintObj.getFieldDotted("$natural");
BSONElement naturalSort = sortObj.getFieldDotted("$natural");
// A hint overrides a $natural sort. This means that we don't force a table
// scan if there is a $natural sort with a non-$natural hint.
if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) {
LOG(5) << "Forcing a table scan due to hinted $natural\n";
// min/max are incompatible with $natural.
if (canTableScan && query.getParsed().getMin().isEmpty() &&
query.getParsed().getMax().isEmpty()) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
LOG(5) << "Predicate over field '" << *it << "'" << endl;
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
// If index filters in the query settings were used to override
// the allowed indices for planning, we should not use the hinted index
// requested in the query.
BSONObj hintIndex;
if (!params.indexFiltersApplied) {
hintIndex = query.getParsed().getHint();
}
// Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real
// plan. If that fails, just scan the _id index.
if (query.getParsed().isSnapshot()) {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
size_t hintIndexNumber = numeric_limits<size_t>::max();
if (hintIndex.isEmpty()) {
QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices);
} else {
// Sigh. If the hint is specified it might be using the index name.
BSONElement firstHintElt = hintIndex.firstElement();
if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) {
string hintName = firstHintElt.String();
for (size_t i = 0; i < params.indices.size(); ++i) {
if (params.indices[i].name == hintName) {
LOG(5) << "Hint by name specified, restricting indices to "
<< params.indices[i].keyPattern.toString() << endl;
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
hintIndexNumber = i;
hintIndex = params.indices[i].keyPattern;
break;
}
}
} else {
for (size_t i = 0; i < params.indices.size(); ++i) {
if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) {
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
LOG(5) << "Hint specified, restricting indices to " << hintIndex.toString()
<< endl;
hintIndexNumber = i;
break;
}
}
}
if (hintIndexNumber == numeric_limits<size_t>::max()) {
return Status(ErrorCodes::BadValue, "bad hint");
}
}
// Deal with the .min() and .max() query options. If either exist we can only use an index
// that matches the object inside.
if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) {
BSONObj minObj = query.getParsed().getMin();
BSONObj maxObj = query.getParsed().getMax();
// The unfinished siblings of these objects may not be proper index keys because they
// may be empty objects or have field names. When an index is picked to use for the
// min/max query, these "finished" objects will always be valid index keys for the
// index's key pattern.
BSONObj finishedMinObj;
BSONObj finishedMaxObj;
// This is the index into params.indices[...] that we use.
size_t idxNo = numeric_limits<size_t>::max();
// If there's an index hinted we need to be able to use it.
if (!hintIndex.isEmpty()) {
if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) {
LOG(5) << "Minobj doesn't work with hint";
return Status(ErrorCodes::BadValue, "hint provided does not work with min query");
}
if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) {
LOG(5) << "Maxobj doesn't work with hint";
return Status(ErrorCodes::BadValue, "hint provided does not work with max query");
}
const BSONObj& kp = params.indices[hintIndexNumber].keyPattern;
finishedMinObj = finishMinObj(kp, minObj, maxObj);
finishedMaxObj = finishMaxObj(kp, minObj, maxObj);
// The min must be less than the max for the hinted index ordering.
if (0 <= finishedMinObj.woCompare(finishedMaxObj, kp, false)) {
LOG(5) << "Minobj/Maxobj don't work with hint";
return Status(ErrorCodes::BadValue,
"hint provided does not work with min/max query");
}
idxNo = hintIndexNumber;
} else {
// No hinted index, look for one that is compatible (has same field names and
// ordering thereof).
for (size_t i = 0; i < params.indices.size(); ++i) {
const BSONObj& kp = params.indices[i].keyPattern;
BSONObj toUse = minObj.isEmpty() ? maxObj : minObj;
if (indexCompatibleMaxMin(toUse, kp)) {
// In order to be fully compatible, the min has to be less than the max
// according to the index key pattern ordering. The first step in verifying
// this is "finish" the min and max by replacing empty objects and stripping
// field names.
finishedMinObj = finishMinObj(kp, minObj, maxObj);
finishedMaxObj = finishMaxObj(kp, minObj, maxObj);
// Now we have the final min and max. This index is only relevant for
// the min/max query if min < max.
if (0 >= finishedMinObj.woCompare(finishedMaxObj, kp, false)) {
// Found a relevant index.
idxNo = i;
break;
}
// This index is not relevant; move on to the next.
}
}
}
if (idxNo == numeric_limits<size_t>::max()) {
LOG(5) << "Can't find relevant index to use for max/min query";
// Can't find an index to use, bail out.
return Status(ErrorCodes::BadValue, "unable to find relevant index for max/min query");
}
LOG(5) << "Max/min query using index " << params.indices[idxNo].toString() << endl;
// Make our scan and output.
QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(
params.indices[idxNo], query, params, finishedMinObj, finishedMaxObj);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
return Status::OK();
}
for (size_t i = 0; i < relevantIndices.size(); ++i) {
LOG(2) << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl;
}
// Figure out how useful each index is to each predicate.
QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices);
QueryPlannerIXSelect::stripInvalidAssignments(query.root(), relevantIndices);
// Unless we have GEO_NEAR, TEXT, or a projection, we may be able to apply an optimization
// in which we strip unnecessary index assignments.
//
// Disallowed with projection because assignment to a non-unique index can allow the plan
// to be covered.
//
// TEXT and GEO_NEAR are special because they require the use of a text/geo index in order
// to be evaluated correctly. Stripping these "mandatory assignments" is therefore invalid.
if (query.getParsed().getProj().isEmpty() &&
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) &&
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
QueryPlannerIXSelect::stripUnneededAssignments(query.root(), relevantIndices);
}
// query.root() is now annotated with RelevantTag(s).
LOG(5) << "Rated tree:" << endl
<< query.root()->toString();
// If there is a GEO_NEAR it must have an index it can use directly.
MatchExpression* gnNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) {
// No index for GEO_NEAR? No query.
RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
LOG(5) << "Unable to find index for $geoNear query." << endl;
// Don't leave tags on query tree.
query.root()->resetTag();
return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query");
}
LOG(5) << "Rated tree after geonear processing:" << query.root()->toString();
}
// Likewise, if there is a TEXT it must have an index it can use directly.
MatchExpression* textNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) {
RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag());
// Exactly one text index required for TEXT. We need to check this explicitly because
// the text stage can't be built if no text index exists or there is an ambiguity as to
// which one to use.
size_t textIndexCount = 0;
for (size_t i = 0; i < params.indices.size(); i++) {
if (INDEX_TEXT == params.indices[i].type) {
textIndexCount++;
}
}
if (textIndexCount != 1) {
// Don't leave tags on query tree.
query.root()->resetTag();
return Status(ErrorCodes::BadValue, "need exactly one text index for $text query");
}
// Error if the text node is tagged with zero indices.
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
// Don't leave tags on query tree.
query.root()->resetTag();
return Status(ErrorCodes::BadValue,
"failed to use text index to satisfy $text query (if text index is "
"compound, are equality predicates given for all prefix fields?)");
}
// At this point, we know that there is only one text index and that the TEXT node is
// assigned to it.
invariant(1 == tag->first.size() + tag->notFirst.size());
LOG(5) << "Rated tree after text processing:" << query.root()->toString();
}
// If we have any relevant indices, we try to create indexed plans.
if (0 < relevantIndices.size()) {
// The enumerator spits out trees tagged with IndexTag(s).
PlanEnumeratorParams enumParams;
enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION;
enumParams.root = query.root();
enumParams.indices = &relevantIndices;
PlanEnumerator isp(enumParams);
isp.init();
MatchExpression* rawTree;
while (isp.getNext(&rawTree) && (out->size() < params.maxIndexedSolutions)) {
LOG(5) << "About to build solntree from tagged tree:" << endl
<< rawTree->toString();
// The tagged tree produced by the plan enumerator is not guaranteed
// to be canonically sorted. In order to be compatible with the cached
// data, sort the tagged tree according to CanonicalQuery ordering.
std::unique_ptr<MatchExpression> clone(rawTree->shallowClone());
CanonicalQuery::sortTree(clone.get());
PlanCacheIndexTree* cacheData;
Status indexTreeStatus =
cacheDataFromTaggedTree(clone.get(), relevantIndices, &cacheData);
if (!indexTreeStatus.isOK()) {
LOG(5) << "Query is not cachable: " << indexTreeStatus.reason() << endl;
}
unique_ptr<PlanCacheIndexTree> autoData(cacheData);
// This can fail if enumeration makes a mistake.
QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(
query, rawTree, false, relevantIndices, params);
if (NULL == solnRoot) {
continue;
}
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
LOG(5) << "Planner: adding solution:" << endl
<< soln->toString();
if (indexTreeStatus.isOK()) {
SolutionCacheData* scd = new SolutionCacheData();
scd->tree.reset(autoData.release());
soln->cacheData.reset(scd);
}
out->push_back(soln);
}
}
}
// Don't leave tags on query tree.
query.root()->resetTag();
LOG(5) << "Planner: outputted " << out->size() << " indexed solutions.\n";
// Produce legible error message for failed OR planning with a TEXT child.
// TODO: support collection scan for non-TEXT children of OR.
if (out->size() == 0 && textNode != NULL && MatchExpression::OR == query.root()->matchType()) {
MatchExpression* root = query.root();
for (size_t i = 0; i < root->numChildren(); ++i) {
if (textNode == root->getChild(i)) {
return Status(ErrorCodes::BadValue,
"Failed to produce a solution for TEXT under OR - "
"other non-TEXT clauses under OR have to be indexed as well.");
}
}
}
// An index was hinted. If there are any solutions, they use the hinted index. If not, we
// scan the entire index to provide results and output that as our plan. This is the
// desired behavior when an index is hinted that is not relevant to the query.
if (!hintIndex.isEmpty()) {
if (0 == out->size()) {
QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params);
verify(NULL != soln);
LOG(5) << "Planner: outputting soln that uses hinted index as scan." << endl;
out->push_back(soln);
}
return Status::OK();
}
// If a sort order is requested, there may be an index that provides it, even if that
// index is not over any predicates in the query.
//
if (!query.getParsed().getSort().isEmpty() &&
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) &&
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
// See if we have a sort provided from an index already.
// This is implied by the presence of a non-blocking solution.
bool usingIndexToSort = false;
for (size_t i = 0; i < out->size(); ++i) {
QuerySolution* soln = (*out)[i];
if (!soln->hasBlockingStage) {
usingIndexToSort = true;
break;
}
}
if (!usingIndexToSort) {
for (size_t i = 0; i < params.indices.size(); ++i) {
const IndexEntry& index = params.indices[i];
// Only regular (non-plugin) indexes can be used to provide a sort, and only
// non-sparse indexes can be used to provide a sort.
//
// TODO: Sparse indexes can't normally provide a sort, because non-indexed
// documents could potentially be missing from the result set. However, if the
// query predicate can be used to guarantee that all documents to be returned
// are indexed, then the index should be able to provide the sort.
//
// For example:
// - Sparse index {a: 1, b: 1} should be able to provide a sort for
// find({b: 1}).sort({a: 1}). SERVER-13908.
// - Index {a: 1, b: "2dsphere"} (which is "geo-sparse", if
// 2dsphereIndexVersion=2) should be able to provide a sort for
// find({b: GEO}).sort({a:1}). SERVER-10801.
if (index.type != INDEX_BTREE) {
continue;
}
if (index.sparse) {
continue;
}
// Partial indexes can only be used to provide a sort only if the query predicate is
// compatible.
if (index.filterExpr && !expression::isSubsetOf(query.root(), index.filterExpr)) {
continue;
}
const BSONObj kp = QueryPlannerAnalysis::getSortPattern(index.keyPattern);
if (providesSort(query, kp)) {
LOG(5) << "Planner: outputting soln that uses index to provide sort." << endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params);
if (NULL != soln) {
PlanCacheIndexTree* indexTree = new PlanCacheIndexTree();
indexTree->setIndexEntry(params.indices[i]);
SolutionCacheData* scd = new SolutionCacheData();
scd->tree.reset(indexTree);
scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN;
scd->wholeIXSolnDir = 1;
soln->cacheData.reset(scd);
out->push_back(soln);
break;
}
}
if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
LOG(5) << "Planner: outputting soln that uses (reverse) index "
<< "to provide sort." << endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1);
if (NULL != soln) {
PlanCacheIndexTree* indexTree = new PlanCacheIndexTree();
indexTree->setIndexEntry(params.indices[i]);
SolutionCacheData* scd = new SolutionCacheData();
scd->tree.reset(indexTree);
scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN;
scd->wholeIXSolnDir = -1;
soln->cacheData.reset(scd);
out->push_back(soln);
break;
}
}
}
}
}
// geoNear and text queries *require* an index.
// Also, if a hint is specified it indicates that we MUST use it.
bool possibleToCollscan =
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) &&
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT) && hintIndex.isEmpty();
// The caller can explicitly ask for a collscan.
bool collscanRequested = (params.options & QueryPlannerParams::INCLUDE_COLLSCAN);
// No indexed plans? We must provide a collscan if possible or else we can't run the query.
bool collscanNeeded = (0 == out->size() && canTableScan);
if (possibleToCollscan && (collscanRequested || collscanNeeded)) {
QuerySolution* collscan = buildCollscanSoln(query, false, params);
if (NULL != collscan) {
SolutionCacheData* scd = new SolutionCacheData();
scd->solnType = SolutionCacheData::COLLSCAN_SOLN;
collscan->cacheData.reset(scd);
out->push_back(collscan);
LOG(5) << "Planner: outputting a collscan:" << endl
<< collscan->toString();
}
}
return Status::OK();
}
// static
Status QueryPlanner::planFromCache(const CanonicalQuery& query,
const QueryPlannerParams& params,
const CachedSolution& cachedSoln,
QuerySolution** out) {
invariant(!cachedSoln.plannerData.empty());
invariant(out);
// A query not suitable for caching should not have made its way into the cache.
invariant(PlanCache::shouldCacheQuery(query));
// Look up winning solution in cached solution's array.
const SolutionCacheData& winnerCacheData = *cachedSoln.plannerData[0];
if (SolutionCacheData::WHOLE_IXSCAN_SOLN == winnerCacheData.solnType) {
// The solution can be constructed by a scan over the entire index.
QuerySolution* soln = buildWholeIXSoln(
*winnerCacheData.tree->entry, query, params, winnerCacheData.wholeIXSolnDir);
if (soln == NULL) {
return Status(ErrorCodes::BadValue,
"plan cache error: soln that uses index to provide sort");
} else {
*out = soln;
return Status::OK();
}
} else if (SolutionCacheData::COLLSCAN_SOLN == winnerCacheData.solnType) {
// The cached solution is a collection scan. We don't cache collscans
// with tailable==true, hence the false below.
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (soln == NULL) {
return Status(ErrorCodes::BadValue, "plan cache error: collection scan soln");
} else {
*out = soln;
return Status::OK();
}
}
// SolutionCacheData::USE_TAGS_SOLN == cacheData->solnType
// If we're here then this is neither the whole index scan or collection scan
// cases, and we proceed by using the PlanCacheIndexTree to tag the query tree.
// Create a copy of the expression tree. We use cachedSoln to annotate this with indices.
unique_ptr<MatchExpression> clone = std::move(query.root()->shallowClone());
LOG(5) << "Tagging the match expression according to cache data: " << endl
<< "Filter:" << endl
<< clone->toString() << "Cache data:" << endl
<< winnerCacheData.toString();
// Map from index name to index number.
// TODO: can we assume that the index numbering has the same lifetime
// as the cache state?
map<BSONObj, size_t> indexMap;
for (size_t i = 0; i < params.indices.size(); ++i) {
const IndexEntry& ie = params.indices[i];
indexMap[ie.keyPattern] = i;
LOG(5) << "Index " << i << ": " << ie.keyPattern.toString() << endl;
}
Status s = tagAccordingToCache(clone.get(), winnerCacheData.tree.get(), indexMap);
if (!s.isOK()) {
return s;
}
// The planner requires a defined sort order.
sortUsingTags(clone.get());
LOG(5) << "Tagged tree:" << endl
<< clone->toString();
// Use the cached index assignments to build solnRoot.
QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(
query, clone.release(), false, params.indices, params);
if (!solnRoot) {
return Status(ErrorCodes::BadValue,
str::stream() << "Failed to create data access plan from cache. Query: "
<< query.toStringShort());
}
// Takes ownership of 'solnRoot'.
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (!soln) {
return Status(ErrorCodes::BadValue,
str::stream()
<< "Failed to analyze plan from cache. Query: " << query.toStringShort());
}
LOG(5) << "Planner: solution constructed from the cache:\n" << soln->toString();
*out = soln;
return Status::OK();
}
// static
void QueryPlanner::plan(const CanonicalQuery& query,
const QueryPlannerParams& params,
vector<QuerySolution*>* out) {
QLOG() << "=============================\n"
<< "Beginning planning, options = " << optionString(params.options) << endl
<< "Canonical query:\n" << query.toString() << endl
<< "============================="
<< endl;
// The shortcut formerly known as IDHACK. See if it's a simple _id query. If so we might
// just make an ixscan over the _id index and bypass the rest of planning entirely.
if (!query.getParsed().isExplain() && !query.getParsed().showDiskLoc()
&& isSimpleIdQuery(query.getParsed().getFilter())
&& !query.getParsed().hasOption(QueryOption_CursorTailable)) {
// See if we can find an _id index.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
const IndexEntry& index = params.indices[i];
QLOG() << "IDHACK using index " << index.toString() << endl;
// If so, we make a simple scan to find the doc.
IndexScanNode* isn = new IndexScanNode();
isn->indexKeyPattern = index.keyPattern;
isn->indexIsMultiKey = index.multikey;
isn->direction = 1;
isn->bounds.isSimpleRange = true;
BSONObj key = getKeyFromQuery(index.keyPattern, query.getParsed().getFilter());
isn->bounds.startKey = isn->bounds.endKey = key;
isn->bounds.endKeyInclusive = true;
isn->computeProperties();
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, isn);
if (NULL != soln) {
out->push_back(soln);
QLOG() << "IDHACK solution is:\n" << (*out)[0]->toString() << endl;
// And that's it.
return;
}
}
}
}
for (size_t i = 0; i < params.indices.size(); ++i) {
QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl;
}
bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan. TODO: This is a policy departure. Previously I think you
// could ask for a tailable cursor and it just tried to give you one. Now, we fail if we
// can't provide one. Is this what we want?
if (query.getParsed().hasOption(QueryOption_CursorTailable)) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, true, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return;
}
// The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of
// saying "table scan for two, please."
if (!query.getParsed().getHint().isEmpty()) {
BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural");
if (!natural.eoo()) {
QLOG() << "forcing a table scan due to hinted $natural\n";
if (canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return;
}
}
// NOR and NOT we can't handle well with indices. If we see them here, they weren't
// rewritten to remove the negation. Just output a collscan for those.
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOT)
|| QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOR)) {
// If there's a near predicate, we can't handle this.
// TODO: Should canonicalized query detect this?
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)) {
warning() << "Can't handle NOT/NOR with GEO_NEAR";
return;
}
QLOG() << "NOT/NOR in plan, just outtping a collscan\n";
if (canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return;
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
QLOG() << "predicate over field " << *it << endl;
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
BSONObj hintIndex = query.getParsed().getHint();
// Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real
// plan. If that fails, just scan the _id index.
if (query.getParsed().isSnapshot()) {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
size_t hintIndexNumber = numeric_limits<size_t>::max();
if (!hintIndex.isEmpty()) {
// Sigh. If the hint is specified it might be using the index name.
BSONElement firstHintElt = hintIndex.firstElement();
if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) {
string hintName = firstHintElt.String();
for (size_t i = 0; i < params.indices.size(); ++i) {
if (params.indices[i].name == hintName) {
QLOG() << "hint by name specified, restricting indices to "
<< params.indices[i].keyPattern.toString() << endl;
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
hintIndexNumber = i;
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
else {
for (size_t i = 0; i < params.indices.size(); ++i) {
if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) {
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
QLOG() << "hint specified, restricting indices to " << hintIndex.toString()
<< endl;
hintIndexNumber = i;
break;
}
}
}
if (hintIndexNumber == numeric_limits<size_t>::max()) {
// This is supposed to be an error.
warning() << "Can't find hint for " << hintIndex.toString();
return;
}
}
else {
QLOG() << "Finding relevant indices\n";
QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices);
}
for (size_t i = 0; i < relevantIndices.size(); ++i) {
QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl;
}
// Figure out how useful each index is to each predicate.
// query.root() is now annotated with RelevantTag(s).
QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices);
QLOG() << "rated tree" << endl;
QLOG() << query.root()->toString() << endl;
// If there is a GEO_NEAR it must have an index it can use directly.
// XXX: move into data access?
MatchExpression* gnNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) {
// No index for GEO_NEAR? No query.
RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return;
}
GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode);
vector<size_t> newFirst;
// 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed
// the full query tree inside it as a matcher.
for (size_t i = 0; i < tag->first.size(); ++i) {
// GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning.
if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) {
newFirst.push_back(i);
continue;
}
// If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the
// entire tree as a filter, if possible.
GeoNear2DNode* solnRoot = new GeoNear2DNode();
solnRoot->nq = gnme->getData();
if (MatchExpression::GEO_NEAR != query.root()->matchType()) {
// root is an AND, clone and delete the GEO_NEAR child.
MatchExpression* filterTree = query.root()->shallowClone();
verify(MatchExpression::AND == filterTree->matchType());
bool foundChild = false;
for (size_t i = 0; i < filterTree->numChildren(); ++i) {
if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) {
foundChild = true;
filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i);
break;
}
}
verify(foundChild);
solnRoot->filter.reset(filterTree);
}
solnRoot->numWanted = query.getParsed().getNumToReturn();
if (0 == solnRoot->numWanted) {
solnRoot->numWanted = 100;
}
solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern;
// Remove the 2d index. 2d can only be the first field, and we know there is
// only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll
// only be first for gnNode.
tag->first.erase(tag->first.begin() + i);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
}
// Continue planning w/non-2d indices tagged for this pred.
tag->first.swap(newFirst);
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return;
}
}
// Likewise, if there is a TEXT it must have an index it can use directly.
MatchExpression* textNode;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) {
RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return;
}
}
// If we have any relevant indices, we try to create indexed plans.
if (0 < relevantIndices.size()) {
// The enumerator spits out trees tagged with IndexTag(s).
PlanEnumerator isp(query.root(), &relevantIndices);
isp.init();
MatchExpression* rawTree;
while (isp.getNext(&rawTree)) {
QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString()
<< endl;
// This can fail if enumeration makes a mistake.
QuerySolutionNode* solnRoot =
QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices);
if (NULL == solnRoot) { continue; }
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
QLOG() << "Planner: adding solution:\n" << soln->toString() << endl;
out->push_back(soln);
}
}
}
QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n";
// An index was hinted. If there are any solutions, they use the hinted index. If not, we
// scan the entire index to provide results and output that as our plan. This is the
// desired behavior when an index is hinted that is not relevant to the query.
if (!hintIndex.isEmpty() && (0 == out->size())) {
QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params);
if (NULL != soln) {
QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl;
out->push_back(soln);
}
return;
}
// If a sort order is requested, there may be an index that provides it, even if that
// index is not over any predicates in the query.
//
// XXX XXX: Can we do this even if the index is sparse? Might we miss things?
if (!query.getParsed().getSort().isEmpty()
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
// See if we have a sort provided from an index already.
bool usingIndexToSort = false;
for (size_t i = 0; i < out->size(); ++i) {
QuerySolution* soln = (*out)[i];
if (!soln->hasSortStage) {
usingIndexToSort = true;
break;
}
}
if (!usingIndexToSort) {
for (size_t i = 0; i < params.indices.size(); ++i) {
const BSONObj& kp = params.indices[i].keyPattern;
if (providesSort(query, kp)) {
QLOG() << "Planner: outputting soln that uses index to provide sort."
<< endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
QLOG() << "Planner: outputting soln that uses (reverse) index "
<< "to provide sort." << endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
}
}
}
// TODO: Do we always want to offer a collscan solution?
// XXX: currently disabling the always-use-a-collscan in order to find more planner bugs.
if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)
&& ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan)))
{
QuerySolution* collscan = buildCollscanSoln(query, false, params);
if (NULL != collscan) {
out->push_back(collscan);
QLOG() << "Planner: outputting a collscan:\n";
QLOG() << collscan->toString() << endl;
}
}
}
BSONObj UpdateStage::transformAndUpdate(const Snapshotted<BSONObj>& oldObj, RecordId& recordId) {
const UpdateRequest* request = _params.request;
UpdateDriver* driver = _params.driver;
CanonicalQuery* cq = _params.canonicalQuery;
UpdateLifecycle* lifecycle = request->getLifecycle();
// If asked to return new doc, default to the oldObj, in case nothing changes.
BSONObj newObj = oldObj.value();
// Ask the driver to apply the mods. It may be that the driver can apply those "in
// place", that is, some values of the old document just get adjusted without any
// change to the binary layout on the bson layer. It may be that a whole new document
// is needed to accomodate the new bson layout of the resulting document. In any event,
// only enable in-place mutations if the underlying storage engine offers support for
// writing damage events.
_doc.reset(oldObj.value(),
(_collection->updateWithDamagesSupported()
? mutablebson::Document::kInPlaceEnabled
: mutablebson::Document::kInPlaceDisabled));
BSONObj logObj;
bool docWasModified = false;
Status status = Status::OK();
const bool validateForStorage = getOpCtx()->writesAreReplicated() && _enforceOkForStorage;
FieldRefSet immutablePaths;
if (getOpCtx()->writesAreReplicated() && !request->isFromMigration()) {
if (lifecycle) {
auto immutablePathsVector =
getImmutableFields(getOpCtx(), request->getNamespaceString());
if (immutablePathsVector) {
immutablePaths.fillFrom(
transitional_tools_do_not_use::unspool_vector(*immutablePathsVector));
}
}
immutablePaths.keepShortest(&idFieldRef);
}
if (!driver->needMatchDetails()) {
// If we don't need match details, avoid doing the rematch
status = driver->update(
StringData(), &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified);
} else {
// If there was a matched field, obtain it.
MatchDetails matchDetails;
matchDetails.requestElemMatchKey();
dassert(cq);
verify(cq->root()->matchesBSON(oldObj.value(), &matchDetails));
string matchedField;
if (matchDetails.hasElemMatchKey())
matchedField = matchDetails.elemMatchKey();
status = driver->update(
matchedField, &_doc, validateForStorage, immutablePaths, &logObj, &docWasModified);
}
if (!status.isOK()) {
uasserted(16837, status.reason());
}
// Skip adding _id field if the collection is capped (since capped collection documents can
// neither grow nor shrink).
const auto createIdField = !_collection->isCapped();
// Ensure if _id exists it is first
status = ensureIdFieldIsFirst(&_doc);
if (status.code() == ErrorCodes::InvalidIdField) {
// Create ObjectId _id field if we are doing that
if (createIdField) {
addObjectIDIdField(&_doc);
}
} else {
uassertStatusOK(status);
}
// See if the changes were applied in place
const char* source = NULL;
const bool inPlace = _doc.getInPlaceUpdates(&_damages, &source);
if (inPlace && _damages.empty()) {
// An interesting edge case. A modifier didn't notice that it was really a no-op
// during its 'prepare' phase. That represents a missed optimization, but we still
// shouldn't do any real work. Toggle 'docWasModified' to 'false'.
//
// Currently, an example of this is '{ $push : { x : {$each: [], $sort: 1} } }' when the 'x'
// array exists and is already sorted.
docWasModified = false;
}
if (docWasModified) {
// Prepare to write back the modified document
WriteUnitOfWork wunit(getOpCtx());
RecordId newRecordId;
OplogUpdateEntryArgs args;
if (!request->isExplain()) {
invariant(_collection);
auto* css = CollectionShardingState::get(getOpCtx(), _collection->ns());
args.nss = _collection->ns();
args.uuid = _collection->uuid();
args.stmtId = request->getStmtId();
args.update = logObj;
args.criteria = css->getMetadata().extractDocumentKey(newObj);
uassert(16980,
"Multi-update operations require all documents to have an '_id' field",
!request->isMulti() || args.criteria.hasField("_id"_sd));
args.fromMigrate = request->isFromMigration();
args.storeDocOption = getStoreDocMode(*request);
if (args.storeDocOption == OplogUpdateEntryArgs::StoreDocOption::PreImage) {
args.preImageDoc = oldObj.value().getOwned();
}
}
if (inPlace) {
if (!request->isExplain()) {
newObj = oldObj.value();
const RecordData oldRec(oldObj.value().objdata(), oldObj.value().objsize());
Snapshotted<RecordData> snap(oldObj.snapshotId(), oldRec);
StatusWith<RecordData> newRecStatus = _collection->updateDocumentWithDamages(
getOpCtx(), recordId, std::move(snap), source, _damages, &args);
newObj = uassertStatusOK(std::move(newRecStatus)).releaseToBson();
}
newRecordId = recordId;
} else {
// The updates were not in place. Apply them through the file manager.
newObj = _doc.getObject();
uassert(17419,
str::stream() << "Resulting document after update is larger than "
<< BSONObjMaxUserSize,
newObj.objsize() <= BSONObjMaxUserSize);
if (!request->isExplain()) {
newRecordId = _collection->updateDocument(getOpCtx(),
recordId,
oldObj,
newObj,
true,
driver->modsAffectIndices(),
_params.opDebug,
&args);
}
}
invariant(oldObj.snapshotId() == getOpCtx()->recoveryUnit()->getSnapshotId());
wunit.commit();
// If the document moved, we might see it again in a collection scan (maybe it's
// a document after our current document).
//
// If the document is indexed and the mod changes an indexed value, we might see
// it again. For an example, see the comment above near declaration of
// updatedRecordIds.
//
// This must be done after the wunit commits so we are sure we won't be rolling back.
if (_updatedRecordIds && (newRecordId != recordId || driver->modsAffectIndices())) {
_updatedRecordIds->insert(newRecordId);
}
}
// Only record doc modifications if they wrote (exclude no-ops). Explains get
// recorded as if they wrote.
if (docWasModified || request->isExplain()) {
_specificStats.nModified++;
}
return newObj;
}
