Status getRunnerFromCache(CanonicalQuery* canonicalQuery,
Collection* collection,
const QueryPlannerParams& plannerParams,
Runner** out) {
// Skip cache look up for non-cacheable queries.
if (!PlanCache::shouldCacheQuery(*canonicalQuery)) {
return Status(ErrorCodes::BadValue, "query is not cacheable");
}
CachedSolution* rawCS;
Status cacheLookupStatus = collection->infoCache()->getPlanCache()->get(*canonicalQuery,
&rawCS);
if (!cacheLookupStatus.isOK()) {
return cacheLookupStatus;
}
// We have a CachedSolution. Have the planner turn it into a QuerySolution.
boost::scoped_ptr<CachedSolution> cs(rawCS);
QuerySolution *qs, *backupQs;
Status status = QueryPlanner::planFromCache(*canonicalQuery,
plannerParams,
*cs,
&qs,
&backupQs);
if (!status.isOK()) {
return status;
}
// If our cached solution is a hit for a count query, try to turn it into a fast count
// thing.
if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) {
if (turnIxscanIntoCount(qs)) {
LOG(2) << "Using fast count: " << canonicalQuery->toStringShort()
<< ", planSummary: " << getPlanSummary(*qs);
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(collection, *qs, &root, &ws));
*out = new SingleSolutionRunner(collection,
canonicalQuery, qs, root, ws);
if (NULL != backupQs) {
delete backupQs;
}
return Status::OK();
}
}
// If we're here, we're going to used the cached plan and things are normal.
LOG(2) << "Using cached query plan: " << canonicalQuery->toStringShort()
<< ", planSummary: " << getPlanSummary(*qs);
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(collection, *qs, &root, &ws));
CachedPlanRunner* cpr = new CachedPlanRunner(collection,
canonicalQuery,
qs,
root,
ws);
// If there's a backup solution, let the CachedPlanRunner know about it.
if (NULL != backupQs) {
WorkingSet* backupWs;
PlanStage* backupRoot;
verify(StageBuilder::build(collection, *backupQs, &backupRoot, &backupWs));
cpr->setBackupPlan(backupQs, backupRoot, backupWs);
}
*out = cpr;
return Status::OK();
}
/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(Collection* collection,
CanonicalQuery* rawCanonicalQuery,
Runner** out,
size_t plannerOptions) {
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// This can happen as we're called by internal clients as well.
if (NULL == collection) {
const string& ns = canonicalQuery->ns();
*out = new EOFRunner(canonicalQuery.release(), ns);
return Status::OK();
}
// If we have an _id index we can use the idhack runner.
if (canUseIDHack(*canonicalQuery) && collection->getIndexCatalog()->findIdIndex()) {
*out = new IDHackRunner(collection, canonicalQuery.release());
return Status::OK();
}
// If it's not NULL, we may have indices. Access the catalog and fill out IndexEntry(s)
QueryPlannerParams plannerParams;
IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false);
while (ii.more()) {
const IndexDescriptor* desc = ii.next();
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName(),
desc->infoObj()));
}
// If query supports index filters, filter params.indices by indices in query settings.
QuerySettings* querySettings = collection->infoCache()->getQuerySettings();
AllowedIndices* allowedIndicesRaw;
// Filter index catalog if index filters are specified for query.
// Also, signal to planner that application hint should be ignored.
if (querySettings->getAllowedIndices(*canonicalQuery, &allowedIndicesRaw)) {
boost::scoped_ptr<AllowedIndices> allowedIndices(allowedIndicesRaw);
filterAllowedIndexEntries(*allowedIndices, &plannerParams.indices);
plannerParams.indexFiltersApplied = true;
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!collection->isCapped()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// Process the planning options.
plannerParams.options = plannerOptions;
if (storageGlobalParams.noTableScan) {
const string& ns = canonicalQuery->ns();
// There are certain cases where we ignore this restriction:
bool ignore = canonicalQuery->getQueryObj().isEmpty()
|| (string::npos != ns.find(".system."))
|| (0 == ns.find("local."));
if (!ignore) {
plannerParams.options |= QueryPlannerParams::NO_TABLE_SCAN;
}
}
if (!(plannerParams.options & QueryPlannerParams::NO_TABLE_SCAN)) {
plannerParams.options |= QueryPlannerParams::INCLUDE_COLLSCAN;
}
// If the caller wants a shard filter, make sure we're actually sharded.
if (plannerParams.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
CollectionMetadataPtr collMetadata =
shardingState.getCollectionMetadata(canonicalQuery->ns());
if (collMetadata) {
plannerParams.shardKey = collMetadata->getKeyPattern();
}
else {
// If there's no metadata don't bother w/the shard filter since we won't know what
// the key pattern is anyway...
plannerParams.options &= ~QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
}
// Try to look up a cached solution for the query.
//
// Skip cache look up for non-cacheable queries.
// See PlanCache::shouldCacheQuery()
//
// TODO: Can the cache have negative data about a solution?
CachedSolution* rawCS;
if (PlanCache::shouldCacheQuery(*canonicalQuery) &&
collection->infoCache()->getPlanCache()->get(*canonicalQuery, &rawCS).isOK()) {
// We have a CachedSolution. Have the planner turn it into a QuerySolution.
boost::scoped_ptr<CachedSolution> cs(rawCS);
QuerySolution *qs, *backupQs;
Status status = QueryPlanner::planFromCache(*canonicalQuery, plannerParams, *cs,
&qs, &backupQs);
// See SERVER-12438. Unfortunately we have to defer to the backup solution
// if both a batch size is set and a sort is requested.
//
// TODO: it would be really nice to delete this block in the future.
if (status.isOK() && NULL != backupQs &&
0 < canonicalQuery->getParsed().getNumToReturn() &&
!canonicalQuery->getParsed().getSort().isEmpty()) {
delete qs;
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*backupQs, &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
backupQs, root, ws);
return Status::OK();
}
if (status.isOK()) {
if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) {
if (turnIxscanIntoCount(qs)) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*qs, &root, &ws));
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(), qs, root, ws);
if (NULL != backupQs) {
delete backupQs;
}
return Status::OK();
}
}
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*qs, &root, &ws));
CachedPlanRunner* cpr = new CachedPlanRunner(collection,
canonicalQuery.release(), qs,
root, ws);
if (NULL != backupQs) {
WorkingSet* backupWs;
PlanStage* backupRoot;
verify(StageBuilder::build(*backupQs, &backupRoot, &backupWs));
cpr->setBackupPlan(backupQs, backupRoot, backupWs);
}
*out = cpr;
return Status::OK();
}
}
if (enableIndexIntersection) {
plannerParams.options |= QueryPlannerParams::INDEX_INTERSECTION;
}
plannerParams.options |= QueryPlannerParams::KEEP_MUTATIONS;
vector<QuerySolution*> solutions;
Status status = QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions);
if (!status.isOK()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" planner returned error: " + status.reason());
}
// We cannot figure out how to answer the query. Perhaps it requires an index
// we do not have?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue,
str::stream()
<< "error processing query: "
<< canonicalQuery->toString()
<< " No query solutions");
}
// See if one of our solutions is a fast count hack in disguise.
if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) {
for (size_t i = 0; i < solutions.size(); ++i) {
if (turnIxscanIntoCount(solutions[i])) {
// Great, we can use solutions[i]. Clean up the other QuerySolution(s).
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
// We're not going to cache anything that's fast count.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
solutions[i],
root,
ws);
return Status::OK();
}
}
}
if (1 == solutions.size()) {
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),solutions[0], root, ws);
return Status::OK();
}
else {
// See SERVER-12438. In an ideal world we should not arbitrarily prefer certain
// solutions over others. But unfortunately for historical reasons we are forced
// to prefer a solution where the index provides the sort, if the batch size
// is set and a sort is requested. Read SERVER-12438 for details, if you dare.
//
// TODO: it would be really nice to delete this entire block in the future.
if (0 < canonicalQuery->getParsed().getNumToReturn()
&& !canonicalQuery->getParsed().getSort().isEmpty()) {
// Look for a solution without a blocking sort stage.
for (size_t i = 0; i < solutions.size(); ++i) {
if (!solutions[i]->hasSortStage) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Free unused solutions.
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
// And, run the plan.
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
solutions[i], root, ws);
return Status::OK();
}
}
}
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(collection,canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
if (solutions[i]->cacheData.get()) {
solutions[i]->cacheData->indexFilterApplied = plannerParams.indexFiltersApplied;
}
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}
/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(Collection* collection, CanonicalQuery* rawCanonicalQuery,
Runner** out, size_t plannerOptions) {
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// This can happen as we're called by internal clients as well.
if (NULL == collection) {
const string& ns = canonicalQuery->ns();
*out = new EOFRunner(canonicalQuery.release(), ns);
return Status::OK();
}
// If we have an _id index we can use the idhack runner.
if (canUseIDHack(*canonicalQuery) && collection->getIndexCatalog()->findIdIndex()) {
*out = new IDHackRunner(collection, canonicalQuery.release());
return Status::OK();
}
// If it's not NULL, we may have indices. Access the catalog and fill out IndexEntry(s)
QueryPlannerParams plannerParams;
IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false);
while (ii.more()) {
const IndexDescriptor* desc = ii.next();
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName(),
desc->infoObj()));
}
// If query supports admin hint, filter params.indices by indexes in query settings.
QuerySettings* querySettings = collection->infoCache()->getQuerySettings();
AllowedIndices* allowedIndicesRaw;
// Filter index catalog if admin hint is specified for query.
// Also, signal to planner that application hint should be ignored.
if (querySettings->getAllowedIndices(*canonicalQuery, &allowedIndicesRaw)) {
boost::scoped_ptr<AllowedIndices> allowedIndices(allowedIndicesRaw);
filterAllowedIndexEntries(*allowedIndices, &plannerParams.indices);
plannerParams.adminHintApplied = true;
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!collection->isCapped()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// Process the planning options.
plannerParams.options = plannerOptions;
if (storageGlobalParams.noTableScan) {
const string& ns = canonicalQuery->ns();
// There are certain cases where we ignore this restriction:
bool ignore = canonicalQuery->getQueryObj().isEmpty()
|| (string::npos != ns.find(".system."))
|| (0 == ns.find("local."));
if (!ignore) {
plannerParams.options |= QueryPlannerParams::NO_TABLE_SCAN;
}
}
if (!(plannerParams.options & QueryPlannerParams::NO_TABLE_SCAN)) {
plannerParams.options |= QueryPlannerParams::INCLUDE_COLLSCAN;
}
// If the caller wants a shard filter, make sure we're actually sharded.
if (plannerParams.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
CollectionMetadataPtr collMetadata =
shardingState.getCollectionMetadata(canonicalQuery->ns());
if (collMetadata) {
plannerParams.shardKey = collMetadata->getKeyPattern();
}
else {
// If there's no metadata don't bother w/the shard filter since we won't know what
// the key pattern is anyway...
plannerParams.options &= ~QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
}
// Try to look up a cached solution for the query.
//
// Skip cache look up for non-cacheable queries.
// See PlanCache::shouldCacheQuery()
//
// TODO: Can the cache have negative data about a solution?
CachedSolution* rawCS;
if (PlanCache::shouldCacheQuery(*canonicalQuery) &&
collection->infoCache()->getPlanCache()->get(*canonicalQuery, &rawCS).isOK()) {
// We have a CachedSolution. Have the planner turn it into a QuerySolution.
boost::scoped_ptr<CachedSolution> cs(rawCS);
QuerySolution *qs, *backupQs;
Status status = QueryPlanner::planFromCache(*canonicalQuery, plannerParams, *cs,
&qs, &backupQs);
if (status.isOK()) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*qs, &root, &ws));
CachedPlanRunner* cpr = new CachedPlanRunner(canonicalQuery.release(), qs,
root, ws);
if (NULL != backupQs) {
WorkingSet* backupWs;
PlanStage* backupRoot;
verify(StageBuilder::build(*backupQs, &backupRoot, &backupWs));
cpr->setBackupPlan(backupQs, backupRoot, backupWs);
}
*out = cpr;
return Status::OK();
}
}
plannerParams.options |= QueryPlannerParams::INDEX_INTERSECTION;
plannerParams.options |= QueryPlannerParams::KEEP_MUTATIONS;
vector<QuerySolution*> solutions;
Status status = QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions);
if (!status.isOK()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" planner returned error: " + status.reason());
}
/*
for (size_t i = 0; i < solutions.size(); ++i) {
QLOG() << "solution " << i << " is " << solutions[i]->toString() << endl;
}
*/
// We cannot figure out how to answer the query. Should this ever happen?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" No query solutions");
}
if (1 == solutions.size()) {
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
return Status::OK();
}
else {
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
if (solutions[i]->cacheData.get()) {
solutions[i]->cacheData->adminHintApplied = plannerParams.adminHintApplied;
}
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}