/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(QueryMessage& q, Runner** out) {
CanonicalQuery* rawCanonicalQuery = NULL;
// Canonicalize the query and wrap it in an auto_ptr so we don't leak it if something goes
// wrong.
Status status = CanonicalQuery::canonicalize(q, &rawCanonicalQuery);
if (!status.isOK()) { return status; }
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// Try to look up a cached solution for the query.
// TODO: Can the cache have negative data about a solution?
PlanCache* localCache = PlanCache::get(canonicalQuery->ns());
CachedSolution* cs = localCache->get(*canonicalQuery);
if (NULL != cs) {
// We have a cached solution. Hand the canonical query and cached solution off to the
// cached plan runner, which takes ownership of both.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*cs->solution, &root, &ws));
*out = new CachedPlanRunner(canonicalQuery.release(), cs, root, ws);
return Status::OK();
}
// No entry in cache for the query. We have to solve the query ourself.
vector<QuerySolution*> solutions;
QueryPlanner::plan(*canonicalQuery, &solutions);
// We cannot figure out how to answer the query. Should this ever happen?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue, "Can't create a plan for the canonical query " +
canonicalQuery->toString());
}
if (1 == solutions.size()) {
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
return Status::OK();
}
else {
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}
/**
* @brief Create an AWS V4 Signature canonical request.
*
* @param[in] operation The HTTP method being used for the request.
* @param[in] request The network request to generate a canonical request for.
* @param[in] payload Optional data being submitted in the request (eg for `PUT` and `POST` operations).
* @param[out] signedHeaders A semi-colon separated list of the names of all headers
* included in the result.
*
* @return An AWS V4 Signature canonical request.
*
* @see http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
*/
QByteArray AwsSignatureV4Private::canonicalRequest(const QNetworkAccessManager::Operation operation,
const QNetworkRequest &request, const QByteArray &payload,
QByteArray * const signedHeaders) const
{
return httpMethod(operation).toUtf8() + '\n' +
canonicalPath(request.url()).toUtf8() + '\n' +
canonicalQuery(QUrlQuery(request.url())) + '\n' +
canonicalHeaders(request, signedHeaders) + '\n' +
*signedHeaders + '\n' +
QCryptographicHash::hash(payload, hashAlgorithm).toHex();
}
Status getExecutorIDHack(OperationContext* txn,
Collection* collection,
CanonicalQuery* rawCanonicalQuery,
const QueryPlannerParams& plannerParams,
PlanExecutor** out) {
invariant(collection);
invariant(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
LOG(2) << "Using idhack: " << canonicalQuery->toStringShort();
WorkingSet* ws = new WorkingSet();
PlanStage* root = new IDHackStage(txn, collection, canonicalQuery.get(), ws);
// Might have to filter out orphaned docs.
if (plannerParams.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
root = new ShardFilterStage(shardingState.getCollectionMetadata(collection->ns()),
ws, root);
}
// There might be a projection. The idhack stage will always fetch the full document,
// so we don't support covered projections. However, we might use the simple inclusion
// fast path.
if (NULL != canonicalQuery->getProj()) {
ProjectionStageParams params(WhereCallbackReal(collection->ns().db()));
params.projObj = canonicalQuery->getProj()->getProjObj();
// Stuff the right data into the params depending on what proj impl we use.
if (canonicalQuery->getProj()->requiresDocument()
|| canonicalQuery->getProj()->wantIndexKey()) {
params.fullExpression = canonicalQuery->root();
params.projImpl = ProjectionStageParams::NO_FAST_PATH;
}
else {
params.projImpl = ProjectionStageParams::SIMPLE_DOC;
}
root = new ProjectionStage(params, ws, root);
}
*out = new PlanExecutor(ws, root, canonicalQuery.release(), collection);
return Status::OK();
}
/**
* @brief Create an AWS V3 Signature canonical request.
*
* Note, this function implments both `AWS3` and `AWS3-HTTPS` variants of the
* AWS Signature version 3 - which are quite different.
*
* @param[in] operation The HTTP method being used for the request.
* @param[in] request The network request to generate a canonical request for.
* @param[in] payload Optional data being submitted in the request (eg for `PUT` and `POST` operations).
* @param[out] signedHeaders A semi-colon separated list of the names of all headers
* included in the result.
*
* @return An AWS V3 Signature canonical request.
*
* @see http://docs.aws.amazon.com/amazonswf/latest/developerguide/HMACAuth-swf.html (AWS3)
* @see http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/RESTAuthentication.html (AWS3-HTTPS)
*/
QByteArray AwsSignatureV3Private::canonicalRequest(const QNetworkAccessManager::Operation operation,
const QNetworkRequest &request, const QByteArray &payload,
QByteArray * const signedHeaders) const
{
// AWS3-HTTPS
if (isHttps(request)) {
Q_ASSERT((request.hasRawHeader("x-amz-date")) || (request.hasRawHeader("Date")));
QByteArray canonicalRequest = request.rawHeader(request.hasRawHeader("x-amz-date") ? "x-amz-date" : "Date");
if (request.hasRawHeader("x-amz-nonce")) {
canonicalRequest += request.rawHeader("x-amz-nonce");
}
return canonicalRequest;
}
// AWS3
return httpMethod(operation).toUtf8() + '\n' +
canonicalPath(request.url()).toUtf8() + '\n' +
canonicalQuery(QUrlQuery(request.url())) + '\n' +
canonicalHeaders(request, signedHeaders) + '\n' +
payload;
}
/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(Collection* collection,
CanonicalQuery* rawCanonicalQuery,
Runner** out,
size_t plannerOptions) {
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// This can happen as we're called by internal clients as well.
if (NULL == collection) {
const string& ns = canonicalQuery->ns();
*out = new EOFRunner(canonicalQuery.release(), ns);
return Status::OK();
}
// If we have an _id index we can use the idhack runner.
if (canUseIDHack(*canonicalQuery) && collection->getIndexCatalog()->findIdIndex()) {
*out = new IDHackRunner(collection, canonicalQuery.release());
return Status::OK();
}
// If it's not NULL, we may have indices. Access the catalog and fill out IndexEntry(s)
QueryPlannerParams plannerParams;
IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false);
while (ii.more()) {
const IndexDescriptor* desc = ii.next();
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName(),
desc->infoObj()));
}
// If query supports index filters, filter params.indices by indices in query settings.
QuerySettings* querySettings = collection->infoCache()->getQuerySettings();
AllowedIndices* allowedIndicesRaw;
// Filter index catalog if index filters are specified for query.
// Also, signal to planner that application hint should be ignored.
if (querySettings->getAllowedIndices(*canonicalQuery, &allowedIndicesRaw)) {
boost::scoped_ptr<AllowedIndices> allowedIndices(allowedIndicesRaw);
filterAllowedIndexEntries(*allowedIndices, &plannerParams.indices);
plannerParams.indexFiltersApplied = true;
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!collection->isCapped()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// Process the planning options.
plannerParams.options = plannerOptions;
if (storageGlobalParams.noTableScan) {
const string& ns = canonicalQuery->ns();
// There are certain cases where we ignore this restriction:
bool ignore = canonicalQuery->getQueryObj().isEmpty()
|| (string::npos != ns.find(".system."))
|| (0 == ns.find("local."));
if (!ignore) {
plannerParams.options |= QueryPlannerParams::NO_TABLE_SCAN;
}
}
if (!(plannerParams.options & QueryPlannerParams::NO_TABLE_SCAN)) {
plannerParams.options |= QueryPlannerParams::INCLUDE_COLLSCAN;
}
// If the caller wants a shard filter, make sure we're actually sharded.
if (plannerParams.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
CollectionMetadataPtr collMetadata =
shardingState.getCollectionMetadata(canonicalQuery->ns());
if (collMetadata) {
plannerParams.shardKey = collMetadata->getKeyPattern();
}
else {
// If there's no metadata don't bother w/the shard filter since we won't know what
// the key pattern is anyway...
plannerParams.options &= ~QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
}
// Try to look up a cached solution for the query.
//
// Skip cache look up for non-cacheable queries.
// See PlanCache::shouldCacheQuery()
//
// TODO: Can the cache have negative data about a solution?
CachedSolution* rawCS;
if (PlanCache::shouldCacheQuery(*canonicalQuery) &&
collection->infoCache()->getPlanCache()->get(*canonicalQuery, &rawCS).isOK()) {
// We have a CachedSolution. Have the planner turn it into a QuerySolution.
boost::scoped_ptr<CachedSolution> cs(rawCS);
QuerySolution *qs, *backupQs;
Status status = QueryPlanner::planFromCache(*canonicalQuery, plannerParams, *cs,
&qs, &backupQs);
// See SERVER-12438. Unfortunately we have to defer to the backup solution
// if both a batch size is set and a sort is requested.
//
// TODO: it would be really nice to delete this block in the future.
if (status.isOK() && NULL != backupQs &&
0 < canonicalQuery->getParsed().getNumToReturn() &&
!canonicalQuery->getParsed().getSort().isEmpty()) {
delete qs;
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*backupQs, &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
backupQs, root, ws);
return Status::OK();
}
if (status.isOK()) {
if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) {
if (turnIxscanIntoCount(qs)) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*qs, &root, &ws));
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(), qs, root, ws);
if (NULL != backupQs) {
delete backupQs;
}
return Status::OK();
}
}
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*qs, &root, &ws));
CachedPlanRunner* cpr = new CachedPlanRunner(collection,
canonicalQuery.release(), qs,
root, ws);
if (NULL != backupQs) {
WorkingSet* backupWs;
PlanStage* backupRoot;
verify(StageBuilder::build(*backupQs, &backupRoot, &backupWs));
cpr->setBackupPlan(backupQs, backupRoot, backupWs);
}
*out = cpr;
return Status::OK();
}
}
if (enableIndexIntersection) {
plannerParams.options |= QueryPlannerParams::INDEX_INTERSECTION;
}
plannerParams.options |= QueryPlannerParams::KEEP_MUTATIONS;
vector<QuerySolution*> solutions;
Status status = QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions);
if (!status.isOK()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" planner returned error: " + status.reason());
}
// We cannot figure out how to answer the query. Perhaps it requires an index
// we do not have?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue,
str::stream()
<< "error processing query: "
<< canonicalQuery->toString()
<< " No query solutions");
}
// See if one of our solutions is a fast count hack in disguise.
if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) {
for (size_t i = 0; i < solutions.size(); ++i) {
if (turnIxscanIntoCount(solutions[i])) {
// Great, we can use solutions[i]. Clean up the other QuerySolution(s).
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
// We're not going to cache anything that's fast count.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
solutions[i],
root,
ws);
return Status::OK();
}
}
}
if (1 == solutions.size()) {
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),solutions[0], root, ws);
return Status::OK();
}
else {
// See SERVER-12438. In an ideal world we should not arbitrarily prefer certain
// solutions over others. But unfortunately for historical reasons we are forced
// to prefer a solution where the index provides the sort, if the batch size
// is set and a sort is requested. Read SERVER-12438 for details, if you dare.
//
// TODO: it would be really nice to delete this entire block in the future.
if (0 < canonicalQuery->getParsed().getNumToReturn()
&& !canonicalQuery->getParsed().getSort().isEmpty()) {
// Look for a solution without a blocking sort stage.
for (size_t i = 0; i < solutions.size(); ++i) {
if (!solutions[i]->hasSortStage) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Free unused solutions.
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
// And, run the plan.
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
solutions[i], root, ws);
return Status::OK();
}
}
}
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(collection,canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
if (solutions[i]->cacheData.get()) {
solutions[i]->cacheData->indexFilterApplied = plannerParams.indexFiltersApplied;
}
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}
/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(CanonicalQuery* rawCanonicalQuery, Runner** out) {
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// Try to look up a cached solution for the query.
// TODO: Can the cache have negative data about a solution?
PlanCache* localCache = PlanCache::get(canonicalQuery->ns());
if (NULL != localCache) {
CachedSolution* cs = localCache->get(*canonicalQuery);
if (NULL != cs) {
// We have a cached solution. Hand the canonical query and cached solution off to
// the cached plan runner, which takes ownership of both.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*cs->solution, &root, &ws));
*out = new CachedPlanRunner(canonicalQuery.release(), cs, root, ws);
return Status::OK();
}
}
// No entry in cache for the query. We have to solve the query ourself.
// Get the indices that we could possibly use.
NamespaceDetails* nsd = nsdetails(canonicalQuery->ns().c_str());
// If this is NULL, there is no data but the query is valid. You're allowed to query for
// data on an empty collection and it's not an error. There just isn't any data...
if (NULL == nsd) {
const std::string& ns = canonicalQuery->ns();
*out = new EOFRunner(canonicalQuery.release(), ns);
return Status::OK();
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!nsd->isCapped()) {
return Status(ErrorCodes::BadValue,
"tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// If it's not NULL, we may have indices.
vector<IndexEntry> indices;
for (int i = 0; i < nsd->getCompletedIndexCount(); ++i) {
auto_ptr<IndexDescriptor> desc(CatalogHack::getDescriptor(nsd, i));
indices.push_back(IndexEntry(desc->keyPattern(), desc->isMultikey(), desc->isSparse(), desc->indexName()));
}
vector<QuerySolution*> solutions;
size_t options = QueryPlanner::DEFAULT;
if (storageGlobalParams.noTableScan) {
const string& ns = canonicalQuery->ns();
// There are certain cases where we ignore this restriction:
bool ignore = canonicalQuery->getQueryObj().isEmpty()
|| (string::npos != ns.find(".system."))
|| (0 == ns.find("local."));
if (!ignore) {
options |= QueryPlanner::NO_TABLE_SCAN;
}
}
else {
options |= QueryPlanner::INCLUDE_COLLSCAN;
}
QueryPlanner::plan(*canonicalQuery, indices, options, &solutions);
/*
for (size_t i = 0; i < solutions.size(); ++i) {
QLOG() << "solution " << i << " is " << solutions[i]->toString() << endl;
}
*/
// We cannot figure out how to answer the query. Should this ever happen?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue, "Can't create a plan for the canonical query " +
canonicalQuery->toString());
}
if (1 == solutions.size()) {
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
return Status::OK();
}
else {
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}
Status getRunnerAlwaysPlan(Collection* collection,
CanonicalQuery* rawCanonicalQuery,
const QueryPlannerParams& plannerParams,
Runner** out) {
invariant(collection);
invariant(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
vector<QuerySolution*> solutions;
Status status = QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions);
if (!status.isOK()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" planner returned error: " + status.reason());
}
// We cannot figure out how to answer the query. Perhaps it requires an index
// we do not have?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue,
str::stream()
<< "error processing query: "
<< canonicalQuery->toString()
<< " No query solutions");
}
// See if one of our solutions is a fast count hack in disguise.
if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) {
for (size_t i = 0; i < solutions.size(); ++i) {
if (turnIxscanIntoCount(solutions[i])) {
// Great, we can use solutions[i]. Clean up the other QuerySolution(s).
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
LOG(2) << "Using fast count: " << canonicalQuery->toStringShort()
<< ", planSummary: " << getPlanSummary(*solutions[i]);
// We're not going to cache anything that's fast count.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(collection, *solutions[i], &root, &ws));
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
solutions[i],
root,
ws);
return Status::OK();
}
}
}
if (1 == solutions.size()) {
LOG(2) << "Only one plan is available; it will be run but will not be cached. "
<< canonicalQuery->toStringShort()
<< ", planSummary: " << getPlanSummary(*solutions[0]);
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(collection, *solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(collection,
canonicalQuery.release(),
solutions[0],
root,
ws);
return Status::OK();
}
else {
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(collection,canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
if (solutions[i]->cacheData.get()) {
solutions[i]->cacheData->indexFilterApplied = plannerParams.indexFiltersApplied;
}
verify(StageBuilder::build(collection, *solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}
/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(Collection* collection,
CanonicalQuery* rawCanonicalQuery,
Runner** out,
size_t plannerOptions) {
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// This can happen as we're called by internal clients as well.
if (NULL == collection) {
const string& ns = canonicalQuery->ns();
LOG(2) << "Collection " << ns << " does not exist."
<< " Using EOF runner: " << canonicalQuery->toStringShort();
*out = new EOFRunner(canonicalQuery.release(), ns);
return Status::OK();
}
// If we have an _id index we can use the idhack runner.
if (IDHackRunner::supportsQuery(*canonicalQuery) &&
collection->getIndexCatalog()->findIdIndex()) {
LOG(2) << "Using idhack: " << canonicalQuery->toStringShort();
*out = new IDHackRunner(collection, canonicalQuery.release());
return Status::OK();
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!collection->isCapped()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// Fill out the planning params. We use these for both cached solutions and non-cached.
QueryPlannerParams plannerParams;
plannerParams.options = plannerOptions;
fillOutPlannerParams(collection, rawCanonicalQuery, &plannerParams);
// See if the cache has what we're looking for.
Status cacheStatus = getRunnerFromCache(canonicalQuery.get(),
collection,
plannerParams,
out);
// This can be not-OK and we can carry on. It just means the query wasn't cached.
if (cacheStatus.isOK()) {
// We got a cached runner.
canonicalQuery.release();
return cacheStatus;
}
if (internalQueryPlanOrChildrenIndependently
&& SubplanRunner::canUseSubplanRunner(*canonicalQuery)) {
QLOG() << "Running query as sub-queries: " << canonicalQuery->toStringShort();
LOG(2) << "Running query as sub-queries: " << canonicalQuery->toStringShort();
SubplanRunner* runner;
Status runnerStatus = SubplanRunner::make(collection, plannerParams,
canonicalQuery.release(), &runner);
if (!runnerStatus.isOK()) {
return runnerStatus;
}
*out = runner;
return Status::OK();
}
return getRunnerAlwaysPlan(collection, canonicalQuery.release(), plannerParams, out);
}
Status getExecutorAlwaysPlan(OperationContext* txn,
Collection* collection,
CanonicalQuery* rawCanonicalQuery,
const QueryPlannerParams& plannerParams,
PlanExecutor** execOut) {
invariant(collection);
invariant(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
*execOut = NULL;
vector<QuerySolution*> solutions;
Status status = QueryPlanner::plan(*canonicalQuery.get(), plannerParams, &solutions);
if (!status.isOK()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" planner returned error: " + status.reason());
}
// We cannot figure out how to answer the query. Perhaps it requires an index
// we do not have?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue,
str::stream()
<< "error processing query: "
<< canonicalQuery->toString()
<< " No query solutions");
}
// See if one of our solutions is a fast count hack in disguise.
if (plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT) {
for (size_t i = 0; i < solutions.size(); ++i) {
if (turnIxscanIntoCount(solutions[i])) {
// Great, we can use solutions[i]. Clean up the other QuerySolution(s).
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
LOG(2) << "Using fast count: " << canonicalQuery->toStringShort()
<< ", planSummary: " << getPlanSummary(*solutions[i]);
// We're not going to cache anything that's fast count.
WorkingSet* ws = new WorkingSet();
PlanStage* root;
verify(StageBuilder::build(txn, collection, *solutions[i], ws, &root));
*execOut = new PlanExecutor(ws, root, solutions[i], canonicalQuery.release(),
collection);
return Status::OK();
}
}
}
if (1 == solutions.size()) {
LOG(2) << "Only one plan is available; it will be run but will not be cached. "
<< canonicalQuery->toStringShort()
<< ", planSummary: " << getPlanSummary(*solutions[0]);
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws = new WorkingSet();
PlanStage* root;
verify(StageBuilder::build(txn, collection, *solutions[0], ws, &root));
*execOut = new PlanExecutor(ws, root, solutions[0], canonicalQuery.release(),
collection);
return Status::OK();
}
else {
// Many solutions. Create a MultiPlanStage to pick the best, update the cache, and so on.
// The working set will be shared by all candidate plans and owned by the containing runner
WorkingSet* sharedWorkingSet = new WorkingSet();
MultiPlanStage* multiPlanStage = new MultiPlanStage(collection, canonicalQuery.get());
for (size_t ix = 0; ix < solutions.size(); ++ix) {
if (solutions[ix]->cacheData.get()) {
solutions[ix]->cacheData->indexFilterApplied = plannerParams.indexFiltersApplied;
}
// version of StageBuild::build when WorkingSet is shared
PlanStage* nextPlanRoot;
verify(StageBuilder::build(txn, collection, *solutions[ix],
sharedWorkingSet, &nextPlanRoot));
// Owns none of the arguments
multiPlanStage->addPlan(solutions[ix], nextPlanRoot, sharedWorkingSet);
}
// Do the plan selection up front.
multiPlanStage->pickBestPlan();
PlanExecutor* exec = new PlanExecutor(sharedWorkingSet, multiPlanStage,
canonicalQuery.release(), collection);
*execOut = exec;
return Status::OK();
}
}
Status getExecutor(OperationContext* txn,
Collection* collection,
CanonicalQuery* rawCanonicalQuery,
PlanExecutor** out,
size_t plannerOptions) {
invariant(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// This can happen as we're called by internal clients as well.
if (NULL == collection) {
const string& ns = canonicalQuery->ns();
LOG(2) << "Collection " << ns << " does not exist."
<< " Using EOF runner: " << canonicalQuery->toStringShort();
EOFStage* eofStage = new EOFStage();
WorkingSet* ws = new WorkingSet();
*out = new PlanExecutor(ws, eofStage, canonicalQuery.release(), collection);
return Status::OK();
}
// Fill out the planning params. We use these for both cached solutions and non-cached.
QueryPlannerParams plannerParams;
plannerParams.options = plannerOptions;
fillOutPlannerParams(collection, canonicalQuery.get(), &plannerParams);
// If we have an _id index we can use the idhack runner.
if (IDHackStage::supportsQuery(*canonicalQuery.get()) &&
collection->getIndexCatalog()->findIdIndex()) {
return getExecutorIDHack(txn, collection, canonicalQuery.release(), plannerParams, out);
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!collection->isCapped()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// Try to look up a cached solution for the query.
CachedSolution* rawCS;
if (PlanCache::shouldCacheQuery(*canonicalQuery) &&
collection->infoCache()->getPlanCache()->get(*canonicalQuery.get(), &rawCS).isOK()) {
// We have a CachedSolution. Have the planner turn it into a QuerySolution.
boost::scoped_ptr<CachedSolution> cs(rawCS);
QuerySolution *qs, *backupQs;
QuerySolution*& chosenSolution=qs; // either qs or backupQs
Status status = QueryPlanner::planFromCache(*canonicalQuery.get(), plannerParams, *cs,
&qs, &backupQs);
if (status.isOK()) {
// the working set will be shared by the root and backupRoot plans
// and owned by the containing single-solution-runner
//
WorkingSet* sharedWs = new WorkingSet();
PlanStage *root, *backupRoot=NULL;
verify(StageBuilder::build(txn, collection, *qs, sharedWs, &root));
if ((plannerParams.options & QueryPlannerParams::PRIVATE_IS_COUNT)
&& turnIxscanIntoCount(qs)) {
LOG(2) << "Using fast count: " << canonicalQuery->toStringShort()
<< ", planSummary: " << getPlanSummary(*qs);
if (NULL != backupQs) {
delete backupQs;
}
}
else if (NULL != backupQs) {
verify(StageBuilder::build(txn, collection, *backupQs, sharedWs, &backupRoot));
}
// add a CachedPlanStage on top of the previous root
root = new CachedPlanStage(collection, canonicalQuery.get(), root, backupRoot);
*out = new PlanExecutor(sharedWs, root, chosenSolution, canonicalQuery.release(),
collection);
return Status::OK();
}
}
if (internalQueryPlanOrChildrenIndependently
&& SubplanStage::canUseSubplanning(*canonicalQuery)) {
QLOG() << "Running query as sub-queries: " << canonicalQuery->toStringShort();
auto_ptr<WorkingSet> ws(new WorkingSet());
SubplanStage* subplan;
Status subplanStatus = SubplanStage::make(txn, collection, ws.get(), plannerParams,
canonicalQuery.get(), &subplan);
if (subplanStatus.isOK()) {
LOG(2) << "Running query as sub-queries: " << canonicalQuery->toStringShort();
*out = new PlanExecutor(ws.release(), subplan, canonicalQuery.release(),
collection);
return Status::OK();
}
else {
QLOG() << "Subplanner: " << subplanStatus.reason();
}
}
return getExecutorAlwaysPlan(txn, collection, canonicalQuery.release(), plannerParams, out);
}
/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(CanonicalQuery* rawCanonicalQuery, Runner** out, size_t plannerOptions) {
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// Try to look up a cached solution for the query.
// TODO: Can the cache have negative data about a solution?
PlanCache* localCache = PlanCache::get(canonicalQuery->ns());
if (NULL != localCache) {
CachedSolution* cs = localCache->get(*canonicalQuery);
if (NULL != cs) {
// We have a cached solution. Hand the canonical query and cached solution off to
// the cached plan runner, which takes ownership of both.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*cs->solution, &root, &ws));
*out = new CachedPlanRunner(canonicalQuery.release(), cs, root, ws);
return Status::OK();
}
}
// No entry in cache for the query. We have to solve the query ourself.
// Get the indices that we could possibly use.
Database* db = cc().database();
verify( db );
Collection* collection = db->getCollection( canonicalQuery->ns() );
// This can happen as we're called by internal clients as well.
if (NULL == collection) {
const string& ns = canonicalQuery->ns();
*out = new EOFRunner(canonicalQuery.release(), ns);
return Status::OK();
}
// If we have an _id index we can use the idhack runner.
if (canUseIDHack(*canonicalQuery) && collection->getIndexCatalog()->findIdIndex()) {
*out = new IDHackRunner(collection, canonicalQuery.release());
return Status::OK();
}
// If it's not NULL, we may have indices. Access the catalog and fill out IndexEntry(s)
QueryPlannerParams plannerParams;
for (int i = 0; i < collection->getIndexCatalog()->numIndexesReady(); ++i) {
IndexDescriptor* desc = collection->getIndexCatalog()->getDescriptor( i );
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName()));
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!collection->isCapped()) {
return Status(ErrorCodes::BadValue,
"tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// Process the planning options.
plannerParams.options = plannerOptions;
if (storageGlobalParams.noTableScan) {
const string& ns = canonicalQuery->ns();
// There are certain cases where we ignore this restriction:
bool ignore = canonicalQuery->getQueryObj().isEmpty()
|| (string::npos != ns.find(".system."))
|| (0 == ns.find("local."));
if (!ignore) {
plannerParams.options |= QueryPlannerParams::NO_TABLE_SCAN;
}
}
if (!(plannerParams.options & QueryPlannerParams::NO_TABLE_SCAN)) {
plannerParams.options |= QueryPlannerParams::INCLUDE_COLLSCAN;
}
// If the caller wants a shard filter, make sure we're actually sharded.
if (plannerParams.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
CollectionMetadataPtr collMetadata = shardingState.getCollectionMetadata(canonicalQuery->ns());
if (collMetadata) {
plannerParams.shardKey = collMetadata->getKeyPattern();
}
else {
// If there's no metadata don't bother w/the shard filter since we won't know what
// the key pattern is anyway...
plannerParams.options &= ~QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
}
vector<QuerySolution*> solutions;
QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions);
/*
for (size_t i = 0; i < solutions.size(); ++i) {
QLOG() << "solution " << i << " is " << solutions[i]->toString() << endl;
}
*/
// We cannot figure out how to answer the query. Should this ever happen?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue, "No query solutions");
}
if (1 == solutions.size()) {
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
return Status::OK();
}
else {
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}
/**
* For a given query, get a runner. The runner could be a SingleSolutionRunner, a
* CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
*/
Status getRunner(Collection* collection, CanonicalQuery* rawCanonicalQuery,
Runner** out, size_t plannerOptions) {
verify(rawCanonicalQuery);
auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);
// This can happen as we're called by internal clients as well.
if (NULL == collection) {
const string& ns = canonicalQuery->ns();
*out = new EOFRunner(canonicalQuery.release(), ns);
return Status::OK();
}
// If we have an _id index we can use the idhack runner.
if (canUseIDHack(*canonicalQuery) && collection->getIndexCatalog()->findIdIndex()) {
*out = new IDHackRunner(collection, canonicalQuery.release());
return Status::OK();
}
// If it's not NULL, we may have indices. Access the catalog and fill out IndexEntry(s)
QueryPlannerParams plannerParams;
IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false);
while (ii.more()) {
const IndexDescriptor* desc = ii.next();
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName(),
desc->infoObj()));
}
// If query supports admin hint, filter params.indices by indexes in query settings.
QuerySettings* querySettings = collection->infoCache()->getQuerySettings();
AllowedIndices* allowedIndicesRaw;
// Filter index catalog if admin hint is specified for query.
// Also, signal to planner that application hint should be ignored.
if (querySettings->getAllowedIndices(*canonicalQuery, &allowedIndicesRaw)) {
boost::scoped_ptr<AllowedIndices> allowedIndices(allowedIndicesRaw);
filterAllowedIndexEntries(*allowedIndices, &plannerParams.indices);
plannerParams.adminHintApplied = true;
}
// Tailable: If the query requests tailable the collection must be capped.
if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
if (!collection->isCapped()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" tailable cursor requested on non capped collection");
}
// If a sort is specified it must be equal to expectedSort.
const BSONObj expectedSort = BSON("$natural" << 1);
const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" invalid sort specified for tailable cursor: "
+ actualSort.toString());
}
}
// Process the planning options.
plannerParams.options = plannerOptions;
if (storageGlobalParams.noTableScan) {
const string& ns = canonicalQuery->ns();
// There are certain cases where we ignore this restriction:
bool ignore = canonicalQuery->getQueryObj().isEmpty()
|| (string::npos != ns.find(".system."))
|| (0 == ns.find("local."));
if (!ignore) {
plannerParams.options |= QueryPlannerParams::NO_TABLE_SCAN;
}
}
if (!(plannerParams.options & QueryPlannerParams::NO_TABLE_SCAN)) {
plannerParams.options |= QueryPlannerParams::INCLUDE_COLLSCAN;
}
// If the caller wants a shard filter, make sure we're actually sharded.
if (plannerParams.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
CollectionMetadataPtr collMetadata =
shardingState.getCollectionMetadata(canonicalQuery->ns());
if (collMetadata) {
plannerParams.shardKey = collMetadata->getKeyPattern();
}
else {
// If there's no metadata don't bother w/the shard filter since we won't know what
// the key pattern is anyway...
plannerParams.options &= ~QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
}
// Try to look up a cached solution for the query.
//
// Skip cache look up for non-cacheable queries.
// See PlanCache::shouldCacheQuery()
//
// TODO: Can the cache have negative data about a solution?
CachedSolution* rawCS;
if (PlanCache::shouldCacheQuery(*canonicalQuery) &&
collection->infoCache()->getPlanCache()->get(*canonicalQuery, &rawCS).isOK()) {
// We have a CachedSolution. Have the planner turn it into a QuerySolution.
boost::scoped_ptr<CachedSolution> cs(rawCS);
QuerySolution *qs, *backupQs;
Status status = QueryPlanner::planFromCache(*canonicalQuery, plannerParams, *cs,
&qs, &backupQs);
if (status.isOK()) {
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*qs, &root, &ws));
CachedPlanRunner* cpr = new CachedPlanRunner(canonicalQuery.release(), qs,
root, ws);
if (NULL != backupQs) {
WorkingSet* backupWs;
PlanStage* backupRoot;
verify(StageBuilder::build(*backupQs, &backupRoot, &backupWs));
cpr->setBackupPlan(backupQs, backupRoot, backupWs);
}
*out = cpr;
return Status::OK();
}
}
plannerParams.options |= QueryPlannerParams::INDEX_INTERSECTION;
plannerParams.options |= QueryPlannerParams::KEEP_MUTATIONS;
vector<QuerySolution*> solutions;
Status status = QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions);
if (!status.isOK()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" planner returned error: " + status.reason());
}
/*
for (size_t i = 0; i < solutions.size(); ++i) {
QLOG() << "solution " << i << " is " << solutions[i]->toString() << endl;
}
*/
// We cannot figure out how to answer the query. Should this ever happen?
if (0 == solutions.size()) {
return Status(ErrorCodes::BadValue,
"error processing query: " + canonicalQuery->toString() +
" No query solutions");
}
if (1 == solutions.size()) {
// Only one possible plan. Run it. Build the stages from the solution.
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(*solutions[0], &root, &ws));
// And, run the plan.
*out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
return Status::OK();
}
else {
// Many solutions. Let the MultiPlanRunner pick the best, update the cache, and so on.
auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
for (size_t i = 0; i < solutions.size(); ++i) {
WorkingSet* ws;
PlanStage* root;
if (solutions[i]->cacheData.get()) {
solutions[i]->cacheData->adminHintApplied = plannerParams.adminHintApplied;
}
verify(StageBuilder::build(*solutions[i], &root, &ws));
// Takes ownership of all arguments.
mpr->addPlan(solutions[i], root, ws);
}
*out = mpr.release();
return Status::OK();
}
}