void run() {
// We insert lots of copies of {a:1, b:1, c: 20}. We have the indices {a:1} and {b:1},
// and the query is {a:1, b:1, c: 999}. No data that matches the query but we won't
// know that during plan ranking. We don't want to choose an intersection plan here.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << 1 << "b" << 1 << "c" << 20));
}
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// There is no data that matches this query but we don't know that until EOF.
CanonicalQuery* cq;
BSONObj queryObj = BSON("a" << 1 << "b" << 1 << "c" << 99);
ASSERT(CanonicalQuery::canonicalize(ns, queryObj, &cq).isOK());
ASSERT(NULL != cq);
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
// Anti-prefer the intersection plan.
bool bestIsScanOverA = QueryPlannerTestLib::solutionMatches(
"{fetch: {node: {ixscan: {pattern: {a: 1}}}}}",
soln->root.get());
bool bestIsScanOverB = QueryPlannerTestLib::solutionMatches(
"{fetch: {node: {ixscan: {pattern: {b: 1}}}}}",
soln->root.get());
ASSERT(bestIsScanOverA || bestIsScanOverB);
}
void run() {
for (int i = 0; i < N; ++i) {
insert(BSON("a" << 1));
insert(BSON("a" << 1 << "b" << 1 << "c" << i));
}
// Indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// Solutions using either 'a' or 'b' will take a long time to start producing
// results. However, an index scan on 'b' will start producing results sooner
// than an index scan on 'a'.
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
fromjson("{a: 1, b: 1, c: {$gte: 5000}}"),
&cq).isOK());
ASSERT(NULL != cq);
// Use index on 'b'.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {node: {ixscan: {pattern: {b: 1}}}}}",
soln->root.get()));
}
void run() {
// Set up the data so that for the query {a: 1, b: 1, c: 1}, the intersection
// between 'b' and 'c' is small, and the other intersections are larger.
for (int i = 0; i < 10; ++i) {
insert(BSON("a" << 1 << "b" << 1 << "c" << 1));
}
for (int i = 0; i < 10; ++i) {
insert(BSON("a" << 2 << "b" << 1 << "c" << 1));
}
for (int i = 0; i < N/2; ++i) {
insert(BSON("a" << 1 << "b" << 1 << "c" << 2));
insert(BSON("a" << 1 << "b" << 2 << "c" << 1));
}
// Add indices on 'a', 'b', and 'c'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
addIndex(BSON("c" << 1));
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
fromjson("{a: 1, b: 1, c: 1}"),
&cq).isOK());
ASSERT(NULL != cq);
// Intersection between 'b' and 'c' should hit EOF while the
// other plans are busy fetching documents.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: {a:1}, node: {andSorted: {nodes: ["
"{ixscan: {filter: null, pattern: {b:1}}},"
"{ixscan: {filter: null, pattern: {c:1}}}]}}}}",
soln->root.get()));
}
void run() {
// We insert lots of copies of {a:1, b:1}. We have the indices {a:1} and {a:1, b:1},
// the query is for a doc that doesn't exist, but there is a projection over 'a' and
// 'b'. We should prefer the index that provides a covered query.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << 1 << "b" << 1));
}
addIndex(BSON("a" << 1));
addIndex(BSON("a" << 1 << "b" << 1));
// There is no data that matches this query ({a:2}). Both scans will hit EOF before
// returning any data.
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
BSON("a" << 2),
BSONObj(),
BSON("_id" << 0 << "a" << 1 << "b" << 1),
&cq).isOK());
ASSERT(NULL != cq);
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
// Prefer the fully covered plan.
ASSERT(QueryPlannerTestLib::solutionMatches(
"{proj: {spec: {_id:0, a:1, b:1}, node: {ixscan: {pattern: {a: 1, b:1}}}}}",
soln->root.get()));
}
Runner::RunnerState MultiPlanRunner::getNext(BSONObj* objOut) {
if (_failure || _killed) {
return Runner::RUNNER_DEAD;
}
// If we haven't picked the best plan yet...
if (NULL == _bestPlan) {
// TODO: Consider rewriting pickBestPlan to return results as it iterates.
if (!pickBestPlan(NULL)) {
verify(_failure);
return Runner::RUNNER_DEAD;
}
}
if (!_alreadyProduced.empty()) {
WorkingSetID id = _alreadyProduced.front();
_alreadyProduced.pop();
WorkingSetMember* member = _bestPlan->getWorkingSet()->get(id);
// TODO: getOwned?
verify(WorkingSetCommon::fetch(member));
*objOut = member->obj;
_bestPlan->getWorkingSet()->free(id);
return Runner::RUNNER_ADVANCED;
}
return _bestPlan->getNext(objOut);
}
void run() {
// Simulate needing lots of FETCH's.
turnOnAlwaysFetch();
// Neither 'a' nor 'b' is selective.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << 1 << "b" << 1));
}
// Add indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// Query {a:1, b:1}, and project out all fields other than 'a' and 'b'.
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
BSON("a" << 1 << "b" << 1),
BSONObj(),
BSON("_id" << 0 << "a" << 1 << "b" << 1),
&cq).isOK());
ASSERT(NULL != cq);
// We should choose an ixisect plan because it requires fewer fetches.
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{proj: {spec: {_id:0,a:1,b:1}, node: {andSorted: {nodes: ["
"{ixscan: {filter: null, pattern: {a:1}}},"
"{ixscan: {filter: null, pattern: {b:1}}}]}}}}",
soln->root.get()));
turnOffAlwaysFetch();
}
void run() {
for (int i = 0; i < N; ++i) {
insert(BSON("a" << 1 << "d" << i));
}
// The index {d: 1, e: 1} provides the desired sort order,
// while index {a: 1, b: 1} can be used to answer the
// query predicate, but does not provide the sort.
addIndex(BSON("a" << 1 << "b" << 1));
addIndex(BSON("d" << 1 << "e" << 1));
// Query: find({a: 1}).sort({d: 1})
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
BSON("a" << 1),
BSON("d" << 1), // sort
BSONObj(), // projection
&cq).isOK());
ASSERT(NULL != cq);
// No results will be returned during the trial period,
// so we expect to choose {d: 1, e: 1}, as it allows us
// to avoid the sort stage.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: {a:1}, node: "
"{ixscan: {filter: null, pattern: {d:1,e:1}}}}}",
soln->root.get()));
}
void run() {
// Simulate needing lots of FETCH's.
turnOnAlwaysFetch();
// Neither 'a' nor 'b' is selective.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << 1 << "b" << 1));
}
// Add indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// Query {a:1, b:1}.
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
BSON("a" << 1 << "b" << 1),
&cq).isOK());
ASSERT(NULL != cq);
// The intersection is large, and ixisect does not make the
// query covered. We should NOT choose an intersection plan.
QuerySolution* soln = pickBestPlan(cq);
bool bestIsScanOverA = QueryPlannerTestLib::solutionMatches(
"{fetch: {node: {ixscan: {pattern: {a: 1}}}}}",
soln->root.get());
bool bestIsScanOverB = QueryPlannerTestLib::solutionMatches(
"{fetch: {node: {ixscan: {pattern: {b: 1}}}}}",
soln->root.get());
ASSERT(bestIsScanOverA || bestIsScanOverB);
turnOffAlwaysFetch();
}
void run() {
for (int i = 0; i < N; ++i) {
insert(BSON("_id" << i));
}
addIndex(BSON("_id" << 1));
// Run a query with a sort. The blocking sort won't produce any data during the
// evaluation period.
CanonicalQuery* cq;
BSONObj queryObj = BSON("_id" << BSON("$gte" << 20 << "$lte" << 200));
BSONObj sortObj = BSON("c" << 1);
BSONObj projObj = BSONObj();
ASSERT(CanonicalQuery::canonicalize(ns,
queryObj,
sortObj,
projObj,
&cq).isOK());
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
// The best must not be a collscan.
ASSERT(QueryPlannerTestLib::solutionMatches(
"{sort: {pattern: {c: 1}, limit: 0, node: {"
"fetch: {filter: null, node: "
"{ixscan: {filter: null, pattern: {_id: 1}}}}}}}}",
soln->root.get()));
}
void run() {
// Set up the data so that for the query {a: 1, b: 1}, the
// intersection is empty. The single index plans have to do
// more fetching from disk in order to determine that the result
// set is empty. As a result, the intersection plan hits EOF first.
for (int i = 0; i < 30; ++i) {
insert(BSON("a" << 1 << "b" << 2));
}
for (int i = 0; i < 30; ++i) {
insert(BSON("a" << 2 << "b" << 1));
}
for (int i = 0; i < N; ++i) {
insert(BSON("a" << 2 << "b" << 2));
}
// Add indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
fromjson("{a: 1, b: 1}"),
&cq).isOK());
ASSERT(NULL != cq);
// Choose the index intersection plan.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: null, node: {andSorted: {nodes: ["
"{ixscan: {filter: null, pattern: {a:1}}},"
"{ixscan: {filter: null, pattern: {b:1}}}]}}}}",
soln->root.get()));
}
void run() {
// Insert data {a:i, b:i}. Index {a:1} and {a:1, b:1}, query on 'a', projection on 'a'
// and 'b'. Should prefer the second index as we can pull the 'b' data out.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << i << "b" << i));
}
addIndex(BSON("a" << 1));
addIndex(BSON("a" << 1 << "b" << 1));
// Query for a==27 with projection that wants 'a' and 'b'. BSONObj() is for sort.
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
BSON("a" << 27),
BSONObj(),
BSON("_id" << 0 << "a" << 1 << "b" << 1),
&cq).isOK());
ASSERT(NULL != cq);
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
// Prefer the fully covered plan.
ASSERT(QueryPlannerTestLib::solutionMatches(
"{proj: {spec: {_id:0, a:1, b:1}, node: {ixscan: {pattern: {a: 1, b:1}}}}}",
soln->root.get()));
}
void run() {
// 'a' is very selective, 'b' is not.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << i << "b" << 1));
}
// Add indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// Run the query {a:1, b:{$gt:1}.
CanonicalQuery* cq;
verify(CanonicalQuery::canonicalize(ns, BSON("a" << 1 << "b" << BSON("$gt" << 1)),
&cq).isOK());
ASSERT(NULL != cq);
// Turn on the "force intersect" option.
// This will be reverted by PlanRankingTestBase's destructor when the test completes.
internalQueryForceIntersectionPlans = true;
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: null, node: {andHash: {nodes: ["
"{ixscan: {filter: null, pattern: {a:1}}},"
"{ixscan: {filter: null, pattern: {b:1}}}]}}}}",
soln->root.get()));
// Confirm that a backup plan is available.
ASSERT(hasBackupPlan());
}
void run() {
// 'a' is very selective, 'b' is not.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << i << "b" << 1));
}
// Add indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// Run the query {a:4, b:1}.
CanonicalQuery* cq;
verify(CanonicalQuery::canonicalize(ns, BSON("a" << 100 << "b" << 1), &cq).isOK());
ASSERT(NULL != cq);
// {a:100} is super selective so choose that.
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: {b:1}, node: {ixscan: {pattern: {a: 1}}}}}",
soln->root.get()));
// Turn on the "force intersect" option.
// This will be reverted by PlanRankingTestBase's destructor when the test completes.
internalQueryForceIntersectionPlans = true;
// And run the same query again.
ASSERT(CanonicalQuery::canonicalize(ns, BSON("a" << 100 << "b" << 1), &cq).isOK());
// With the "ranking picks ixisect always" option we pick an intersection plan that uses
// both the {a:1} and {b:1} indices even though it performs poorly.
// Takes ownership of cq.
soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: null, node: {andSorted: {nodes: ["
"{ixscan: {filter: null, pattern: {a:1}}},"
"{ixscan: {filter: null, pattern: {b:1}}}]}}}}",
soln->root.get()));
}
Runner::RunnerState MultiPlanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) {
if (_killed) { return Runner::RUNNER_DEAD; }
if (_failure) { return Runner::RUNNER_ERROR; }
// If we haven't picked the best plan yet...
if (NULL == _bestPlan) {
if (!pickBestPlan(NULL)) {
verify(_failure || _killed);
if (_killed) { return Runner::RUNNER_DEAD; }
if (_failure) { return Runner::RUNNER_ERROR; }
}
}
if (!_alreadyProduced.empty()) {
WorkingSetID id = _alreadyProduced.front();
_alreadyProduced.pop_front();
WorkingSetMember* member = _bestPlan->getWorkingSet()->get(id);
// Note that this copies code from PlanExecutor.
if (NULL != objOut) {
if (WorkingSetMember::LOC_AND_IDX == member->state) {
if (1 != member->keyData.size()) {
_bestPlan->getWorkingSet()->free(id);
return Runner::RUNNER_ERROR;
}
*objOut = member->keyData[0].keyData;
}
else if (member->hasObj()) {
*objOut = member->obj;
}
else {
// TODO: Checking the WSM for covered fields goes here.
_bestPlan->getWorkingSet()->free(id);
return Runner::RUNNER_ERROR;
}
}
if (NULL != dlOut) {
if (member->hasLoc()) {
*dlOut = member->loc;
}
else {
_bestPlan->getWorkingSet()->free(id);
return Runner::RUNNER_ERROR;
}
}
_bestPlan->getWorkingSet()->free(id);
return Runner::RUNNER_ADVANCED;
}
return _bestPlan->getNext(objOut, dlOut);
}
void run() {
// Insert data for which we have no index.
for (int i = 0; i < N; ++i) {
insert(BSON("foo" << i));
}
// Look for A Space Odyssey.
CanonicalQuery* cq;
verify(CanonicalQuery::canonicalize(ns, BSON("foo" << 2001), &cq).isOK());
ASSERT(NULL != cq);
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
// The best must be a collscan.
ASSERT(QueryPlannerTestLib::solutionMatches(
"{cscan: {dir: 1, filter: {foo: 2001}}}",
soln->root.get()));
}
void run() {
// Simulate needing lots of FETCH's.
turnOnAlwaysFetch();
// Set up data so that the following conditions hold:
// 1) Documents matching {a: 1} are of high cardinality.
// 2) Documents matching {b: 1} are of high cardinality.
// 3) Documents matching {a: 1, b: 1} are of low cardinality---
// the intersection is small.
// 4) At least one of the documents in the intersection is
// returned during the trial period.
insert(BSON("a" << 1 << "b" << 1));
for (int i = 0; i < N/2; ++i) {
insert(BSON("a" << 1 << "b" << 2));
}
for (int i = 0; i < N/2; ++i) {
insert(BSON("a" << 2 << "b" << 1));
}
// Add indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// Neither the predicate on 'b' nor the predicate on 'a' is
// very selective: both retrieve about half the documents.
// However, the intersection is very small, which makes
// the intersection plan desirable.
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
fromjson("{a: 1, b: 1}"),
&cq).isOK());
ASSERT(NULL != cq);
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: null, node: {andSorted: {nodes: ["
"{ixscan: {filter: null, pattern: {a:1}}},"
"{ixscan: {filter: null, pattern: {b:1}}}]}}}}",
soln->root.get()));
turnOffAlwaysFetch();
}
void run() {
// 'a' is very selective, 'b' is not.
for (int i = 0; i < N; ++i) {
insert(BSON("a" << i << "b" << 1));
}
// Add indices on 'a' and 'b'.
addIndex(BSON("a" << 1));
addIndex(BSON("b" << 1));
// Run the query {a:N+1, b:1}. (No such document.)
CanonicalQuery* cq;
verify(CanonicalQuery::canonicalize(ns, BSON("a" << N + 1 << "b" << 1), &cq).isOK());
ASSERT(NULL != cq);
// {a: 100} is super selective so choose that.
// Takes ownership of cq.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {filter: {b:1}, node: {ixscan: {pattern: {a: 1}}}}}",
soln->root.get()));
}
void run() {
for (int i = 0; i < 100; ++i) {
insert(BSON("a" << i << "b" << i << "c" << i));
}
// These indices look equivalent to the ranker for the query below unless we account
// for key skipping. We should pick index {a: 1} if we account for key skipping
// properly.
addIndex(BSON("b" << 1 << "c" << 1));
addIndex(BSON("a" << 1));
CanonicalQuery* cq;
ASSERT(CanonicalQuery::canonicalize(ns,
fromjson("{a: 9, b: {$ne: 10}, c: 9}"),
&cq).isOK());
ASSERT(NULL != cq);
// Expect to use index {a: 1, b: 1}.
QuerySolution* soln = pickBestPlan(cq);
ASSERT(QueryPlannerTestLib::solutionMatches(
"{fetch: {node: {ixscan: {pattern: {a: 1}}}}}",
soln->root.get()));
}
Runner::RunnerState MultiPlanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) {
if (_killed) { return Runner::RUNNER_DEAD; }
if (_failure) { return Runner::RUNNER_ERROR; }
// If we haven't picked the best plan yet...
if (NULL == _bestPlan) {
if (!pickBestPlan(NULL, objOut)) {
verify(_failure || _killed);
if (_killed) { return Runner::RUNNER_DEAD; }
if (_failure) { return Runner::RUNNER_ERROR; }
}
}
// Look for an already produced result that provides the data the caller wants.
while (!_alreadyProduced.empty()) {
WorkingSetID id = _alreadyProduced.front();
_alreadyProduced.pop_front();
WorkingSetMember* member = _bestPlan->getWorkingSet()->get(id);
// Note that this copies code from PlanExecutor.
if (NULL != objOut) {
if (WorkingSetMember::LOC_AND_IDX == member->state) {
if (1 != member->keyData.size()) {
_bestPlan->getWorkingSet()->free(id);
// If the caller needs the key data and the WSM doesn't have it, drop the
// result and carry on.
continue;
}
*objOut = member->keyData[0].keyData;
}
else if (member->hasObj()) {
*objOut = member->obj;
}
else {
// If the caller needs an object and the WSM doesn't have it, drop and
// try the next result.
_bestPlan->getWorkingSet()->free(id);
continue;
}
}
if (NULL != dlOut) {
if (member->hasLoc()) {
*dlOut = member->loc;
}
else {
// If the caller needs a DiskLoc and the WSM doesn't have it, drop and carry on.
_bestPlan->getWorkingSet()->free(id);
continue;
}
}
// If we're here, the caller has all the data needed and we've set the out
// parameters. Remove the result from the WorkingSet.
_bestPlan->getWorkingSet()->free(id);
return Runner::RUNNER_ADVANCED;
}
RunnerState state = _bestPlan->getNext(objOut, dlOut);
if (Runner::RUNNER_ERROR == state && (NULL != _backupSolution)) {
QLOG() << "Best plan errored out switching to backup\n";
// Uncache the bad solution if we fall back
// on the backup solution.
//
// XXX: Instead of uncaching we should find a way for the
// cached plan runner to fall back on a different solution
// if the best solution fails. Alternatively we could try to
// defer cache insertion to be after the first produced result.
Database* db = cc().database();
verify(NULL != db);
Collection* collection = db->getCollection(_query->ns());
verify(NULL != collection);
PlanCache* cache = collection->infoCache()->getPlanCache();
cache->remove(*_query);
_bestPlan.reset(_backupPlan);
_backupPlan = NULL;
_bestSolution.reset(_backupSolution);
_backupSolution = NULL;
_alreadyProduced = _backupAlreadyProduced;
return getNext(objOut, dlOut);
}
if (NULL != _backupSolution && Runner::RUNNER_ADVANCED == state) {
QLOG() << "Best plan had a blocking sort, became unblocked, deleting backup plan\n";
delete _backupSolution;
delete _backupPlan;
_backupSolution = NULL;
_backupPlan = NULL;
// TODO: free from WS?
_backupAlreadyProduced.clear();
}
return state;
}