Ejemplo n.º 1
0
    void CachedPlanRunner::updateCache() {
        _updatedCache = true;

        if (_killed) {
            return;
        }

        Database* db = cc().database();
        // XXX: We need to check for NULL because this is called upon
        // destruction of the CachedPlanRunner. In some cases, the db
        // or collection could be dropped without kill() being called
        // on the runner (for example, timeout of a ClientCursor holding
        // the runner).
        if (NULL == db) { return; }
        Collection* collection = db->getCollection(_canonicalQuery->ns());
        if (NULL == collection) { return; }
        PlanCache* cache = collection->infoCache()->getPlanCache();

        std::auto_ptr<PlanCacheEntryFeedback> feedback(new PlanCacheEntryFeedback());
        // XXX: what else can we provide here?
        feedback->stats.reset(_exec->getStats());
        feedback->score = PlanRanker::scoreTree(feedback->stats.get());

        Status fbs = cache->feedback(*_canonicalQuery, feedback.release());

        if (!fbs.isOK()) {
            QLOG() << _canonicalQuery->ns() << ": Failed to update cache with feedback: "
                   << fbs.toString() << " - "
                   << "(query: " << _canonicalQuery->getQueryObj()
                   << "; sort: " << _canonicalQuery->getParsed().getSort()
                   << "; projection: " << _canonicalQuery->getParsed().getProj()
                   << ") is no longer in plan cache.";
        }
    }
Ejemplo n.º 2
0
    void CachedPlanRunner::updateCache() {
        _updatedCache = true;

        // We're done.  Update the cache.
        PlanCache* cache = PlanCache::get(_canonicalQuery->ns());

        // TODO: Is this an error?
        if (NULL == cache) { return; }

        // TODO: How do we decide this?
        bool shouldRemovePlan = false;

        if (shouldRemovePlan) {
            if (!cache->remove(*_canonicalQuery, *_cachedQuery->solution)) {
                warning() << "Cached plan runner couldn't remove plan from cache.  Maybe"
                    " somebody else did already?";
                return;
            }
        }

        // We're done running.  Update cache.
        auto_ptr<CachedSolutionFeedback> feedback(new CachedSolutionFeedback());
        feedback->stats = _exec->getStats();
        cache->feedback(*_canonicalQuery, *_cachedQuery->solution, feedback.release());
    }
Ejemplo n.º 3
0
    /**
     * For a given query, get a runner.  The runner could be a SingleSolutionRunner, a
     * CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
     */
    Status getRunner(QueryMessage& q, Runner** out) {
        CanonicalQuery* rawCanonicalQuery = NULL;

        // Canonicalize the query and wrap it in an auto_ptr so we don't leak it if something goes
        // wrong.
        Status status = CanonicalQuery::canonicalize(q, &rawCanonicalQuery);
        if (!status.isOK()) { return status; }
        verify(rawCanonicalQuery);
        auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);

        // Try to look up a cached solution for the query.
        // TODO: Can the cache have negative data about a solution?
        PlanCache* localCache = PlanCache::get(canonicalQuery->ns());
        CachedSolution* cs = localCache->get(*canonicalQuery);
        if (NULL != cs) {
            // We have a cached solution.  Hand the canonical query and cached solution off to the
            // cached plan runner, which takes ownership of both.
            WorkingSet* ws;
            PlanStage* root;
            verify(StageBuilder::build(*cs->solution, &root, &ws));
            *out = new CachedPlanRunner(canonicalQuery.release(), cs, root, ws);
            return Status::OK();
        }

        // No entry in cache for the query.  We have to solve the query ourself.
        vector<QuerySolution*> solutions;
        QueryPlanner::plan(*canonicalQuery, &solutions);

        // We cannot figure out how to answer the query.  Should this ever happen?
        if (0 == solutions.size()) {
            return Status(ErrorCodes::BadValue, "Can't create a plan for the canonical query " +
                                                 canonicalQuery->toString());
        }

        if (1 == solutions.size()) {
            // Only one possible plan.  Run it.  Build the stages from the solution.
            WorkingSet* ws;
            PlanStage* root;
            verify(StageBuilder::build(*solutions[0], &root, &ws));

            // And, run the plan.
            *out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
            return Status::OK();
        }
        else {
            // Many solutions.  Let the MultiPlanRunner pick the best, update the cache, and so on.
            auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
            for (size_t i = 0; i < solutions.size(); ++i) {
                WorkingSet* ws;
                PlanStage* root;
                verify(StageBuilder::build(*solutions[i], &root, &ws));
                // Takes ownership of all arguments.
                mpr->addPlan(solutions[i], root, ws);
            }
            *out = mpr.release();
            return Status::OK();
        }
    }
Ejemplo n.º 4
0
    void run() {
        AutoGetCollectionForRead ctx(&_txn, nss.ns());
        Collection* collection = ctx.getCollection();
        ASSERT(collection);

        // Query can be answered by either index on "a" or index on "b".
        auto statusWithCQ = CanonicalQuery::canonicalize(nss, fromjson("{a: {$gte: 8}, b: 1}"));
        ASSERT_OK(statusWithCQ.getStatus());
        const std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());

        // We shouldn't have anything in the plan cache for this shape yet.
        PlanCache* cache = collection->infoCache()->getPlanCache();
        ASSERT(cache);
        CachedSolution* rawCachedSolution;
        ASSERT_NOT_OK(cache->get(*cq, &rawCachedSolution));

        // Get planner params.
        QueryPlannerParams plannerParams;
        fillOutPlannerParams(&_txn, collection, cq.get(), &plannerParams);

        // Queued data stage will return a failure during the cached plan trial period.
        auto mockChild = stdx::make_unique<QueuedDataStage>(&_txn, &_ws);
        mockChild->pushBack(PlanStage::FAILURE);

        // High enough so that we shouldn't trigger a replan based on works.
        const size_t decisionWorks = 50;
        CachedPlanStage cachedPlanStage(
            &_txn, collection, &_ws, cq.get(), plannerParams, decisionWorks, mockChild.release());

        // This should succeed after triggering a replan.
        PlanYieldPolicy yieldPolicy(nullptr, PlanExecutor::YIELD_MANUAL);
        ASSERT_OK(cachedPlanStage.pickBestPlan(&yieldPolicy));

        // Make sure that we get 2 legit results back.
        size_t numResults = 0;
        PlanStage::StageState state = PlanStage::NEED_TIME;
        while (state != PlanStage::IS_EOF) {
            WorkingSetID id = WorkingSet::INVALID_ID;
            state = cachedPlanStage.work(&id);

            ASSERT_NE(state, PlanStage::FAILURE);
            ASSERT_NE(state, PlanStage::DEAD);

            if (state == PlanStage::ADVANCED) {
                WorkingSetMember* member = _ws.get(id);
                ASSERT(cq->root()->matchesBSON(member->obj.value()));
                numResults++;
            }
        }

        ASSERT_EQ(numResults, 2U);

        // Plan cache should still be empty, as we don't write to it when we replan a failed
        // query.
        ASSERT_NOT_OK(cache->get(*cq, &rawCachedSolution));
    }
Ejemplo n.º 5
0
void CachedPlanStage::updatePlanCache() {
    const double score = PlanRanker::scoreTree(getStats()->children[0].get());

    PlanCache* cache = collection()->infoCache()->getPlanCache();
    Status fbs = cache->feedback(*_canonicalQuery, score);
    if (!fbs.isOK()) {
        LOG(5) << _canonicalQuery->ns() << ": Failed to update cache with feedback: " << redact(fbs)
               << " - "
               << "(query: " << redact(_canonicalQuery->getQueryObj())
               << "; sort: " << _canonicalQuery->getQueryRequest().getSort()
               << "; projection: " << _canonicalQuery->getQueryRequest().getProj()
               << ") is no longer in plan cache.";
    }
}
TEST_F(QueryStageCachedPlan, DeactivatesEntriesOnReplan) {
    AutoGetCollectionForReadCommand ctx(&_opCtx, nss);
    Collection* collection = ctx.getCollection();
    ASSERT(collection);

    // Never run - just used as a key for the cache's get() functions, since all of the other
    // CanonicalQueries created in this test will have this shape.
    const auto shapeCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 123}, b: {$gte: 123}}"));

    // Query can be answered by either index on "a" or index on "b".
    const auto noResultsCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 11}, b: {$gte: 11}}"));

    // We shouldn't have anything in the plan cache for this shape yet.
    PlanCache* cache = collection->infoCache()->getPlanCache();
    ASSERT(cache);
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kNotPresent);

    // Run the CachedPlanStage with a long-running child plan. Replanning should be
    // triggered and an inactive entry will be added.
    forceReplanning(collection, noResultsCq.get());

    // Check for an inactive cache entry.
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentInactive);

    // Run the plan again, to create an active entry.
    forceReplanning(collection, noResultsCq.get());

    // The works should be 1 for the entry since the query we ran should not have any results.
    ASSERT_EQ(cache->get(*noResultsCq.get()).state, PlanCache::CacheEntryState::kPresentActive);
    auto entry = assertGet(cache->getEntry(*shapeCq));
    size_t works = 1U;
    ASSERT_EQ(entry->works, works);

    // Run another query which takes long enough to evict the active cache entry. The current
    // cache entry's works value is a very low number. When replanning is triggered, the cache
    // entry will be deactivated, but the new plan will not overwrite it, since the new plan will
    // have a higher works. Therefore, we will be left in an inactive entry which has had its works
    // value doubled from 1 to 2.
    auto highWorksCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 0}, b: {$gte:0}}"));
    forceReplanning(collection, highWorksCq.get());
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentInactive);
    ASSERT_EQ(assertGet(cache->getEntry(*shapeCq))->works, 2U);

    // Again, force replanning. This time run the initial query which finds no results. The multi
    // planner will choose a plan with works value lower than the existing inactive
    // entry. Replanning will thus deactivate the existing entry (it's already
    // inactive so this is a noop), then create a new entry with a works value of 1.
    forceReplanning(collection, noResultsCq.get());
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentActive);
    ASSERT_EQ(assertGet(cache->getEntry(*shapeCq))->works, 1U);
}
Ejemplo n.º 7
0
void CachedPlanStage::updatePlanCache() {
    std::unique_ptr<PlanCacheEntryFeedback> feedback = stdx::make_unique<PlanCacheEntryFeedback>();
    feedback->stats = getStats();
    feedback->score = PlanRanker::scoreTree(feedback->stats.get());

    PlanCache* cache = _collection->infoCache()->getPlanCache();
    Status fbs = cache->feedback(*_canonicalQuery, feedback.release());
    if (!fbs.isOK()) {
        LOG(5) << _canonicalQuery->ns()
               << ": Failed to update cache with feedback: " << fbs.toString() << " - "
               << "(query: " << _canonicalQuery->getQueryObj()
               << "; sort: " << _canonicalQuery->getParsed().getSort()
               << "; projection: " << _canonicalQuery->getParsed().getProj()
               << ") is no longer in plan cache.";
    }
}
/**
 * Test the way cache entries are added (either "active" or "inactive") to the plan cache.
 */
TEST_F(QueryStageCachedPlan, QueryStageCachedPlanAddsActiveCacheEntries) {
    AutoGetCollectionForReadCommand ctx(&_opCtx, nss);
    Collection* collection = ctx.getCollection();
    ASSERT(collection);

    // Never run - just used as a key for the cache's get() functions, since all of the other
    // CanonicalQueries created in this test will have this shape.
    const auto shapeCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 123}, b: {$gte: 123}}"));

    // Query can be answered by either index on "a" or index on "b".
    const auto noResultsCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 11}, b: {$gte: 11}}"));

    // We shouldn't have anything in the plan cache for this shape yet.
    PlanCache* cache = collection->infoCache()->getPlanCache();
    ASSERT(cache);
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kNotPresent);

    // Run the CachedPlanStage with a long-running child plan. Replanning should be
    // triggered and an inactive entry will be added.
    forceReplanning(collection, noResultsCq.get());

    // Check for an inactive cache entry.
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentInactive);

    // The works should be 1 for the entry since the query we ran should not have any results.
    auto entry = assertGet(cache->getEntry(*shapeCq));
    size_t works = 1U;
    ASSERT_EQ(entry->works, works);

    const size_t kExpectedNumWorks = 10;
    for (int i = 0; i < std::ceil(std::log(kExpectedNumWorks) / std::log(2)); ++i) {
        works *= 2;
        // Run another query of the same shape, which is less selective, and therefore takes
        // longer).
        auto someResultsCq =
            canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 1}, b: {$gte: 0}}"));
        forceReplanning(collection, someResultsCq.get());

        ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentInactive);
        // The works on the cache entry should have doubled.
        entry = assertGet(cache->getEntry(*shapeCq));
        ASSERT_EQ(entry->works, works);
    }

    // Run another query which takes less time, and be sure an active entry is created.
    auto fewResultsCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 6}, b: {$gte: 0}}"));
    forceReplanning(collection, fewResultsCq.get());

    // Now there should be an active cache entry.
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentActive);
    entry = assertGet(cache->getEntry(*shapeCq));
    // This will query will match {a: 6} through {a:9} (4 works), plus one for EOF = 5 works.
    ASSERT_EQ(entry->works, 5U);
}
/**
 * Test that hitting the cached plan stage trial period's threshold for work cycles causes the
 * query to be replanned. Also verify that the replanning results in a new plan cache entry.
 */
TEST_F(QueryStageCachedPlan, QueryStageCachedPlanHitMaxWorks) {
    AutoGetCollectionForReadCommand ctx(&_opCtx, nss);
    Collection* collection = ctx.getCollection();
    ASSERT(collection);

    // Query can be answered by either index on "a" or index on "b".
    auto qr = stdx::make_unique<QueryRequest>(nss);
    qr->setFilter(fromjson("{a: {$gte: 8}, b: 1}"));
    auto statusWithCQ = CanonicalQuery::canonicalize(opCtx(), std::move(qr));
    ASSERT_OK(statusWithCQ.getStatus());
    const std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());

    // We shouldn't have anything in the plan cache for this shape yet.
    PlanCache* cache = collection->infoCache()->getPlanCache();
    ASSERT(cache);
    ASSERT_EQ(cache->get(*cq).state, PlanCache::CacheEntryState::kNotPresent);

    // Get planner params.
    QueryPlannerParams plannerParams;
    fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams);

    // Set up queued data stage to take a long time before returning EOF. Should be long
    // enough to trigger a replan.
    const size_t decisionWorks = 10;
    const size_t mockWorks =
        1U + static_cast<size_t>(internalQueryCacheEvictionRatio * decisionWorks);
    auto mockChild = stdx::make_unique<QueuedDataStage>(&_opCtx, &_ws);
    for (size_t i = 0; i < mockWorks; i++) {
        mockChild->pushBack(PlanStage::NEED_TIME);
    }

    CachedPlanStage cachedPlanStage(
        &_opCtx, collection, &_ws, cq.get(), plannerParams, decisionWorks, mockChild.release());

    // This should succeed after triggering a replan.
    PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD,
                                _opCtx.getServiceContext()->getFastClockSource());
    ASSERT_OK(cachedPlanStage.pickBestPlan(&yieldPolicy));

    ASSERT_EQ(getNumResultsForStage(_ws, &cachedPlanStage, cq.get()), 2U);

    // This time we expect to find something in the plan cache. Replans after hitting the
    // works threshold result in a cache entry.
    ASSERT_EQ(cache->get(*cq).state, PlanCache::CacheEntryState::kPresentInactive);
}
Ejemplo n.º 10
0
TEST_F(QueryStageCachedPlan, EntriesAreNotDeactivatedWhenInactiveEntriesDisabled) {
    // Set the global flag for disabling active entries.
    internalQueryCacheDisableInactiveEntries.store(true);
    ON_BLOCK_EXIT([] { internalQueryCacheDisableInactiveEntries.store(false); });

    AutoGetCollectionForReadCommand ctx(&_opCtx, nss);
    Collection* collection = ctx.getCollection();
    ASSERT(collection);

    // Never run - just used as a key for the cache's get() functions, since all of the other
    // CanonicalQueries created in this test will have this shape.
    const auto shapeCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 123}, b: {$gte: 123}}"));

    // Query can be answered by either index on "a" or index on "b".
    const auto noResultsCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 11}, b: {$gte: 11}}"));

    // We shouldn't have anything in the plan cache for this shape yet.
    PlanCache* cache = collection->infoCache()->getPlanCache();
    ASSERT(cache);
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kNotPresent);

    // Run the CachedPlanStage with a long-running child plan. Replanning should be
    // triggered and an _active_ entry will be added (since the disableInactiveEntries flag is on).
    forceReplanning(collection, noResultsCq.get());

    // Check for an inactive cache entry.
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentActive);

    // Run the plan again. The entry should still be active.
    forceReplanning(collection, noResultsCq.get());
    ASSERT_EQ(cache->get(*noResultsCq.get()).state, PlanCache::CacheEntryState::kPresentActive);

    // Run another query which takes long enough to evict the active cache entry. After replanning
    // is triggered, be sure that the the cache entry is still active.
    auto highWorksCq =
        canonicalQueryFromFilterObj(opCtx(), nss, fromjson("{a: {$gte: 0}, b: {$gte:0}}"));
    forceReplanning(collection, highWorksCq.get());
    ASSERT_EQ(cache->get(*shapeCq).state, PlanCache::CacheEntryState::kPresentActive);
}
Ejemplo n.º 11
0
/**
 * Test that on failure, the cached plan stage replans the query but does not create a new cache
 * entry.
 */
TEST_F(QueryStageCachedPlan, QueryStageCachedPlanFailure) {
    AutoGetCollectionForReadCommand ctx(&_opCtx, nss);
    Collection* collection = ctx.getCollection();
    ASSERT(collection);

    // Query can be answered by either index on "a" or index on "b".
    auto qr = stdx::make_unique<QueryRequest>(nss);
    qr->setFilter(fromjson("{a: {$gte: 8}, b: 1}"));
    auto statusWithCQ = CanonicalQuery::canonicalize(opCtx(), std::move(qr));
    ASSERT_OK(statusWithCQ.getStatus());
    const std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());

    // We shouldn't have anything in the plan cache for this shape yet.
    PlanCache* cache = collection->infoCache()->getPlanCache();
    ASSERT(cache);
    ASSERT_EQ(cache->get(*cq).state, PlanCache::CacheEntryState::kNotPresent);

    // Get planner params.
    QueryPlannerParams plannerParams;
    fillOutPlannerParams(&_opCtx, collection, cq.get(), &plannerParams);

    // Queued data stage will return a failure during the cached plan trial period.
    auto mockChild = stdx::make_unique<QueuedDataStage>(&_opCtx, &_ws);
    mockChild->pushBack(PlanStage::FAILURE);

    // High enough so that we shouldn't trigger a replan based on works.
    const size_t decisionWorks = 50;
    CachedPlanStage cachedPlanStage(
        &_opCtx, collection, &_ws, cq.get(), plannerParams, decisionWorks, mockChild.release());

    // This should succeed after triggering a replan.
    PlanYieldPolicy yieldPolicy(PlanExecutor::NO_YIELD,
                                _opCtx.getServiceContext()->getFastClockSource());
    ASSERT_OK(cachedPlanStage.pickBestPlan(&yieldPolicy));

    ASSERT_EQ(getNumResultsForStage(_ws, &cachedPlanStage, cq.get()), 2U);

    // Plan cache should still be empty, as we don't write to it when we replan a failed
    // query.
    ASSERT_EQ(cache->get(*cq).state, PlanCache::CacheEntryState::kNotPresent);
}
Ejemplo n.º 12
0
    void CachedPlanRunner::updateCache() {
        _updatedCache = true;

        Database* db = cc().database();
        verify(NULL != db);
        Collection* collection = db->getCollection(_canonicalQuery->ns());
        verify(NULL != collection);
        PlanCache* cache = collection->infoCache()->getPlanCache();

        std::auto_ptr<PlanCacheEntryFeedback> feedback(new PlanCacheEntryFeedback());
        // XXX: what else can we provide here?
        feedback->stats.reset(_exec->getStats());
        feedback->score = PlanRanker::scoreTree(feedback->stats.get());

        Status fbs = cache->feedback(*_canonicalQuery, feedback.release());

        if (!fbs.isOK()) {
            // XXX: what should happen here?
            warning() << "Failed to update cache with feedback: " << fbs.toString() << endl;
        }
    }
// static
Status PlanCacheListPlans::list(OperationContext* opCtx,
                                const PlanCache& planCache,
                                const std::string& ns,
                                const BSONObj& cmdObj,
                                BSONObjBuilder* bob) {
    auto statusWithCQ = canonicalize(opCtx, ns, cmdObj);
    if (!statusWithCQ.isOK()) {
        return statusWithCQ.getStatus();
    }

    if (!internalQueryCacheListPlansNewOutput.load())
        return listPlansOriginalFormat(std::move(statusWithCQ.getValue()), planCache, bob);

    unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
    auto entry = uassertStatusOK(planCache.getEntry(*cq));

    // internalQueryCacheDisableInactiveEntries is True and we should use the new output format.
    Explain::planCacheEntryToBSON(*entry, bob);
    return Status::OK();
}
Ejemplo n.º 14
0
    // static
    Status PlanCacheListPlans::list(const PlanCache& planCache, const std::string& ns,
                                    const BSONObj& cmdObj, BSONObjBuilder* bob) {
        CanonicalQuery* cqRaw;
        Status status = canonicalize(ns, cmdObj, &cqRaw);
        if (!status.isOK()) {
            return status;
        }

        scoped_ptr<CanonicalQuery> cq(cqRaw);
        CachedSolution* crRaw;
        Status result = planCache.get(*cq, &crRaw);
        if (!result.isOK()) {
            return result;
        }
        scoped_ptr<CachedSolution> cr(crRaw);

        BSONArrayBuilder plansBuilder(bob->subarrayStart("plans"));
        size_t numPlans = cr->plannerData.size();
        for (size_t i = 0; i < numPlans; ++i) {
            BSONObjBuilder planBob(plansBuilder.subobjStart());

            // Create plan details field.
            // Currently, simple string representationg of
            // SolutionCacheData. Need to revisit format when we
            // need to parse user-provided plan details for planCacheAddPlan.
            SolutionCacheData* scd = cr->plannerData[i];
            BSONObjBuilder detailsBob(planBob.subobjStart("details"));
            detailsBob.append("solution", scd->toString());
            detailsBob.doneFast();

            // XXX: Fix these field values once we have fleshed out cache entries.
            //      reason should contain initial plan stats and score from ranking process.
            //      feedback should contain execution stats from running the query to completion.
            planBob.append("reason", BSONObj());
            planBob.append("feedback", BSONObj());
            planBob.append("hint", scd->adminHintApplied);
        }
        plansBuilder.doneFast();

        return Status::OK();
    }
Ejemplo n.º 15
0
    // static
    Status PlanCacheListQueryShapes::list(const PlanCache& planCache, BSONObjBuilder* bob) {
        invariant(bob);

        // Fetch all cached solutions from plan cache.
        vector<PlanCacheEntry*> solutions = planCache.getAllEntries();

        BSONArrayBuilder arrayBuilder(bob->subarrayStart("shapes"));
        for (vector<PlanCacheEntry*>::const_iterator i = solutions.begin(); i != solutions.end(); i++) {
            PlanCacheEntry* entry = *i;
            invariant(entry);

            BSONObjBuilder shapeBuilder(arrayBuilder.subobjStart());
            shapeBuilder.append("query", entry->query);
            shapeBuilder.append("sort", entry->sort);
            shapeBuilder.append("projection", entry->projection);
            shapeBuilder.doneFast();

            // Release resources for cached solution after extracting query shape.
            delete entry;
        }
        arrayBuilder.doneFast();

        return Status::OK();
    }
// static
Status PlanCacheListQueryShapes::list(const PlanCache& planCache, BSONObjBuilder* bob) {
    invariant(bob);

    // Fetch all cached solutions from plan cache.
    auto entries = planCache.getAllEntries();

    BSONArrayBuilder arrayBuilder(bob->subarrayStart("shapes"));
    for (auto&& entry : entries) {
        invariant(entry);

        BSONObjBuilder shapeBuilder(arrayBuilder.subobjStart());
        shapeBuilder.append("query", entry->query);
        shapeBuilder.append("sort", entry->sort);
        shapeBuilder.append("projection", entry->projection);
        if (!entry->collation.isEmpty()) {
            shapeBuilder.append("collation", entry->collation);
        }
        shapeBuilder.append("queryHash", unsignedIntToFixedLengthHex(entry->queryHash));
        shapeBuilder.doneFast();
    }
    arrayBuilder.doneFast();

    return Status::OK();
}
Ejemplo n.º 17
0
    /**
     * For a given query, get a runner.  The runner could be a SingleSolutionRunner, a
     * CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
     */
    Status getRunner(CanonicalQuery* rawCanonicalQuery, Runner** out, size_t plannerOptions) {
        verify(rawCanonicalQuery);
        auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);

        // Try to look up a cached solution for the query.
        // TODO: Can the cache have negative data about a solution?
        PlanCache* localCache = PlanCache::get(canonicalQuery->ns());
        if (NULL != localCache) {
            CachedSolution* cs = localCache->get(*canonicalQuery);
            if (NULL != cs) {
                // We have a cached solution.  Hand the canonical query and cached solution off to
                // the cached plan runner, which takes ownership of both.
                WorkingSet* ws;
                PlanStage* root;
                verify(StageBuilder::build(*cs->solution, &root, &ws));
                *out = new CachedPlanRunner(canonicalQuery.release(), cs, root, ws);
                return Status::OK();
            }
        }

        // No entry in cache for the query.  We have to solve the query ourself.

        // Get the indices that we could possibly use.
        Database* db = cc().database();
        verify( db );
        Collection* collection = db->getCollection( canonicalQuery->ns() );

        // This can happen as we're called by internal clients as well.
        if (NULL == collection) {
            const string& ns = canonicalQuery->ns();
            *out = new EOFRunner(canonicalQuery.release(), ns);
            return Status::OK();
        }

        // If we have an _id index we can use the idhack runner.
        if (canUseIDHack(*canonicalQuery) && collection->getIndexCatalog()->findIdIndex()) {
            *out = new IDHackRunner(collection, canonicalQuery.release());
            return Status::OK();
        }

        // If it's not NULL, we may have indices.  Access the catalog and fill out IndexEntry(s)
        QueryPlannerParams plannerParams;
        for (int i = 0; i < collection->getIndexCatalog()->numIndexesReady(); ++i) {
            IndexDescriptor* desc = collection->getIndexCatalog()->getDescriptor( i );
            plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
                                                       desc->isMultikey(),
                                                       desc->isSparse(),
                                                       desc->indexName()));
        }

        // Tailable: If the query requests tailable the collection must be capped.
        if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
            if (!collection->isCapped()) {
                return Status(ErrorCodes::BadValue,
                              "tailable cursor requested on non capped collection");
            }

            // If a sort is specified it must be equal to expectedSort.
            const BSONObj expectedSort = BSON("$natural" << 1);
            const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
            if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
                return Status(ErrorCodes::BadValue,
                              "invalid sort specified for tailable cursor: "
                              + actualSort.toString());
            }
        }

        // Process the planning options.
        plannerParams.options = plannerOptions;
        if (storageGlobalParams.noTableScan) {
            const string& ns = canonicalQuery->ns();
            // There are certain cases where we ignore this restriction:
            bool ignore = canonicalQuery->getQueryObj().isEmpty()
                          || (string::npos != ns.find(".system."))
                          || (0 == ns.find("local."));
            if (!ignore) {
                plannerParams.options |= QueryPlannerParams::NO_TABLE_SCAN;
            }
        }

        if (!(plannerParams.options & QueryPlannerParams::NO_TABLE_SCAN)) {
            plannerParams.options |= QueryPlannerParams::INCLUDE_COLLSCAN;
        }

        // If the caller wants a shard filter, make sure we're actually sharded.
        if (plannerParams.options & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
            CollectionMetadataPtr collMetadata = shardingState.getCollectionMetadata(canonicalQuery->ns());
            if (collMetadata) {
                plannerParams.shardKey = collMetadata->getKeyPattern();
            }
            else {
                // If there's no metadata don't bother w/the shard filter since we won't know what
                // the key pattern is anyway...
                plannerParams.options &= ~QueryPlannerParams::INCLUDE_SHARD_FILTER;
            }
        }

        vector<QuerySolution*> solutions;
        QueryPlanner::plan(*canonicalQuery, plannerParams, &solutions);

        /*
        for (size_t i = 0; i < solutions.size(); ++i) {
            QLOG() << "solution " << i << " is " << solutions[i]->toString() << endl;
        }
        */

        // We cannot figure out how to answer the query.  Should this ever happen?
        if (0 == solutions.size()) {
            return Status(ErrorCodes::BadValue, "No query solutions");
        }

        if (1 == solutions.size()) {
            // Only one possible plan.  Run it.  Build the stages from the solution.
            WorkingSet* ws;
            PlanStage* root;
            verify(StageBuilder::build(*solutions[0], &root, &ws));

            // And, run the plan.
            *out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
            return Status::OK();
        }
        else {
            // Many solutions.  Let the MultiPlanRunner pick the best, update the cache, and so on.
            auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
            for (size_t i = 0; i < solutions.size(); ++i) {
                WorkingSet* ws;
                PlanStage* root;
                verify(StageBuilder::build(*solutions[i], &root, &ws));
                // Takes ownership of all arguments.
                mpr->addPlan(solutions[i], root, ws);
            }
            *out = mpr.release();
            return Status::OK();
        }
    }
Ejemplo n.º 18
0
Status CachedPlanStage::replan(PlanYieldPolicy* yieldPolicy, bool shouldCache) {
    // We're going to start over with a new plan. Clear out info from our old plan.
    _results.clear();
    _ws->clear();
    _children.clear();

    // Use the query planning module to plan the whole query.
    std::vector<QuerySolution*> rawSolutions;
    Status status = QueryPlanner::plan(*_canonicalQuery, _plannerParams, &rawSolutions);
    if (!status.isOK()) {
        return Status(ErrorCodes::BadValue,
                      str::stream() << "error processing query: " << _canonicalQuery->toString()
                                    << " planner returned error: " << status.reason());
    }

    OwnedPointerVector<QuerySolution> solutions(rawSolutions);

    // We cannot figure out how to answer the query.  Perhaps it requires an index
    // we do not have?
    if (0 == solutions.size()) {
        return Status(ErrorCodes::BadValue,
                      str::stream() << "error processing query: " << _canonicalQuery->toString()
                                    << " No query solutions");
    }

    if (1 == solutions.size()) {
        // If there's only one solution, it won't get cached. Make sure to evict the existing
        // cache entry if requested by the caller.
        if (shouldCache) {
            PlanCache* cache = _collection->infoCache()->getPlanCache();
            cache->remove(*_canonicalQuery);
        }

        PlanStage* newRoot;
        // Only one possible plan. Build the stages from the solution.
        verify(StageBuilder::build(
            getOpCtx(), _collection, *_canonicalQuery, *solutions[0], _ws, &newRoot));
        _children.emplace_back(newRoot);
        _replannedQs.reset(solutions.popAndReleaseBack());

        LOG(1)
            << "Replanning of query resulted in single query solution, which will not be cached. "
            << _canonicalQuery->toStringShort()
            << " plan summary after replan: " << Explain::getPlanSummary(child().get())
            << " previous cache entry evicted: " << (shouldCache ? "yes" : "no");
        return Status::OK();
    }

    // Many solutions. Create a MultiPlanStage to pick the best, update the cache,
    // and so on. The working set will be shared by all candidate plans.
    auto cachingMode = shouldCache ? MultiPlanStage::CachingMode::AlwaysCache
                                   : MultiPlanStage::CachingMode::NeverCache;
    _children.emplace_back(
        new MultiPlanStage(getOpCtx(), _collection, _canonicalQuery, cachingMode));
    MultiPlanStage* multiPlanStage = static_cast<MultiPlanStage*>(child().get());

    for (size_t ix = 0; ix < solutions.size(); ++ix) {
        if (solutions[ix]->cacheData.get()) {
            solutions[ix]->cacheData->indexFilterApplied = _plannerParams.indexFiltersApplied;
        }

        PlanStage* nextPlanRoot;
        verify(StageBuilder::build(
            getOpCtx(), _collection, *_canonicalQuery, *solutions[ix], _ws, &nextPlanRoot));

        // Takes ownership of 'solutions[ix]' and 'nextPlanRoot'.
        multiPlanStage->addPlan(solutions.releaseAt(ix), nextPlanRoot, _ws);
    }

    // Delegate to the MultiPlanStage's plan selection facility.
    Status pickBestPlanStatus = multiPlanStage->pickBestPlan(yieldPolicy);
    if (!pickBestPlanStatus.isOK()) {
        return pickBestPlanStatus;
    }

    LOG(1) << "Replanning " << _canonicalQuery->toStringShort()
           << " resulted in plan with summary: " << Explain::getPlanSummary(child().get())
           << ", which " << (shouldCache ? "has" : "has not") << " been written to the cache";
    return Status::OK();
}
Ejemplo n.º 19
0
    void run() {
        AutoGetCollectionForRead ctx(&_txn, nss.ns());
        Collection* collection = ctx.getCollection();
        ASSERT(collection);

        // Query can be answered by either index on "a" or index on "b".
        auto statusWithCQ = CanonicalQuery::canonicalize(nss, fromjson("{a: {$gte: 8}, b: 1}"));
        ASSERT_OK(statusWithCQ.getStatus());
        const std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());

        // We shouldn't have anything in the plan cache for this shape yet.
        PlanCache* cache = collection->infoCache()->getPlanCache();
        ASSERT(cache);
        CachedSolution* rawCachedSolution;
        ASSERT_NOT_OK(cache->get(*cq, &rawCachedSolution));

        // Get planner params.
        QueryPlannerParams plannerParams;
        fillOutPlannerParams(&_txn, collection, cq.get(), &plannerParams);

        // Set up queued data stage to take a long time before returning EOF. Should be long
        // enough to trigger a replan.
        const size_t decisionWorks = 10;
        const size_t mockWorks =
            1U + static_cast<size_t>(internalQueryCacheEvictionRatio * decisionWorks);
        auto mockChild = stdx::make_unique<QueuedDataStage>(&_txn, &_ws);
        for (size_t i = 0; i < mockWorks; i++) {
            mockChild->pushBack(PlanStage::NEED_TIME);
        }

        CachedPlanStage cachedPlanStage(
            &_txn, collection, &_ws, cq.get(), plannerParams, decisionWorks, mockChild.release());

        // This should succeed after triggering a replan.
        PlanYieldPolicy yieldPolicy(nullptr, PlanExecutor::YIELD_MANUAL);
        ASSERT_OK(cachedPlanStage.pickBestPlan(&yieldPolicy));

        // Make sure that we get 2 legit results back.
        size_t numResults = 0;
        PlanStage::StageState state = PlanStage::NEED_TIME;
        while (state != PlanStage::IS_EOF) {
            WorkingSetID id = WorkingSet::INVALID_ID;
            state = cachedPlanStage.work(&id);

            ASSERT_NE(state, PlanStage::FAILURE);
            ASSERT_NE(state, PlanStage::DEAD);

            if (state == PlanStage::ADVANCED) {
                WorkingSetMember* member = _ws.get(id);
                ASSERT(cq->root()->matchesBSON(member->obj.value()));
                numResults++;
            }
        }

        ASSERT_EQ(numResults, 2U);

        // This time we expect to find something in the plan cache. Replans after hitting the
        // works threshold result in a cache entry.
        ASSERT_OK(cache->get(*cq, &rawCachedSolution));
        const std::unique_ptr<CachedSolution> cachedSolution(rawCachedSolution);
    }
Ejemplo n.º 20
0
    bool MultiPlanRunner::pickBestPlan(size_t* out, BSONObj* objOut) {
        static const int timesEachPlanIsWorked = 100;

        // Run each plan some number of times.
        for (int i = 0; i < timesEachPlanIsWorked; ++i) {
            bool moreToDo = workAllPlans(objOut);
            if (!moreToDo) { break; }
        }

        if (_failure || _killed) { return false; }

        // After picking best plan, ranking will own plan stats from
        // candidate solutions (winner and losers).
        std::auto_ptr<PlanRankingDecision> ranking(new PlanRankingDecision);
        size_t bestChild = PlanRanker::pickBestPlan(_candidates, ranking.get());

        // Copy candidate order. We will need this to sort candidate stats for explain
        // after transferring ownership of 'ranking' to plan cache.
        std::vector<size_t> candidateOrder = ranking->candidateOrder;

        // Run the best plan.  Store it.
        _bestPlan.reset(new PlanExecutor(_candidates[bestChild].ws,
                                         _candidates[bestChild].root));
        _bestPlan->setYieldPolicy(_policy);
        _alreadyProduced = _candidates[bestChild].results;
        _bestSolution.reset(_candidates[bestChild].solution);

        QLOG() << "Winning solution:\n" << _bestSolution->toString() << endl;

        size_t backupChild = bestChild;
        if (_bestSolution->hasBlockingStage && (0 == _alreadyProduced.size())) {
            QLOG() << "Winner has blocking stage, looking for backup plan...\n";
            for (size_t i = 0; i < _candidates.size(); ++i) {
                if (!_candidates[i].solution->hasBlockingStage) {
                    QLOG() << "Candidate " << i << " is backup child\n";
                    backupChild = i;
                    _backupSolution = _candidates[i].solution;
                    _backupAlreadyProduced = _candidates[i].results;
                    _backupPlan = new PlanExecutor(_candidates[i].ws, _candidates[i].root);
                    _backupPlan->setYieldPolicy(_policy);
                    break;
                }
            }
        }

        // Store the choice we just made in the cache. We do
        // not cache the query if:
        //   1) The query is of a type that is not safe to cache, or
        //   2) the winning plan did not actually produce any results,
        //   without hitting EOF. In this case, we have no information to
        //   suggest that this plan is good.
        const PlanStageStats* bestStats = ranking->stats.vector()[0];
        if (PlanCache::shouldCacheQuery(*_query)
            && (!_alreadyProduced.empty() || bestStats->common.isEOF)) {
            Database* db = cc().database();
            verify(NULL != db);
            Collection* collection = db->getCollection(_query->ns());
            verify(NULL != collection);
            PlanCache* cache = collection->infoCache()->getPlanCache();
            // Create list of candidate solutions for the cache with
            // the best solution at the front.
            std::vector<QuerySolution*> solutions;

            // Generate solutions and ranking decisions sorted by score.
            for (size_t orderingIndex = 0;
                 orderingIndex < candidateOrder.size(); ++orderingIndex) {
                // index into candidates/ranking
                size_t i = candidateOrder[orderingIndex];
                solutions.push_back(_candidates[i].solution);
            }

            // Check solution cache data. Do not add to cache if
            // we have any invalid SolutionCacheData data.
            // XXX: One known example is 2D queries
            bool validSolutions = true;
            for (size_t i = 0; i < solutions.size(); ++i) {
                if (NULL == solutions[i]->cacheData.get()) {
                    QLOG() << "Not caching query because this solution has no cache data: "
                           << solutions[i]->toString();
                    validSolutions = false;
                    break;
                }
            }

            if (validSolutions) {
                cache->add(*_query, solutions, ranking.release());
            }
        }

        // Clear out the candidate plans, leaving only stats as we're all done w/them.
        // Traverse candidate plans in order or score
        for (size_t orderingIndex = 0;
             orderingIndex < candidateOrder.size(); ++orderingIndex) {
            // index into candidates/ranking
            size_t i = candidateOrder[orderingIndex];

            if (i == bestChild) { continue; }
            if (i == backupChild) { continue; }

            delete _candidates[i].solution;

            // Remember the stats for the candidate plan because we always show it on an
            // explain. (The {verbose:false} in explain() is client-side trick; we always
            // generate a "verbose" explain.)
            PlanStageStats* stats = _candidates[i].root->getStats();
            if (stats) {
                _candidateStats.push_back(stats);
            }
            delete _candidates[i].root;

            // ws must die after the root.
            delete _candidates[i].ws;
        }

        _candidates.clear();
        if (NULL != out) { *out = bestChild; }
        return true;
    }
Ejemplo n.º 21
0
    /**
     * For a given query, get a runner.  The runner could be a SingleSolutionRunner, a
     * CachedQueryRunner, or a MultiPlanRunner, depending on the cache/query solver/etc.
     */
    Status getRunner(CanonicalQuery* rawCanonicalQuery, Runner** out) {
        verify(rawCanonicalQuery);
        auto_ptr<CanonicalQuery> canonicalQuery(rawCanonicalQuery);

        // Try to look up a cached solution for the query.
        // TODO: Can the cache have negative data about a solution?
        PlanCache* localCache = PlanCache::get(canonicalQuery->ns());
        if (NULL != localCache) {
            CachedSolution* cs = localCache->get(*canonicalQuery);
            if (NULL != cs) {
                // We have a cached solution.  Hand the canonical query and cached solution off to
                // the cached plan runner, which takes ownership of both.
                WorkingSet* ws;
                PlanStage* root;
                verify(StageBuilder::build(*cs->solution, &root, &ws));
                *out = new CachedPlanRunner(canonicalQuery.release(), cs, root, ws);
                return Status::OK();
            }
        }

        // No entry in cache for the query.  We have to solve the query ourself.

        // Get the indices that we could possibly use.
        NamespaceDetails* nsd = nsdetails(canonicalQuery->ns().c_str());

        // If this is NULL, there is no data but the query is valid.  You're allowed to query for
        // data on an empty collection and it's not an error.  There just isn't any data...
        if (NULL == nsd) {
            const std::string& ns = canonicalQuery->ns();
            *out = new EOFRunner(canonicalQuery.release(), ns);
            return Status::OK();
        }

        // Tailable: If the query requests tailable the collection must be capped.
        if (canonicalQuery->getParsed().hasOption(QueryOption_CursorTailable)) {
            if (!nsd->isCapped()) {
                return Status(ErrorCodes::BadValue,
                              "tailable cursor requested on non capped collection");
            }

            // If a sort is specified it must be equal to expectedSort.
            const BSONObj expectedSort = BSON("$natural" << 1);
            const BSONObj& actualSort = canonicalQuery->getParsed().getSort();
            if (!actualSort.isEmpty() && !(actualSort == expectedSort)) {
                return Status(ErrorCodes::BadValue,
                              "invalid sort specified for tailable cursor: "
                              + actualSort.toString());
            }
        }

        // If it's not NULL, we may have indices.
        vector<IndexEntry> indices;
        for (int i = 0; i < nsd->getCompletedIndexCount(); ++i) {
            auto_ptr<IndexDescriptor> desc(CatalogHack::getDescriptor(nsd, i));
            indices.push_back(IndexEntry(desc->keyPattern(), desc->isMultikey(), desc->isSparse(), desc->indexName()));
        }

        vector<QuerySolution*> solutions;
        size_t options = QueryPlanner::DEFAULT;
        if (storageGlobalParams.noTableScan) {
            const string& ns = canonicalQuery->ns();
            // There are certain cases where we ignore this restriction:
            bool ignore = canonicalQuery->getQueryObj().isEmpty()
                          || (string::npos != ns.find(".system."))
                          || (0 == ns.find("local."));
            if (!ignore) {
                options |= QueryPlanner::NO_TABLE_SCAN;
            }
        }
        else {
            options |= QueryPlanner::INCLUDE_COLLSCAN;
        }
        QueryPlanner::plan(*canonicalQuery, indices, options, &solutions);

        /*
        for (size_t i = 0; i < solutions.size(); ++i) {
            QLOG() << "solution " << i << " is " << solutions[i]->toString() << endl;
        }
        */

        // We cannot figure out how to answer the query.  Should this ever happen?
        if (0 == solutions.size()) {
            return Status(ErrorCodes::BadValue, "Can't create a plan for the canonical query " +
                                                 canonicalQuery->toString());
        }

        if (1 == solutions.size()) {
            // Only one possible plan.  Run it.  Build the stages from the solution.
            WorkingSet* ws;
            PlanStage* root;
            verify(StageBuilder::build(*solutions[0], &root, &ws));

            // And, run the plan.
            *out = new SingleSolutionRunner(canonicalQuery.release(), solutions[0], root, ws);
            return Status::OK();
        }
        else {
            // Many solutions.  Let the MultiPlanRunner pick the best, update the cache, and so on.
            auto_ptr<MultiPlanRunner> mpr(new MultiPlanRunner(canonicalQuery.release()));
            for (size_t i = 0; i < solutions.size(); ++i) {
                WorkingSet* ws;
                PlanStage* root;
                verify(StageBuilder::build(*solutions[i], &root, &ws));
                // Takes ownership of all arguments.
                mpr->addPlan(solutions[i], root, ws);
            }
            *out = mpr.release();
            return Status::OK();
        }
    }
Ejemplo n.º 22
0
    // static
    Status PlanCacheListPlans::list(OperationContext* txn,
                                    const PlanCache& planCache,
                                    const std::string& ns,
                                    const BSONObj& cmdObj,
                                    BSONObjBuilder* bob) {
        CanonicalQuery* cqRaw;
        Status status = canonicalize(txn, ns, cmdObj, &cqRaw);
        if (!status.isOK()) {
            return status;
        }

        scoped_ptr<CanonicalQuery> cq(cqRaw);

        if (!planCache.contains(*cq)) {
            // Return empty plans in results if query shape does not
            // exist in plan cache.
            BSONArrayBuilder plansBuilder(bob->subarrayStart("plans"));
            plansBuilder.doneFast();
            return Status::OK();
        }

        PlanCacheEntry* entryRaw;
        Status result = planCache.getEntry(*cq, &entryRaw);
        if (!result.isOK()) {
            return result;
        }
        scoped_ptr<PlanCacheEntry> entry(entryRaw);

        BSONArrayBuilder plansBuilder(bob->subarrayStart("plans"));
        size_t numPlans = entry->plannerData.size();
        invariant(numPlans == entry->decision->stats.size());
        invariant(numPlans == entry->decision->scores.size());
        for (size_t i = 0; i < numPlans; ++i) {
            BSONObjBuilder planBob(plansBuilder.subobjStart());

            // Create plan details field.
            // Currently, simple string representationg of
            // SolutionCacheData. Need to revisit format when we
            // need to parse user-provided plan details for planCacheAddPlan.
            SolutionCacheData* scd = entry->plannerData[i];
            BSONObjBuilder detailsBob(planBob.subobjStart("details"));
            detailsBob.append("solution", scd->toString());
            detailsBob.doneFast();

            // reason is comprised of score and initial stats provided by
            // multi plan runner.
            BSONObjBuilder reasonBob(planBob.subobjStart("reason"));
            reasonBob.append("score", entry->decision->scores[i]);
            BSONObjBuilder statsBob(reasonBob.subobjStart("stats"));
            PlanStageStats* stats = entry->decision->stats.vector()[i];
            if (stats) {
                Explain::statsToBSON(*stats, &statsBob);
            }
            statsBob.doneFast();
            reasonBob.doneFast();

            // BSON object for 'feedback' field is created from query executions
            // and shows number of executions since this cached solution was
            // created as well as score data (average and standard deviation).
            BSONObjBuilder feedbackBob(planBob.subobjStart("feedback"));
            if (i == 0U) {
                feedbackBob.append("nfeedback", int(entry->feedback.size()));
                feedbackBob.append("averageScore", entry->averageScore.get_value_or(0));
                feedbackBob.append("stdDevScore",entry->stddevScore.get_value_or(0));
                BSONArrayBuilder scoresBob(feedbackBob.subarrayStart("scores"));
                for (size_t i = 0; i < entry->feedback.size(); ++i) {
                    BSONObjBuilder scoreBob(scoresBob.subobjStart());
                    scoreBob.append("score", entry->feedback[i]->score);
                }
                scoresBob.doneFast();
            }
            feedbackBob.doneFast();

            planBob.append("filterSet", scd->indexFilterApplied);
        }
        plansBuilder.doneFast();

        return Status::OK();
    }
Ejemplo n.º 23
0
    Status CachedPlanStage::replan(PlanYieldPolicy* yieldPolicy, bool shouldCache) {
        // We're going to start over with a new plan. No need for only old buffered results.
        _results.clear();

        // Clear out the working set. We'll start with a fresh working set.
        _ws->clear();

        // Use the query planning module to plan the whole query.
        std::vector<QuerySolution*> rawSolutions;
        Status status = QueryPlanner::plan(*_canonicalQuery, _plannerParams, &rawSolutions);
        if (!status.isOK()) {
            return Status(ErrorCodes::BadValue,
                          str::stream()
                          << "error processing query: " << _canonicalQuery->toString()
                          << " planner returned error: " << status.reason());
        }

        OwnedPointerVector<QuerySolution> solutions(rawSolutions);

        // We cannot figure out how to answer the query.  Perhaps it requires an index
        // we do not have?
        if (0 == solutions.size()) {
            return Status(ErrorCodes::BadValue,
                          str::stream()
                          << "error processing query: "
                          << _canonicalQuery->toString()
                          << " No query solutions");
        }

        if (1 == solutions.size()) {
            // If there's only one solution, it won't get cached. Make sure to evict the existing
            // cache entry if requested by the caller.
            if (shouldCache) {
                PlanCache* cache = _collection->infoCache()->getPlanCache();
                cache->remove(*_canonicalQuery);
            }

            PlanStage* newRoot;
            // Only one possible plan. Build the stages from the solution.
            verify(StageBuilder::build(_txn, _collection, *solutions[0], _ws, &newRoot));
            _root.reset(newRoot);
            _replannedQs.reset(solutions.popAndReleaseBack());
            return Status::OK();
        }

        // Many solutions. Create a MultiPlanStage to pick the best, update the cache,
        // and so on. The working set will be shared by all candidate plans.
        _root.reset(new MultiPlanStage(_txn, _collection, _canonicalQuery, shouldCache));
        MultiPlanStage* multiPlanStage = static_cast<MultiPlanStage*>(_root.get());

        for (size_t ix = 0; ix < solutions.size(); ++ix) {
            if (solutions[ix]->cacheData.get()) {
                solutions[ix]->cacheData->indexFilterApplied = _plannerParams.indexFiltersApplied;
            }

            PlanStage* nextPlanRoot;
            verify(StageBuilder::build(_txn, _collection, *solutions[ix], _ws, &nextPlanRoot));

            // Takes ownership of 'solutions[ix]' and 'nextPlanRoot'.
            multiPlanStage->addPlan(solutions.releaseAt(ix), nextPlanRoot, _ws);
        }

        // Delegate to the MultiPlanStage's plan selection facility.
        return multiPlanStage->pickBestPlan(yieldPolicy);
    }
Ejemplo n.º 24
0
    Runner::RunnerState MultiPlanRunner::getNext(BSONObj* objOut, DiskLoc* dlOut) {
        if (_killed) { return Runner::RUNNER_DEAD; }
        if (_failure) { return Runner::RUNNER_ERROR; }

        // If we haven't picked the best plan yet...
        if (NULL == _bestPlan) {
            if (!pickBestPlan(NULL, objOut)) {
                verify(_failure || _killed);
                if (_killed) { return Runner::RUNNER_DEAD; }
                if (_failure) { return Runner::RUNNER_ERROR; }
            }
        }

        // Look for an already produced result that provides the data the caller wants.
        while (!_alreadyProduced.empty()) {
            WorkingSetID id = _alreadyProduced.front();
            _alreadyProduced.pop_front();

            WorkingSetMember* member = _bestPlan->getWorkingSet()->get(id);

            // Note that this copies code from PlanExecutor.
            if (NULL != objOut) {
                if (WorkingSetMember::LOC_AND_IDX == member->state) {
                    if (1 != member->keyData.size()) {
                        _bestPlan->getWorkingSet()->free(id);
                        // If the caller needs the key data and the WSM doesn't have it, drop the
                        // result and carry on.
                        continue;
                    }
                    *objOut = member->keyData[0].keyData;
                }
                else if (member->hasObj()) {
                    *objOut = member->obj;
                }
                else {
                    // If the caller needs an object and the WSM doesn't have it, drop and
                    // try the next result.
                    _bestPlan->getWorkingSet()->free(id);
                    continue;
                }
            }

            if (NULL != dlOut) {
                if (member->hasLoc()) {
                    *dlOut = member->loc;
                }
                else {
                    // If the caller needs a DiskLoc and the WSM doesn't have it, drop and carry on.
                    _bestPlan->getWorkingSet()->free(id);
                    continue;
                }
            }

            // If we're here, the caller has all the data needed and we've set the out
            // parameters.  Remove the result from the WorkingSet.
            _bestPlan->getWorkingSet()->free(id);
            return Runner::RUNNER_ADVANCED;
        }

        RunnerState state = _bestPlan->getNext(objOut, dlOut);

        if (Runner::RUNNER_ERROR == state && (NULL != _backupSolution)) {
            QLOG() << "Best plan errored out switching to backup\n";
            // Uncache the bad solution if we fall back
            // on the backup solution.
            //
            // XXX: Instead of uncaching we should find a way for the
            // cached plan runner to fall back on a different solution
            // if the best solution fails. Alternatively we could try to
            // defer cache insertion to be after the first produced result.
            Database* db = cc().database();
            verify(NULL != db);
            Collection* collection = db->getCollection(_query->ns());
            verify(NULL != collection);
            PlanCache* cache = collection->infoCache()->getPlanCache();
            cache->remove(*_query);

            _bestPlan.reset(_backupPlan);
            _backupPlan = NULL;
            _bestSolution.reset(_backupSolution);
            _backupSolution = NULL;
            _alreadyProduced = _backupAlreadyProduced;
            return getNext(objOut, dlOut);
        }

        if (NULL != _backupSolution && Runner::RUNNER_ADVANCED == state) {
            QLOG() << "Best plan had a blocking sort, became unblocked, deleting backup plan\n";
            delete _backupSolution;
            delete _backupPlan;
            _backupSolution = NULL;
            _backupPlan = NULL;
            // TODO: free from WS?
            _backupAlreadyProduced.clear();
        }

        return state;
    }
Ejemplo n.º 25
0
    bool MultiPlanRunner::pickBestPlan(size_t* out) {
        static const int timesEachPlanIsWorked = 100;

        // Run each plan some number of times.
        for (int i = 0; i < timesEachPlanIsWorked; ++i) {
            bool moreToDo = workAllPlans();
            if (!moreToDo) { break; }
        }

        if (_failure || _killed) { return false; }

        auto_ptr<PlanRankingDecision> ranking(new PlanRankingDecision());
        size_t bestChild = PlanRanker::pickBestPlan(_candidates, ranking.get());

        // Run the best plan.  Store it.
        _bestPlan.reset(new PlanExecutor(_candidates[bestChild].ws,
                                         _candidates[bestChild].root));
        _bestPlan->setYieldPolicy(_policy);
        _alreadyProduced = _candidates[bestChild].results;
        _bestSolution.reset(_candidates[bestChild].solution);

        QLOG() << "Winning solution:\n" << _bestSolution->toString() << endl;

        size_t backupChild = bestChild;
        if (_bestSolution->hasSortStage && (0 == _alreadyProduced.size())) {
            QLOG() << "Winner has blocked sort, looking for backup plan...\n";
            for (size_t i = 0; i < _candidates.size(); ++i) {
                if (!_candidates[i].solution->hasSortStage) {
                    QLOG() << "Candidate " << i << " is backup child\n";
                    backupChild = i;
                    _backupSolution = _candidates[i].solution;
                    _backupAlreadyProduced = _candidates[i].results;
                    _backupPlan = new PlanExecutor(_candidates[i].ws, _candidates[i].root);
                    _backupPlan->setYieldPolicy(_policy);
                    break;
                }
            }
        }

        // Store the choice we just made in the cache.
        if (PlanCache::shouldCacheQuery(*_query)) {
            Database* db = cc().database();
            verify(NULL != db);
            Collection* collection = db->getCollection(_query->ns());
            verify(NULL != collection);
            PlanCache* cache = collection->infoCache()->getPlanCache();
            // Create list of candidate solutions for the cache with
            // the best solution at the front.
            std::vector<QuerySolution*> solutions;
            solutions.push_back(_bestSolution.get());
            for (size_t i = 0; i < _candidates.size(); ++i) {
                if (i == bestChild) { continue; }
                solutions.push_back(_candidates[i].solution);
            }

            // Check solution cache data. Do not add to cache if
            // we have any invalid SolutionCacheData data.
            // XXX: One known example is 2D queries
            bool validSolutions = true;
            for (size_t i = 0; i < solutions.size(); ++i) {
                if (NULL == solutions[i]->cacheData.get()) {
                    QLOG() << "Not caching query because this solution has no cache data: "
                           << solutions[i]->toString();
                    validSolutions = false;
                    break;
                }
            }

            if (validSolutions) {
                cache->add(*_query, solutions, ranking.release());
            }
        }

        // Clear out the candidate plans, leaving only stats as we're all done w/them.
        for (size_t i = 0; i < _candidates.size(); ++i) {
            if (i == bestChild) { continue; }
            if (i == backupChild) { continue; }

            delete _candidates[i].solution;

            // Remember the stats for the candidate plan because we always show it on an
            // explain. (The {verbose:false} in explain() is client-side trick; we always
            // generate a "verbose" explain.)
            PlanStageStats* stats = _candidates[i].root->getStats();
            if (stats) {
                _candidateStats.push_back(stats);
            }
            delete _candidates[i].root;

            // ws must die after the root.
            delete _candidates[i].ws;
        }

        _candidates.clear();
        if (NULL != out) { *out = bestChild; }
        return true;
    }