StatusWith<CursorId> ClusterFind::runQuery(OperationContext* txn,
const CanonicalQuery& query,
const ReadPreferenceSetting& readPref,
std::vector<BSONObj>* results) {
invariant(results);
// Projection on the reserved sort key field is illegal in mongos.
if (query.getParsed().getProj().hasField(ClusterClientCursorParams::kSortKeyField)) {
return {ErrorCodes::BadValue,
str::stream() << "Projection contains illegal field '"
<< ClusterClientCursorParams::kSortKeyField
<< "': " << query.getParsed().getProj()};
}
auto dbConfig = grid.catalogCache()->getDatabase(txn, query.nss().db().toString());
if (dbConfig.getStatus() == ErrorCodes::NamespaceNotFound) {
// If the database doesn't exist, we successfully return an empty result set without
// creating a cursor.
return CursorId(0);
} else if (!dbConfig.isOK()) {
return dbConfig.getStatus();
}
std::shared_ptr<ChunkManager> chunkManager;
std::shared_ptr<Shard> primary;
dbConfig.getValue()->getChunkManagerOrPrimary(txn, query.nss().ns(), chunkManager, primary);
// Re-target and re-send the initial find command to the shards until we have established the
// shard version.
for (size_t retries = 1; retries <= kMaxStaleConfigRetries; ++retries) {
auto cursorId = runQueryWithoutRetrying(
txn, query, readPref, chunkManager.get(), std::move(primary), results);
if (cursorId.isOK()) {
return cursorId;
}
auto status = std::move(cursorId.getStatus());
if (status != ErrorCodes::SendStaleConfig && status != ErrorCodes::RecvStaleConfig &&
status != ErrorCodes::HostUnreachable) {
// Errors other than receiving a stale config message from mongoD or an unreachable host
// are fatal to the operation.
return status;
}
LOG(1) << "Received error status for query " << query.toStringShort() << " on attempt "
<< retries << " of " << kMaxStaleConfigRetries << ": " << status;
chunkManager = dbConfig.getValue()->getChunkManagerIfExists(txn, query.nss().ns(), true);
if (!chunkManager) {
dbConfig.getValue()->getChunkManagerOrPrimary(
txn, query.nss().ns(), chunkManager, primary);
}
}
return {ErrorCodes::StaleShardVersion,
str::stream() << "Retried " << kMaxStaleConfigRetries
<< " times without establishing shard version on a reachable host."};
}
StatusWith<CursorId> ClusterFind::runQuery(OperationContext* txn,
const CanonicalQuery& query,
const ReadPreferenceSetting& readPref,
std::vector<BSONObj>* results) {
invariant(results);
auto dbConfig = grid.catalogCache()->getDatabase(txn, query.nss().db().toString());
if (dbConfig.getStatus() == ErrorCodes::DatabaseNotFound) {
// If the database doesn't exist, we successfully return an empty result set without
// creating a cursor.
return CursorId(0);
} else if (!dbConfig.isOK()) {
return dbConfig.getStatus();
}
std::shared_ptr<ChunkManager> chunkManager;
std::shared_ptr<Shard> primary;
dbConfig.getValue()->getChunkManagerOrPrimary(query.nss().ns(), chunkManager, primary);
// Re-target and re-send the initial find command to the shards until we have established the
// shard version.
for (size_t retries = 1; retries <= kMaxStaleConfigRetries; ++retries) {
auto cursorId = runQueryWithoutRetrying(
txn, query, readPref, chunkManager.get(), std::move(primary), results);
if (cursorId.isOK()) {
return cursorId;
}
auto status = std::move(cursorId.getStatus());
if (status != ErrorCodes::RecvStaleConfig) {
// Errors other than receiving a stale config message from mongoD are fatal to the
// operation.
return status;
}
LOG(1) << "Received stale config for query " << query.toStringShort() << " on attempt "
<< retries << " of " << kMaxStaleConfigRetries << ": " << status.reason();
invariant(chunkManager);
chunkManager = chunkManager->reload(txn);
}
return {ErrorCodes::StaleShardVersion,
str::stream() << "Retried " << kMaxStaleConfigRetries
<< " times without establishing shard version."};
}
std::string newRunQuery(OperationContext* txn,
Message& m,
QueryMessage& q,
CurOp& curop,
Message &result,
bool fromDBDirectClient) {
// Validate the namespace.
const char *ns = q.ns;
uassert(16332, "can't have an empty ns", ns[0]);
const NamespaceString nsString(ns);
uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid());
// Set curop information.
curop.debug().ns = ns;
curop.debug().ntoreturn = q.ntoreturn;
curop.debug().query = q.query;
curop.setQuery(q.query);
// If the query is really a command, run it.
if (nsString.isCommand()) {
int nToReturn = q.ntoreturn;
uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn
<< ") for $cmd type ns - can only be 1 or -1",
nToReturn == 1 || nToReturn == -1);
curop.markCommand();
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder cmdResBuf;
if (!runCommands(txn, ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) {
uasserted(13530, "bad or malformed command request?");
}
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
const NamespaceString nss(q.ns);
// Parse the qm into a CanonicalQuery.
CanonicalQuery* cq;
Status canonStatus = CanonicalQuery::canonicalize(
q, &cq, WhereCallbackReal(txn, StringData(nss.db())));
if (!canonStatus.isOK()) {
uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString());
}
QLOG() << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a plan executor.
PlanExecutor* rawExec = NULL;
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
AutoGetCollectionForRead ctx(txn, nss);
const int dbProfilingLevel = (ctx.getDb() != NULL) ? ctx.getDb()->getProfilingLevel() :
serverGlobalParams.defaultProfile;
Collection* collection = ctx.getCollection();
// We'll now try to get the query executor that will execute this query for us. There
// are a few cases in which we know upfront which executor we should get and, therefore,
// we shortcut the selection process here.
//
// (a) If the query is over a collection that doesn't exist, we use an EOFStage.
//
// (b) if the query is a replication's initial sync one, we use a specifically designed
// stage that skips extents faster (see details in exec/oplogstart.h).
//
// Otherwise we go through the selection of which executor is most suited to the
// query + run-time context at hand.
Status status = Status::OK();
if (NULL != collection && pq.getOptions().oplogReplay) {
// Takes ownership of 'cq'.
status = getOplogStartHack(txn, collection, cq, &rawExec);
}
else {
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(pq.ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
// Takes ownership of 'cq'.
status = getExecutor(txn, collection, cq, PlanExecutor::YIELD_AUTO, &rawExec, options);
}
if (!status.isOK()) {
// NOTE: Do not access cq as getExecutor has deleted it.
uasserted(17007, "Unable to execute query: " + status.reason());
}
verify(NULL != rawExec);
auto_ptr<PlanExecutor> exec(rawExec);
// If it's actually an explain, do the explain and return rather than falling through
// to the normal query execution loop.
if (pq.isExplain()) {
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder explainBob;
Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob);
// Add the resulting object to the return buffer.
BSONObj explainObj = explainBob.obj();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
// Set query result fields.
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
bool slaveOK = pq.getOptions().slaveOk || pq.hasReadPref();
status = repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor(
txn,
NamespaceString(cq->ns()),
slaveOK);
uassertStatusOK(status);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult::Value));
// How many results have we obtained from the executor?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the PlanExecutor in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
BSONObj obj;
PlanExecutor::ExecState state;
// uint64_t numMisplacedDocs = 0;
// Get summary info about which plan the executor is using.
curop.debug().planSummary = Explain::getPlanSummary(exec.get());
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.getOptions().oplogReplay) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
QLOG() << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
QLOG() << " executor EOF=" << exec->isEOF() << endl;
saveClientCursor = !exec->isEOF();
}
break;
}
}
// If we cache the executor later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the executor later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the executor.
exec->deregisterExec();
// Caller expects exceptions thrown in certain cases.
if (PlanExecutor::EXEC_ERROR == state) {
scoped_ptr<PlanStageStats> stats(exec->getStats());
error() << "Plan executor error, stats: "
<< Explain::statsToBSON(*stats);
uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj));
}
// Why save a dead executor?
if (PlanExecutor::DEAD == state) {
saveClientCursor = false;
}
else if (pq.getOptions().tailable) {
// If we're tailing a capped collection, we don't bother saving the cursor if the
// collection is empty. Otherwise, the semantics of the tailable cursor is that the
// client will keep trying to read from it. So we'll keep it around.
if (collection && collection->numRecords(txn) != 0 && pq.getNumToReturn() != 1) {
saveClientCursor = true;
}
}
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
const logger::LogComponent queryLogComponent = logger::LogComponent::kQuery;
const logger::LogSeverity logLevelOne = logger::LogSeverity::Debug(1);
PlanSummaryStats summaryStats;
Explain::getSummaryStats(exec.get(), &summaryStats);
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
curop.debug().scanAndOrder = summaryStats.hasSortStage;
curop.debug().nscanned = summaryStats.totalKeysExamined;
curop.debug().nscannedObjects = summaryStats.totalDocsExamined;
curop.debug().idhack = summaryStats.isIdhack;
// Set debug information for consumption by the profiler.
if (dbProfilingLevel > 0 ||
curop.elapsedMillis() > serverGlobalParams.slowMS ||
logger::globalLogDomain()->shouldLog(queryLogComponent, logLevelOne)) {
// Get BSON stats.
scoped_ptr<PlanStageStats> execStats(exec->getStats());
BSONObjBuilder statsBob;
Explain::statsToBSON(*execStats, &statsBob);
curop.debug().execStats.set(statsBob.obj());
// Replace exec stats with plan summary if stats cannot fit into CachedBSONObj.
if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) {
BSONObjBuilder bob;
bob.append("summary", curop.debug().planSummary.toString());
curop.debug().execStats.set(bob.done());
}
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the executor until it's getMore'd.
exec->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(collection, exec.get(),
cq->getParsed().getOptions().toInt(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
if (fromDBDirectClient) {
cc->setUnownedRecoveryUnit(txn->recoveryUnit());
}
else if (state == PlanExecutor::IS_EOF && pq.getOptions().tailable) {
// Don't stash the RU for tailable cursors at EOF, let them get a new RU on their
// next getMore.
}
else {
// We stash away the RecoveryUnit in the ClientCursor. It's used for subsequent
// getMore requests. The calling OpCtx gets a fresh RecoveryUnit.
cc->setOwnedRecoveryUnit(txn->releaseRecoveryUnit());
StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine();
txn->setRecoveryUnit(storageEngine->newRecoveryUnit(txn));
}
QLOG() << "caching executor with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// ClientCursor takes ownership of executor. Release to make sure it's not deleted.
exec.release();
// TODO document
if (pq.getOptions().oplogReplay && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.getOptions().exhaust) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
else {
QLOG() << "Not caching executor but returning " << numResults << " results.\n";
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult::View qr = result.header().view2ptr();
qr.setCursorId(ccId);
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr.setResultFlagsToOk();
qr.msgdata().setOperation(opReply);
qr.setStartingFrom(0);
qr.setNReturned(numResults);
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
// static
Status QueryPlanner::planFromCache(const CanonicalQuery& query,
const QueryPlannerParams& params,
const CachedSolution& cachedSoln,
QuerySolution** out) {
invariant(!cachedSoln.plannerData.empty());
invariant(out);
// A query not suitable for caching should not have made its way into the cache.
invariant(PlanCache::shouldCacheQuery(query));
// Look up winning solution in cached solution's array.
const SolutionCacheData& winnerCacheData = *cachedSoln.plannerData[0];
if (SolutionCacheData::WHOLE_IXSCAN_SOLN == winnerCacheData.solnType) {
// The solution can be constructed by a scan over the entire index.
QuerySolution* soln = buildWholeIXSoln(
*winnerCacheData.tree->entry, query, params, winnerCacheData.wholeIXSolnDir);
if (soln == NULL) {
return Status(ErrorCodes::BadValue,
"plan cache error: soln that uses index to provide sort");
} else {
*out = soln;
return Status::OK();
}
} else if (SolutionCacheData::COLLSCAN_SOLN == winnerCacheData.solnType) {
// The cached solution is a collection scan. We don't cache collscans
// with tailable==true, hence the false below.
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (soln == NULL) {
return Status(ErrorCodes::BadValue, "plan cache error: collection scan soln");
} else {
*out = soln;
return Status::OK();
}
}
// SolutionCacheData::USE_TAGS_SOLN == cacheData->solnType
// If we're here then this is neither the whole index scan or collection scan
// cases, and we proceed by using the PlanCacheIndexTree to tag the query tree.
// Create a copy of the expression tree. We use cachedSoln to annotate this with indices.
unique_ptr<MatchExpression> clone = std::move(query.root()->shallowClone());
LOG(5) << "Tagging the match expression according to cache data: " << endl
<< "Filter:" << endl
<< clone->toString() << "Cache data:" << endl
<< winnerCacheData.toString();
// Map from index name to index number.
// TODO: can we assume that the index numbering has the same lifetime
// as the cache state?
map<BSONObj, size_t> indexMap;
for (size_t i = 0; i < params.indices.size(); ++i) {
const IndexEntry& ie = params.indices[i];
indexMap[ie.keyPattern] = i;
LOG(5) << "Index " << i << ": " << ie.keyPattern.toString() << endl;
}
Status s = tagAccordingToCache(clone.get(), winnerCacheData.tree.get(), indexMap);
if (!s.isOK()) {
return s;
}
// The planner requires a defined sort order.
sortUsingTags(clone.get());
LOG(5) << "Tagged tree:" << endl
<< clone->toString();
// Use the cached index assignments to build solnRoot.
QuerySolutionNode* solnRoot = QueryPlannerAccess::buildIndexedDataAccess(
query, clone.release(), false, params.indices, params);
if (!solnRoot) {
return Status(ErrorCodes::BadValue,
str::stream() << "Failed to create data access plan from cache. Query: "
<< query.toStringShort());
}
// Takes ownership of 'solnRoot'.
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (!soln) {
return Status(ErrorCodes::BadValue,
str::stream()
<< "Failed to analyze plan from cache. Query: " << query.toStringShort());
}
LOG(5) << "Planner: solution constructed from the cache:\n" << soln->toString();
*out = soln;
return Status::OK();
}
Status getRunnerDistinct(Collection* collection,
const BSONObj& query,
const string& field,
Runner** out) {
// This should'a been checked by the distinct command.
verify(collection);
// TODO: check for idhack here?
// When can we do a fast distinct hack?
// 1. There is a plan with just one leaf and that leaf is an ixscan.
// 2. The ixscan indexes the field we're interested in.
// 2a: We are correct if the index contains the field but for now we look for prefix.
// 3. The query is covered/no fetch.
//
// We go through normal planning (with limited parameters) to see if we can produce
// a soln with the above properties.
QueryPlannerParams plannerParams;
plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN;
IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false);
while (ii.more()) {
const IndexDescriptor* desc = ii.next();
// The distinct hack can work if any field is in the index but it's not always clear
// if it's a win unless it's the first field.
if (desc->keyPattern().firstElement().fieldName() == field) {
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->getAccessMethodName(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName(),
desc->infoObj()));
}
}
// If there are no suitable indices for the distinct hack bail out now into regular planning
// with no projection.
if (plannerParams.indices.empty()) {
CanonicalQuery* cq;
Status status = CanonicalQuery::canonicalize(collection->ns().ns(),
query,
BSONObj(),
BSONObj(),
&cq);
if (!status.isOK()) {
return status;
}
// Takes ownership of cq.
return getRunner(collection, cq, out);
}
//
// If we're here, we have an index prefixed by the field we're distinct-ing over.
//
// Applying a projection allows the planner to try to give us covered plans that we can turn
// into the projection hack. getDistinctProjection deals with .find() projection semantics
// (ie _id:1 being implied by default).
BSONObj projection = getDistinctProjection(field);
// Apply a projection of the key. Empty BSONObj() is for the sort.
CanonicalQuery* cq;
Status status = CanonicalQuery::canonicalize(collection->ns().ns(),
query,
BSONObj(),
projection,
&cq);
if (!status.isOK()) {
return status;
}
// If there's no query, we can just distinct-scan one of the indices.
// Not every index in plannerParams.indices may be suitable. Refer to
// getDistinctNodeIndex().
size_t distinctNodeIndex = 0;
if (query.isEmpty() &&
getDistinctNodeIndex(plannerParams.indices, field, &distinctNodeIndex)) {
DistinctNode* dn = new DistinctNode();
dn->indexKeyPattern = plannerParams.indices[distinctNodeIndex].keyPattern;
dn->direction = 1;
IndexBoundsBuilder::allValuesBounds(dn->indexKeyPattern, &dn->bounds);
dn->fieldNo = 0;
QueryPlannerParams params;
// Takes ownership of 'dn'.
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*cq, params, dn);
verify(soln);
LOG(2) << "Using fast distinct: " << cq->toStringShort()
<< ", planSummary: " << getPlanSummary(*soln);
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(collection, *soln, &root, &ws));
*out = new SingleSolutionRunner(collection, cq, soln, root, ws);
return Status::OK();
}
// See if we can answer the query in a fast-distinct compatible fashion.
vector<QuerySolution*> solutions;
status = QueryPlanner::plan(*cq, plannerParams, &solutions);
if (!status.isOK()) {
return getRunner(collection, cq, out);
}
// We look for a solution that has an ixscan we can turn into a distinctixscan
for (size_t i = 0; i < solutions.size(); ++i) {
if (turnIxscanIntoDistinctIxscan(solutions[i], field)) {
// Great, we can use solutions[i]. Clean up the other QuerySolution(s).
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
LOG(2) << "Using fast distinct: " << cq->toStringShort()
<< ", planSummary: " << getPlanSummary(*solutions[i]);
// Build and return the SSR over solutions[i].
WorkingSet* ws;
PlanStage* root;
verify(StageBuilder::build(collection, *solutions[i], &root, &ws));
*out = new SingleSolutionRunner(collection, cq, solutions[i], root, ws);
return Status::OK();
}
}
// If we're here, the planner made a soln with the restricted index set but we couldn't
// translate any of them into a distinct-compatible soln. So, delete the solutions and just
// go through normal planning.
for (size_t i = 0; i < solutions.size(); ++i) {
delete solutions[i];
}
// We drop the projection from the 'cq'. Unfortunately this is not trivial.
delete cq;
status = CanonicalQuery::canonicalize(collection->ns().ns(),
query,
BSONObj(),
BSONObj(),
&cq);
if (!status.isOK()) {
return status;
}
// Takes ownership of cq.
return getRunner(collection, cq, out);
}
std::string newRunQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) {
// Validate the namespace.
const char *ns = q.ns;
uassert(16332, "can't have an empty ns", ns[0]);
const NamespaceString nsString(ns);
uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid());
// Set curop information.
curop.debug().ns = ns;
curop.debug().ntoreturn = q.ntoreturn;
curop.debug().query = q.query;
curop.setQuery(q.query);
// If the query is really a command, run it.
if (nsString.isCommand()) {
int nToReturn = q.ntoreturn;
uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn
<< ") for $cmd type ns - can only be 1 or -1",
nToReturn == 1 || nToReturn == -1);
curop.markCommand();
BufBuilder bb;
bb.skip(sizeof(QueryResult));
BSONObjBuilder cmdResBuf;
if (!runCommands(ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) {
uasserted(13530, "bad or malformed command request?");
}
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
QueryResult* qr = reinterpret_cast<QueryResult*>(bb.buf());
bb.decouple();
qr->setResultFlagsToOk();
qr->len = bb.len();
curop.debug().responseLength = bb.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
result.setData(qr, true);
return "";
}
// This is a read lock. We require this because if we're parsing a $where, the
// where-specific parsing code assumes we have a lock and creates execution machinery that
// requires it.
Client::ReadContext ctx(q.ns);
Collection* collection = ctx.ctx().db()->getCollection( ns );
// Parse the qm into a CanonicalQuery.
CanonicalQuery* cq;
Status canonStatus = CanonicalQuery::canonicalize(q, &cq);
if (!canonStatus.isOK()) {
uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString());
}
verify(cq);
QLOG() << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a runner.
Runner* rawRunner = NULL;
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
// We'll now try to get the query runner that will execute this query for us. There
// are a few cases in which we know upfront which runner we should get and, therefore,
// we shortcut the selection process here.
//
// (a) If the query is over a collection that doesn't exist, we get a special runner
// that's is so (a runner) which doesn't return results, the EOFRunner.
//
// (b) if the query is a replication's initial sync one, we get a SingleSolutinRunner
// that uses a specifically designed stage that skips extents faster (see details in
// exec/oplogstart.h)
//
// Otherwise we go through the selection of which runner is most suited to the
// query + run-time context at hand.
Status status = Status::OK();
if (collection == NULL) {
rawRunner = new EOFRunner(cq, cq->ns());
}
else if (pq.hasOption(QueryOption_OplogReplay)) {
status = getOplogStartHack(collection, cq, &rawRunner);
}
else {
// Takes ownership of cq.
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(pq.ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
status = getRunner(cq, &rawRunner, options);
}
if (!status.isOK()) {
// NOTE: Do not access cq as getRunner has deleted it.
uasserted(17007, "Unable to execute query: " + status.reason());
}
verify(NULL != rawRunner);
auto_ptr<Runner> runner(rawRunner);
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
killCurrentOp.checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
replVerifyReadsOk(&pq);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult));
// How many results have we obtained from the runner?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the Runner in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
// We turn on auto-yielding for the runner here. The runner registers itself with the
// active runners list in ClientCursor.
auto_ptr<ScopedRunnerRegistration> safety(new ScopedRunnerRegistration(runner.get()));
runner->setYieldPolicy(Runner::YIELD_AUTO);
BSONObj obj;
Runner::RunnerState state;
// uint64_t numMisplacedDocs = 0;
// set this outside loop. we will need to use this both within loop and when deciding
// to fill in explain information
const bool isExplain = pq.isExplain();
// Have we retrieved info about which plan the runner will
// use to execute the query yet?
bool gotPlanInfo = false;
PlanInfo* rawInfo;
boost::scoped_ptr<PlanInfo> planInfo;
while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, NULL))) {
// Add result to output buffer. This is unnecessary if explain info is requested
if (!isExplain) {
bb.appendBuf((void*)obj.objdata(), obj.objsize());
}
// Count the result.
++numResults;
// In the case of the multi plan runner, we may not be able to
// successfully retrieve plan info until after the query starts
// to run. This is because the multi plan runner doesn't know what
// plan it will end up using until it runs candidates and selects
// the best.
//
// TODO: Do we ever want to output what the MPR is comparing?
if (!gotPlanInfo) {
Status infoStatus = runner->getInfo(NULL, &rawInfo);
if (infoStatus.isOK()) {
gotPlanInfo = true;
planInfo.reset(rawInfo);
// planSummary is really a ThreadSafeString which copies the data from
// the provided pointer.
curop.debug().planSummary = planInfo->planSummary.c_str();
}
}
// Possibly note slave's position in the oplog.
if (pq.hasOption(QueryOption_OplogReplay)) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
if (isExplain) {
if (enoughForExplain(pq, numResults)) {
break;
}
}
else if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
QLOG() << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
QLOG() << " runner EOF=" << runner->isEOF() << endl;
saveClientCursor = !runner->isEOF();
}
break;
}
}
// Try to get information about the plan which the runner
// will use to execute the query, it we don't have it already.
if (!gotPlanInfo) {
Status infoStatus = runner->getInfo(NULL, &rawInfo);
if (infoStatus.isOK()) {
gotPlanInfo = true;
planInfo.reset(rawInfo);
// planSummary is really a ThreadSafeString which copies the data from
// the provided pointer.
curop.debug().planSummary = planInfo->planSummary.c_str();
}
}
// If we cache the runner later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the runner later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the runner.
safety.reset();
// Caller expects exceptions thrown in certain cases.
if (Runner::RUNNER_ERROR == state) {
TypeExplain* bareExplain;
Status res = runner->getInfo(&bareExplain, NULL);
if (res.isOK()) {
boost::scoped_ptr<TypeExplain> errorExplain(bareExplain);
error() << "Runner error, stats:\n"
<< errorExplain->stats.jsonString(Strict, true);
}
uasserted(17144, "Runner error: " + WorkingSetCommon::toStatusString(obj));
}
// Why save a dead runner?
if (Runner::RUNNER_DEAD == state) {
saveClientCursor = false;
}
else if (pq.hasOption(QueryOption_CursorTailable)) {
// If we're tailing a capped collection, we don't bother saving the cursor if the
// collection is empty. Otherwise, the semantics of the tailable cursor is that the
// client will keep trying to read from it. So we'll keep it around.
Collection* collection = ctx.ctx().db()->getCollection(cq->ns());
if (collection && collection->numRecords() != 0 && pq.getNumToReturn() != 1) {
saveClientCursor = true;
}
}
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
// Used to fill in explain and to determine if the query is slow enough to be logged.
int elapsedMillis = curop.elapsedMillis();
// Get explain information if:
// 1) it is needed by an explain query;
// 2) profiling is enabled; or
// 3) profiling is disabled but we still need explain details to log a "slow" query.
// Producing explain information is expensive and should be done only if we are certain
// the information will be used.
boost::scoped_ptr<TypeExplain> explain(NULL);
if (isExplain ||
ctx.ctx().db()->getProfilingLevel() > 0 ||
elapsedMillis > serverGlobalParams.slowMS) {
// Ask the runner to produce explain information.
TypeExplain* bareExplain;
Status res = runner->getInfo(&bareExplain, NULL);
if (res.isOK()) {
explain.reset(bareExplain);
}
else if (isExplain) {
error() << "could not produce explain of query '" << pq.getFilter()
<< "', error: " << res.reason();
// If numResults and the data in bb don't correspond, we'll crash later when rooting
// through the reply msg.
BSONObj emptyObj;
bb.appendBuf((void*)emptyObj.objdata(), emptyObj.objsize());
// The explain output is actually a result.
numResults = 1;
// TODO: we can fill out millis etc. here just fine even if the plan screwed up.
}
}
// Fill in the missing run-time fields in explain, starting with propeties of
// the process running the query.
if (isExplain && NULL != explain.get()) {
std::string server = mongoutils::str::stream()
<< getHostNameCached() << ":" << serverGlobalParams.port;
explain->setServer(server);
// We might have skipped some results due to chunk migration etc. so our count is
// correct.
explain->setN(numResults);
// Clock the whole operation.
explain->setMillis(elapsedMillis);
BSONObj explainObj = explain->toBSON();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
// The explain output is actually a result.
numResults = 1;
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the runner until it's getMore'd.
runner->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(collection, runner.get(),
cq->getParsed().getOptions(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
QLOG() << "caching runner with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// ClientCursor takes ownership of runner. Release to make sure it's not deleted.
runner.release();
// TODO document
if (pq.hasOption(QueryOption_OplogReplay) && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.hasOption(QueryOption_Exhaust)) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
else {
QLOG() << "Not caching runner but returning " << numResults << " results.\n";
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult* qr = static_cast<QueryResult*>(result.header());
qr->cursorId = ccId;
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr->setResultFlagsToOk();
qr->setOperation(opReply);
qr->startingFrom = 0;
qr->nReturned = numResults;
// Set debug information for consumption by the profiler.
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
if (NULL != explain.get()) {
if (explain->isScanAndOrderSet()) {
curop.debug().scanAndOrder = explain->getScanAndOrder();
}
else {
curop.debug().scanAndOrder = false;
}
if (explain->isNScannedSet()) {
curop.debug().nscanned = explain->getNScanned();
}
if (explain->isNScannedObjectsSet()) {
curop.debug().nscannedObjects = explain->getNScannedObjects();
}
if (explain->isIDHackSet()) {
curop.debug().idhack = explain->getIDHack();
}
if (!explain->stats.isEmpty()) {
// execStats is a CachedBSONObj because it lives in the race-prone
// curop.
curop.debug().execStats.set(explain->stats);
// Replace exec stats with plan summary if stats cannot fit into CachedBSONObj.
if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) {
BSONObjBuilder bob;
bob.append("summary", curop.debug().planSummary.toString());
curop.debug().execStats.set(bob.done());
}
}
}
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
StatusWith<CursorId> ClusterFind::runQuery(OperationContext* txn,
const CanonicalQuery& query,
const ReadPreferenceSetting& readPref,
std::vector<BSONObj>* results,
BSONObj* viewDefinition) {
invariant(results);
// Projection on the reserved sort key field is illegal in mongos.
if (query.getQueryRequest().getProj().hasField(ClusterClientCursorParams::kSortKeyField)) {
return {ErrorCodes::BadValue,
str::stream() << "Projection contains illegal field '"
<< ClusterClientCursorParams::kSortKeyField
<< "': "
<< query.getQueryRequest().getProj()};
}
auto dbConfig = Grid::get(txn)->catalogCache()->getDatabase(txn, query.nss().db().toString());
if (dbConfig.getStatus() == ErrorCodes::NamespaceNotFound) {
// If the database doesn't exist, we successfully return an empty result set without
// creating a cursor.
return CursorId(0);
} else if (!dbConfig.isOK()) {
return dbConfig.getStatus();
}
std::shared_ptr<ChunkManager> chunkManager;
std::shared_ptr<Shard> primary;
dbConfig.getValue()->getChunkManagerOrPrimary(txn, query.nss().ns(), chunkManager, primary);
// Re-target and re-send the initial find command to the shards until we have established the
// shard version.
for (size_t retries = 1; retries <= kMaxStaleConfigRetries; ++retries) {
auto cursorId = runQueryWithoutRetrying(
txn, query, readPref, chunkManager.get(), std::move(primary), results, viewDefinition);
if (cursorId.isOK()) {
return cursorId;
}
auto status = std::move(cursorId.getStatus());
if (!ErrorCodes::isStaleShardingError(status.code()) &&
status != ErrorCodes::ShardNotFound) {
// Errors other than trying to reach a non existent shard or receiving a stale
// metadata message from MongoD are fatal to the operation. Network errors and
// replication retries happen at the level of the AsyncResultsMerger.
return status;
}
LOG(1) << "Received error status for query " << redact(query.toStringShort())
<< " on attempt " << retries << " of " << kMaxStaleConfigRetries << ": "
<< redact(status);
const bool staleEpoch = (status == ErrorCodes::StaleEpoch);
if (staleEpoch) {
if (!dbConfig.getValue()->reload(txn)) {
// If the reload failed that means the database wasn't found, so successfully return
// an empty result set without creating a cursor.
return CursorId(0);
}
}
chunkManager =
dbConfig.getValue()->getChunkManagerIfExists(txn, query.nss().ns(), true, staleEpoch);
if (!chunkManager) {
dbConfig.getValue()->getChunkManagerOrPrimary(
txn, query.nss().ns(), chunkManager, primary);
}
}
return {ErrorCodes::StaleShardVersion,
str::stream() << "Retried " << kMaxStaleConfigRetries
<< " times without successfully establishing shard version."};
}