BSONObj ShardKeyPattern::moveToFront(const BSONObj& obj) const {
vector<const char*> keysToMove;
keysToMove.push_back("_id");
BSONForEach(e, pattern) {
if (strchr(e.fieldName(), '.') == NULL)
keysToMove.push_back(e.fieldName());
}
if (keysToMove.size() == 1) {
return obj;
}
else {
BufBuilder buf (obj.objsize());
buf.appendNum(obj.objsize());
vector<pair<const char*, size_t> > copies;
pair<const char*, size_t> toCopy ((const char*)NULL, 0); // C++ NULL isn't a pointer type yet
BSONForEach(e, obj) {
bool moveToFront = false;
for (vector<const char*>::const_iterator it(keysToMove.begin()), end(keysToMove.end()); it!=end; ++it) {
if (strcmp(e.fieldName(), *it) == 0) {
moveToFront = true;
break;
}
}
if (moveToFront) {
buf.appendBuf(e.fieldName()-1, e.size());
if (toCopy.first) {
copies.push_back(toCopy);
toCopy.first = NULL;
}
}
else {
if (!toCopy.first) {
toCopy.first = e.fieldName()-1;
toCopy.second = e.size();
}
else {
toCopy.second += e.size();
}
}
}
for (vector<pair<const char*, size_t> >::const_iterator it(copies.begin()), end(copies.end()); it!=end; ++it) {
buf.appendBuf(it->first, it->second);
}
if (toCopy.first) {
buf.appendBuf(toCopy.first, toCopy.second);
}
buf.appendChar('\0');
BSONObj out (buf.buf(), true);
buf.decouple();
return out;
}
/** caller handles locking */
static bool PREPLOGBUFFER(BufBuilder& bb) {
if( writes.empty() )
return false;
bb.reset();
unsigned *lenInBlockHeader;
{
// JSectHeader
bb.appendStr("\nHH\n", false);
lenInBlockHeader = (unsigned *) bb.skip(4);
}
string lastFilePath;
{
scoped_lock lk(privateViews._mutex());
for( vector<WriteIntent>::iterator i = writes.begin(); i != writes.end(); i++ ) {
size_t ofs;
MongoMMF *mmf = privateViews._find(i->p, ofs);
if( mmf == 0 ) {
journalingFailure("view pointer cannot be resolved");
}
else {
if( mmf->filePath() != lastFilePath ) {
lastFilePath = mmf->filePath();
JDbContext c;
bb.appendStruct(c);
bb.appendStr(lastFilePath);
}
JEntry e;
e.len = i->len;
e.fileNo = mmf->fileSuffixNo();
bb.appendStruct(e);
bb.appendBuf(i->p, i->len);
}
}
}
{
JSectFooter f;
f.hash = 0;
bb.appendStruct(f);
}
{
unsigned L = (bb.len() + 8191) & 0xffffe000; // fill to alignment
dassert( L >= (unsigned) bb.len() );
*lenInBlockHeader = L;
unsigned padding = L - bb.len();
bb.skip(padding);
dassert( bb.len() % 8192 == 0 );
}
writes.clear();
alreadyNoted.clear();
return true;
}
static bool receivedQuery(Client& c, DbResponse& dbresponse, Message& m ){
bool ok = true;
MSGID responseTo = m.header()->id;
DbMessage d(m);
QueryMessage q(d);
auto_ptr< Message > resp( new Message() );
CurOp& op = *(c.curop());
try {
dbresponse.exhaust = runQuery(m, q, op, *resp);
assert( !resp->empty() );
}
catch ( AssertionException& e ) {
ok = false;
op.debug().str << " exception ";
LOGSOME {
log() << "assertion " << e.toString() << " ns:" << q.ns << " query:" <<
(q.query.valid() ? q.query.toString() : "query object is corrupt") << endl;
if( q.ntoskip || q.ntoreturn )
log() << " ntoskip:" << q.ntoskip << " ntoreturn:" << q.ntoreturn << endl;
}
BSONObjBuilder err;
e.getInfo().append( err );
BSONObj errObj = err.done();
BufBuilder b;
b.skip(sizeof(QueryResult));
b.appendBuf((void*) errObj.objdata(), errObj.objsize());
// todo: call replyToQuery() from here instead of this!!! see dbmessage.h
QueryResult * msgdata = (QueryResult *) b.buf();
b.decouple();
QueryResult *qr = msgdata;
qr->_resultFlags() = ResultFlag_ErrSet;
if ( e.getCode() == StaleConfigInContextCode )
qr->_resultFlags() |= ResultFlag_ShardConfigStale;
qr->len = b.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
resp.reset( new Message() );
resp->setData( msgdata, true );
}
if ( op.shouldDBProfile( 0 ) ){
op.debug().str << " bytes:" << resp->header()->dataLen();
}
dbresponse.response = resp.release();
dbresponse.responseTo = responseTo;
return ok;
}
void generateLegacyQueryErrorResponse(const AssertionException* exception,
const QueryMessage& queryMessage,
CurOp* curop,
Message* response) {
curop->debug().exceptionInfo = exception->getInfo();
log(LogComponent::kQuery) << "assertion " << exception->toString() << " ns:" << queryMessage.ns
<< " query:" << (queryMessage.query.valid()
? queryMessage.query.toString()
: "query object is corrupt");
if (queryMessage.ntoskip || queryMessage.ntoreturn) {
log(LogComponent::kQuery) << " ntoskip:" << queryMessage.ntoskip
<< " ntoreturn:" << queryMessage.ntoreturn;
}
const SendStaleConfigException* scex = (exception->getCode() == ErrorCodes::SendStaleConfig)
? static_cast<const SendStaleConfigException*>(exception)
: NULL;
BSONObjBuilder err;
exception->getInfo().append(err);
if (scex) {
err.append("ok", 0.0);
err.append("ns", scex->getns());
scex->getVersionReceived().addToBSON(err, "vReceived");
scex->getVersionWanted().addToBSON(err, "vWanted");
}
BSONObj errObj = err.done();
if (scex) {
log(LogComponent::kQuery) << "stale version detected during query over " << queryMessage.ns
<< " : " << errObj;
}
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
bb.appendBuf((void*)errObj.objdata(), errObj.objsize());
// TODO: call replyToQuery() from here instead of this!!! see dbmessage.h
QueryResult::View msgdata = bb.buf();
bb.decouple();
QueryResult::View qr = msgdata;
qr.setResultFlags(ResultFlag_ErrSet);
if (scex)
qr.setResultFlags(qr.getResultFlags() | ResultFlag_ShardConfigStale);
qr.msgdata().setLen(bb.len());
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
response->setData(msgdata.view2ptr(), true);
}
bool ShardedClientCursor::sendNextBatch( Request& r , int ntoreturn ,
BufBuilder& buffer, int& docCount ) {
uassert( 10191 , "cursor already done" , ! _done );
int maxSize = 1024 * 1024;
if ( _totalSent > 0 )
maxSize *= 3;
docCount = 0;
// Send more if ntoreturn is 0, or any value > 1
// (one is assumed to be a single doc return, with no cursor)
bool sendMore = ntoreturn == 0 || ntoreturn > 1;
ntoreturn = abs( ntoreturn );
while ( _cursor->more() ) {
BSONObj o = _cursor->next();
buffer.appendBuf( (void*)o.objdata() , o.objsize() );
docCount++;
if ( buffer.len() > maxSize ) {
break;
}
if ( docCount == ntoreturn ) {
// soft limit aka batch size
break;
}
if ( ntoreturn == 0 && _totalSent == 0 && docCount >= 100 ) {
// first batch should be max 100 unless batch size specified
break;
}
}
bool hasMore = sendMore && _cursor->more();
LOG(5) << "\t hasMore: " << hasMore
<< " sendMore: " << sendMore
<< " cursorMore: " << _cursor->more()
<< " ntoreturn: " << ntoreturn
<< " num: " << docCount
<< " wouldSendMoreIfHad: " << sendMore
<< " id:" << getId()
<< " totalSent: " << _totalSent << endl;
_totalSent += docCount;
_done = ! hasMore;
return hasMore;
}
// XXX clean up
static bool runCommands(const char *ns, BSONObj& jsobj, CurOp& curop, BufBuilder &b, BSONObjBuilder& anObjBuilder, bool fromRepl, int queryOptions) {
try {
return _runCommands(ns, jsobj, b, anObjBuilder, fromRepl, queryOptions);
}
catch( SendStaleConfigException& ){
throw;
}
catch ( AssertionException& e ) {
verify( e.getCode() != SendStaleConfigCode && e.getCode() != RecvStaleConfigCode );
Command::appendCommandStatus(anObjBuilder, e.toStatus());
curop.debug().exceptionInfo = e.getInfo();
}
BSONObj x = anObjBuilder.done();
b.appendBuf((void*) x.objdata(), x.objsize());
return true;
}
void replyToQuery( int queryResultFlags, Message& response, const BSONObj& resultObj ) {
BufBuilder bufBuilder;
bufBuilder.skip( sizeof( QueryResult::Value ));
bufBuilder.appendBuf( reinterpret_cast< void *>(
const_cast< char* >( resultObj.objdata() )), resultObj.objsize() );
QueryResult::View queryResult = bufBuilder.buf();
bufBuilder.decouple();
queryResult.setResultFlags(queryResultFlags);
queryResult.msgdata().setLen(bufBuilder.len());
queryResult.msgdata().setOperation( opReply );
queryResult.setCursorId(0);
queryResult.setStartingFrom(0);
queryResult.setNReturned(1);
response.setData( queryResult.view2ptr(), true ); // transport will free
}
void replyToQuery(int queryResultFlags, Message &m, DbResponse &dbresponse, BSONObj obj) {
BufBuilder b;
b.skip(sizeof(QueryResult));
b.appendBuf((void*) obj.objdata(), obj.objsize());
QueryResult* msgdata = (QueryResult *) b.buf();
b.decouple();
QueryResult *qr = msgdata;
qr->_resultFlags() = queryResultFlags;
qr->len = b.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
Message *resp = new Message();
resp->setData(msgdata, true); // transport will free
dbresponse.response = resp;
dbresponse.responseTo = m.header()->id;
}
void replyToQuery( int queryResultFlags, Message& response, const BSONObj& resultObj ) {
BufBuilder bufBuilder;
bufBuilder.skip( sizeof( QueryResult ));
bufBuilder.appendBuf( reinterpret_cast< void *>(
const_cast< char* >( resultObj.objdata() )), resultObj.objsize() );
QueryResult* queryResult = reinterpret_cast< QueryResult* >( bufBuilder.buf() );
bufBuilder.decouple();
queryResult->_resultFlags() = queryResultFlags;
queryResult->len = bufBuilder.len();
queryResult->setOperation( opReply );
queryResult->cursorId = 0;
queryResult->startingFrom = 0;
queryResult->nReturned = 1;
response.setData( queryResult, true ); // transport will free
}
bool runCommands(const char *ns, BSONObj& jsobj, CurOp& curop, BufBuilder &b, BSONObjBuilder& anObjBuilder, bool fromRepl, int queryOptions) {
try {
return _runCommands(ns, jsobj, b, anObjBuilder, fromRepl, queryOptions);
}
catch( SendStaleConfigException& ){
throw;
}
catch ( AssertionException& e ) {
verify( e.getCode() != SendStaleConfigCode && e.getCode() != RecvStaleConfigCode );
e.getInfo().append( anObjBuilder , "assertion" , "assertionCode" );
curop.debug().exceptionInfo = e.getInfo();
}
anObjBuilder.append("errmsg", "db assertion failure");
anObjBuilder.append("ok", 0.0);
BSONObj x = anObjBuilder.done();
b.appendBuf((void*) x.objdata(), x.objsize());
return true;
}
std::string newRunQuery(OperationContext* txn,
Message& m,
QueryMessage& q,
CurOp& curop,
Message &result,
bool fromDBDirectClient) {
// Validate the namespace.
const char *ns = q.ns;
uassert(16332, "can't have an empty ns", ns[0]);
const NamespaceString nsString(ns);
uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid());
// Set curop information.
curop.debug().ns = ns;
curop.debug().ntoreturn = q.ntoreturn;
curop.debug().query = q.query;
curop.setQuery(q.query);
// If the query is really a command, run it.
if (nsString.isCommand()) {
int nToReturn = q.ntoreturn;
uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn
<< ") for $cmd type ns - can only be 1 or -1",
nToReturn == 1 || nToReturn == -1);
curop.markCommand();
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder cmdResBuf;
if (!runCommands(txn, ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) {
uasserted(13530, "bad or malformed command request?");
}
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
const NamespaceString nss(q.ns);
// Parse the qm into a CanonicalQuery.
CanonicalQuery* cq;
Status canonStatus = CanonicalQuery::canonicalize(
q, &cq, WhereCallbackReal(txn, StringData(nss.db())));
if (!canonStatus.isOK()) {
uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString());
}
QLOG() << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a plan executor.
PlanExecutor* rawExec = NULL;
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
AutoGetCollectionForRead ctx(txn, nss);
const int dbProfilingLevel = (ctx.getDb() != NULL) ? ctx.getDb()->getProfilingLevel() :
serverGlobalParams.defaultProfile;
Collection* collection = ctx.getCollection();
// We'll now try to get the query executor that will execute this query for us. There
// are a few cases in which we know upfront which executor we should get and, therefore,
// we shortcut the selection process here.
//
// (a) If the query is over a collection that doesn't exist, we use an EOFStage.
//
// (b) if the query is a replication's initial sync one, we use a specifically designed
// stage that skips extents faster (see details in exec/oplogstart.h).
//
// Otherwise we go through the selection of which executor is most suited to the
// query + run-time context at hand.
Status status = Status::OK();
if (NULL != collection && pq.getOptions().oplogReplay) {
// Takes ownership of 'cq'.
status = getOplogStartHack(txn, collection, cq, &rawExec);
}
else {
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(pq.ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
// Takes ownership of 'cq'.
status = getExecutor(txn, collection, cq, PlanExecutor::YIELD_AUTO, &rawExec, options);
}
if (!status.isOK()) {
// NOTE: Do not access cq as getExecutor has deleted it.
uasserted(17007, "Unable to execute query: " + status.reason());
}
verify(NULL != rawExec);
auto_ptr<PlanExecutor> exec(rawExec);
// If it's actually an explain, do the explain and return rather than falling through
// to the normal query execution loop.
if (pq.isExplain()) {
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder explainBob;
Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob);
// Add the resulting object to the return buffer.
BSONObj explainObj = explainBob.obj();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
// Set query result fields.
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
bool slaveOK = pq.getOptions().slaveOk || pq.hasReadPref();
status = repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor(
txn,
NamespaceString(cq->ns()),
slaveOK);
uassertStatusOK(status);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult::Value));
// How many results have we obtained from the executor?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the PlanExecutor in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
BSONObj obj;
PlanExecutor::ExecState state;
// uint64_t numMisplacedDocs = 0;
// Get summary info about which plan the executor is using.
curop.debug().planSummary = Explain::getPlanSummary(exec.get());
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.getOptions().oplogReplay) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
QLOG() << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
QLOG() << " executor EOF=" << exec->isEOF() << endl;
saveClientCursor = !exec->isEOF();
}
break;
}
}
// If we cache the executor later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the executor later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the executor.
exec->deregisterExec();
// Caller expects exceptions thrown in certain cases.
if (PlanExecutor::EXEC_ERROR == state) {
scoped_ptr<PlanStageStats> stats(exec->getStats());
error() << "Plan executor error, stats: "
<< Explain::statsToBSON(*stats);
uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj));
}
// Why save a dead executor?
if (PlanExecutor::DEAD == state) {
saveClientCursor = false;
}
else if (pq.getOptions().tailable) {
// If we're tailing a capped collection, we don't bother saving the cursor if the
// collection is empty. Otherwise, the semantics of the tailable cursor is that the
// client will keep trying to read from it. So we'll keep it around.
if (collection && collection->numRecords(txn) != 0 && pq.getNumToReturn() != 1) {
saveClientCursor = true;
}
}
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
const logger::LogComponent queryLogComponent = logger::LogComponent::kQuery;
const logger::LogSeverity logLevelOne = logger::LogSeverity::Debug(1);
PlanSummaryStats summaryStats;
Explain::getSummaryStats(exec.get(), &summaryStats);
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
curop.debug().scanAndOrder = summaryStats.hasSortStage;
curop.debug().nscanned = summaryStats.totalKeysExamined;
curop.debug().nscannedObjects = summaryStats.totalDocsExamined;
curop.debug().idhack = summaryStats.isIdhack;
// Set debug information for consumption by the profiler.
if (dbProfilingLevel > 0 ||
curop.elapsedMillis() > serverGlobalParams.slowMS ||
logger::globalLogDomain()->shouldLog(queryLogComponent, logLevelOne)) {
// Get BSON stats.
scoped_ptr<PlanStageStats> execStats(exec->getStats());
BSONObjBuilder statsBob;
Explain::statsToBSON(*execStats, &statsBob);
curop.debug().execStats.set(statsBob.obj());
// Replace exec stats with plan summary if stats cannot fit into CachedBSONObj.
if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) {
BSONObjBuilder bob;
bob.append("summary", curop.debug().planSummary.toString());
curop.debug().execStats.set(bob.done());
}
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the executor until it's getMore'd.
exec->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(collection, exec.get(),
cq->getParsed().getOptions().toInt(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
if (fromDBDirectClient) {
cc->setUnownedRecoveryUnit(txn->recoveryUnit());
}
else if (state == PlanExecutor::IS_EOF && pq.getOptions().tailable) {
// Don't stash the RU for tailable cursors at EOF, let them get a new RU on their
// next getMore.
}
else {
// We stash away the RecoveryUnit in the ClientCursor. It's used for subsequent
// getMore requests. The calling OpCtx gets a fresh RecoveryUnit.
cc->setOwnedRecoveryUnit(txn->releaseRecoveryUnit());
StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine();
txn->setRecoveryUnit(storageEngine->newRecoveryUnit(txn));
}
QLOG() << "caching executor with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// ClientCursor takes ownership of executor. Release to make sure it's not deleted.
exec.release();
// TODO document
if (pq.getOptions().oplogReplay && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.getOptions().exhaust) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
else {
QLOG() << "Not caching executor but returning " << numResults << " results.\n";
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult::View qr = result.header().view2ptr();
qr.setCursorId(ccId);
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr.setResultFlagsToOk();
qr.msgdata().setOperation(opReply);
qr.setStartingFrom(0);
qr.setNReturned(numResults);
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
bool ShardedClientCursor::sendNextBatch(int batchSize, BufBuilder& buffer, int& docCount) {
uassert(10191, "cursor already done", !_done);
int maxSize = 1024 * 1024;
if (_totalSent > 0)
maxSize *= 3;
docCount = 0;
// If batchSize is negative, it means that we should send up to -batchSize results
// back to the client, and that we should only send a *single batch*. An batchSize of
// 1 is also a special case which means "return up to 1 result in a single batch" (so
// that +1 actually has the same meaning of -1). For all other values of batchSize, we
// may have to return multiple batches.
const bool sendMoreBatches = batchSize == 0 || batchSize > 1;
batchSize = abs(batchSize);
// Set the initial batch size to 101, just like mongoD.
if (batchSize == 0 && _totalSent == 0)
batchSize = 101;
// Set batch size to batchSize requested by the current operation unconditionally. This is
// necessary because if the loop exited due to docCount == batchSize then setBatchSize(0) was
// called, so the next _cusor->more() will be called with a batch size of 0 if the cursor
// buffer was drained the previous run. Unconditionally setting the batch size ensures that
// we don't ask for a batch size of zero as a side effect.
_cursor->setBatchSize(batchSize);
bool cursorHasMore = true;
while ((cursorHasMore = _cursor->more())) {
BSONObj o = _cursor->next();
buffer.appendBuf((void*)o.objdata(), o.objsize());
++docCount;
// Ensure that the next batch will never wind up requesting more docs from the shard
// than are remaining to satisfy the initial batchSize.
if (batchSize != 0) {
if (docCount == batchSize)
break;
_cursor->setBatchSize(batchSize - docCount);
}
if (buffer.len() > maxSize) {
break;
}
}
// We need to request another batch if the following two conditions hold:
//
// 1. batchSize is positive and not equal to 1 (see the comment above). This condition
// is stored in 'sendMoreBatches'.
//
// 2. The last call to _cursor->more() was true (i.e. we never explicitly got a false
// value from _cursor->more()). This condition is stored in 'cursorHasMore'. If the server
// hits EOF while executing a query or a getmore, it will pass a cursorId of 0 in the
// query response to indicate that there are no more results. In this case, _cursor->more()
// will be explicitly false, and we know for sure that we do not have to send more batches.
//
// On the other hand, if _cursor->more() is true there may or may not be more results.
// Suppose that the mongod generates enough results to fill this batch. In this case it
// does not know whether not there are more, because doing so would require requesting an
// extra result and seeing whether we get EOF. The mongod sends a valid cursorId to
// indicate that there may be more. We do the same here: we indicate that there may be
// more results to retrieve by setting 'hasMoreBatches' to true.
bool hasMoreBatches = sendMoreBatches && cursorHasMore;
LOG(5) << "\t hasMoreBatches: " << hasMoreBatches << " sendMoreBatches: " << sendMoreBatches
<< " cursorHasMore: " << cursorHasMore << " batchSize: " << batchSize
<< " num: " << docCount << " id:" << getId() << " totalSent: " << _totalSent << endl;
_totalSent += docCount;
_done = !hasMoreBatches;
return hasMoreBatches;
}
bool ShardedClientCursor::sendNextBatch(int ntoreturn, BufBuilder& buffer, int& docCount) {
uassert( 10191 , "cursor already done" , ! _done );
int maxSize = 1024 * 1024;
if ( _totalSent > 0 )
maxSize *= 3;
docCount = 0;
// If ntoreturn is negative, it means that we should send up to -ntoreturn results
// back to the client, and that we should only send a *single batch*. An ntoreturn of
// 1 is also a special case which means "return up to 1 result in a single batch" (so
// that +1 actually has the same meaning of -1). For all other values of ntoreturn, we
// may have to return multiple batches.
const bool sendMoreBatches = ntoreturn == 0 || ntoreturn > 1;
ntoreturn = abs( ntoreturn );
bool cursorHasMore = true;
while ( ( cursorHasMore = _cursor->more() ) ) {
BSONObj o = _cursor->next();
buffer.appendBuf( (void*)o.objdata() , o.objsize() );
docCount++;
// Ensure that the next batch will never wind up requesting more docs from the shard
// than are remaining to satisfy the initial ntoreturn.
if (ntoreturn != 0) {
_cursor->setBatchSize(ntoreturn - docCount);
}
if ( buffer.len() > maxSize ) {
break;
}
if ( docCount == ntoreturn ) {
// soft limit aka batch size
break;
}
if ( ntoreturn == 0 && _totalSent == 0 && docCount >= 100 ) {
// first batch should be max 100 unless batch size specified
break;
}
}
// We need to request another batch if the following two conditions hold:
//
// 1. ntoreturn is positive and not equal to 1 (see the comment above). This condition
// is stored in 'sendMoreBatches'.
//
// 2. The last call to _cursor->more() was true (i.e. we never explicitly got a false
// value from _cursor->more()). This condition is stored in 'cursorHasMore'. If the server
// hits EOF while executing a query or a getmore, it will pass a cursorId of 0 in the
// query response to indicate that there are no more results. In this case, _cursor->more()
// will be explicitly false, and we know for sure that we do not have to send more batches.
//
// On the other hand, if _cursor->more() is true there may or may not be more results.
// Suppose that the mongod generates enough results to fill this batch. In this case it
// does not know whether not there are more, because doing so would require requesting an
// extra result and seeing whether we get EOF. The mongod sends a valid cursorId to
// indicate that there may be more. We do the same here: we indicate that there may be
// more results to retrieve by setting 'hasMoreBatches' to true.
bool hasMoreBatches = sendMoreBatches && cursorHasMore;
LOG(5) << "\t hasMoreBatches: " << hasMoreBatches
<< " sendMoreBatches: " << sendMoreBatches
<< " cursorHasMore: " << cursorHasMore
<< " ntoreturn: " << ntoreturn
<< " num: " << docCount
<< " id:" << getId()
<< " totalSent: " << _totalSent << endl;
_totalSent += docCount;
_done = ! hasMoreBatches;
return hasMoreBatches;
}
std::string runQuery(OperationContext* txn,
QueryMessage& q,
const NamespaceString& nss,
CurOp& curop,
Message &result) {
// Validate the namespace.
uassert(16256, str::stream() << "Invalid ns [" << nss.ns() << "]", nss.isValid());
invariant(!nss.isCommand());
// Set curop information.
beginQueryOp(nss, q.query, q.ntoreturn, q.ntoskip, &curop);
// Parse the qm into a CanonicalQuery.
std::auto_ptr<CanonicalQuery> cq;
{
CanonicalQuery* cqRaw;
Status canonStatus = CanonicalQuery::canonicalize(q,
&cqRaw,
WhereCallbackReal(txn, nss.db()));
if (!canonStatus.isOK()) {
uasserted(17287, str::stream() << "Can't canonicalize query: "
<< canonStatus.toString());
}
cq.reset(cqRaw);
}
invariant(cq.get());
LOG(5) << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a plan executor.
AutoGetCollectionForRead ctx(txn, nss);
Collection* collection = ctx.getCollection();
const int dbProfilingLevel = ctx.getDb() ? ctx.getDb()->getProfilingLevel() :
serverGlobalParams.defaultProfile;
// We have a parsed query. Time to get the execution plan for it.
std::unique_ptr<PlanExecutor> exec;
{
PlanExecutor* rawExec;
Status execStatus = getExecutorFind(txn,
collection,
nss,
cq.release(),
PlanExecutor::YIELD_AUTO,
&rawExec);
uassertStatusOK(execStatus);
exec.reset(rawExec);
}
const LiteParsedQuery& pq = exec->getCanonicalQuery()->getParsed();
// If it's actually an explain, do the explain and return rather than falling through
// to the normal query execution loop.
if (pq.isExplain()) {
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder explainBob;
Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob);
// Add the resulting object to the return buffer.
BSONObj explainObj = explainBob.obj();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
// Set query result fields.
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(nss.ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
bool slaveOK = pq.isSlaveOk() || pq.hasReadPref();
Status serveReadsStatus = repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor(
txn,
nss,
slaveOK);
uassertStatusOK(serveReadsStatus);
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult::Value));
// How many results have we obtained from the executor?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
Timestamp slaveReadTill;
BSONObj obj;
PlanExecutor::ExecState state;
// uint64_t numMisplacedDocs = 0;
// Get summary info about which plan the executor is using.
curop.debug().planSummary = Explain::getPlanSummary(exec.get());
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.isOplogReplay()) {
BSONElement e = obj["ts"];
if (Date == e.type() || bsonTimestamp == e.type()) {
slaveReadTill = e.timestamp();
}
}
if (enoughForFirstBatch(pq, numResults, bb.len())) {
LOG(5) << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
break;
}
}
// If we cache the executor later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the executor later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the executor.
exec->deregisterExec();
// Caller expects exceptions thrown in certain cases.
if (PlanExecutor::FAILURE == state) {
scoped_ptr<PlanStageStats> stats(exec->getStats());
error() << "Plan executor error, stats: "
<< Explain::statsToBSON(*stats);
uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj));
}
// TODO: Currently, chunk ranges are kept around until all ClientCursors created while the
// chunk belonged on this node are gone. Separating chunk lifetime management from
// ClientCursor should allow this check to go away.
if (!shardingState.getVersion(nss.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(nss.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(nss.ns()));
}
// Fill out curop based on query results. If we have a cursorid, we will fill out curop with
// this cursorid later.
long long ccId = 0;
if (shouldSaveCursor(txn, collection, state, exec.get())) {
// We won't use the executor until it's getMore'd.
exec->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(collection->getCursorManager(),
exec.release(),
nss.ns(),
pq.getOptions(),
pq.getFilter());
ccId = cc->cursorid();
if (txn->getClient()->isInDirectClient()) {
cc->setUnownedRecoveryUnit(txn->recoveryUnit());
}
else if (state == PlanExecutor::IS_EOF && pq.isTailable()) {
// Don't stash the RU for tailable cursors at EOF, let them get a new RU on their
// next getMore.
}
else {
// We stash away the RecoveryUnit in the ClientCursor. It's used for subsequent
// getMore requests. The calling OpCtx gets a fresh RecoveryUnit.
txn->recoveryUnit()->abandonSnapshot();
cc->setOwnedRecoveryUnit(txn->releaseRecoveryUnit());
StorageEngine* storageEngine = getGlobalServiceContext()->getGlobalStorageEngine();
invariant(txn->setRecoveryUnit(storageEngine->newRecoveryUnit(),
OperationContext::kNotInUnitOfWork)
== OperationContext::kNotInUnitOfWork);
}
LOG(5) << "caching executor with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// TODO document
if (pq.isOplogReplay() && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.isExhaust()) {
curop.debug().exhaust = true;
}
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
endQueryOp(cc->getExecutor(), dbProfilingLevel, numResults, ccId, &curop);
}
else {
LOG(5) << "Not caching executor but returning " << numResults << " results.\n";
endQueryOp(exec.get(), dbProfilingLevel, numResults, ccId, &curop);
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult::View qr = result.header().view2ptr();
qr.setCursorId(ccId);
qr.setResultFlagsToOk();
qr.msgdata().setOperation(opReply);
qr.setStartingFrom(0);
qr.setNReturned(numResults);
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? nss.ns() : "";
}
static bool receivedQuery(Client& c, DbResponse& dbresponse, Message& m ) {
bool ok = true;
MSGID responseTo = m.header()->id;
DbMessage d(m);
QueryMessage q(d);
auto_ptr< Message > resp( new Message() );
CurOp& op = *(c.curop());
shared_ptr<AssertionException> ex;
try {
if (!NamespaceString(d.getns()).isCommand()) {
// Auth checking for Commands happens later.
Status status = cc().getAuthorizationManager()->checkAuthForQuery(d.getns());
uassert(16550, status.reason(), status.isOK());
}
dbresponse.exhaustNS = runQuery(m, q, op, *resp);
verify( !resp->empty() );
}
catch ( SendStaleConfigException& e ){
ex.reset( new SendStaleConfigException( e.getns(), e.getInfo().msg, e.getVersionReceived(), e.getVersionWanted() ) );
ok = false;
}
catch ( AssertionException& e ) {
ex.reset( new AssertionException( e.getInfo().msg, e.getCode() ) );
ok = false;
}
if( ex ){
op.debug().exceptionInfo = ex->getInfo();
LOGWITHRATELIMIT {
log() << "assertion " << ex->toString() << " ns:" << q.ns << " query:" <<
(q.query.valid() ? q.query.toString() : "query object is corrupt") << endl;
if( q.ntoskip || q.ntoreturn )
log() << " ntoskip:" << q.ntoskip << " ntoreturn:" << q.ntoreturn << endl;
}
SendStaleConfigException* scex = NULL;
if ( ex->getCode() == SendStaleConfigCode ) scex = static_cast<SendStaleConfigException*>( ex.get() );
BSONObjBuilder err;
ex->getInfo().append( err );
if( scex ){
err.append( "ns", scex->getns() );
scex->getVersionReceived().addToBSON( err, "vReceived" );
scex->getVersionWanted().addToBSON( err, "vWanted" );
}
BSONObj errObj = err.done();
if( scex ){
log() << "stale version detected during query over "
<< q.ns << " : " << errObj << endl;
}
else{
log() << "problem detected during query over "
<< q.ns << " : " << errObj << endl;
}
BufBuilder b;
b.skip(sizeof(QueryResult));
b.appendBuf((void*) errObj.objdata(), errObj.objsize());
// todo: call replyToQuery() from here instead of this!!! see dbmessage.h
QueryResult * msgdata = (QueryResult *) b.buf();
b.decouple();
QueryResult *qr = msgdata;
qr->_resultFlags() = ResultFlag_ErrSet;
if( scex ) qr->_resultFlags() |= ResultFlag_ShardConfigStale;
qr->len = b.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
resp.reset( new Message() );
resp->setData( msgdata, true );
}
op.debug().responseLength = resp->header()->dataLen();
dbresponse.response = resp.release();
dbresponse.responseTo = responseTo;
return ok;
}
BSONObj KeyV1::toBson(BufBuilder &bb) const {
verify( _keyData != 0 );
if( !isCompactFormat() )
return bson();
BSONObjBuilder b(bb);
const unsigned char *p = _keyData;
while( 1 ) {
unsigned bits = *p++;
switch( bits & cFULLTYPEMASK ) {
case cminkey: b.appendMinKey(""); break;
case cnull: b.appendNull(""); break;
case cfalse: b.appendBool("", false); break;
case ctrue: b.appendBool("", true); break;
case cmaxkey:
b.appendMaxKey("");
break;
case cstring:
{
unsigned sz = *p++;
// we build the element ourself as we have to null terminate it
BufBuilder &bb = b.bb();
bb.appendNum((char) String);
bb.appendUChar(0); // fieldname ""
bb.appendNum(sz+1);
bb.appendBuf(p, sz);
bb.appendUChar(0); // null char at end of string
p += sz;
break;
}
case coid:
b.appendOID("", (OID *) p);
p += sizeof(OID);
break;
case cbindata:
{
int len = binDataCodeToLength(*p);
int subtype = (*p) & BinDataTypeMask;
if( subtype & 0x8 ) {
subtype = (subtype & 0x7) | 0x80;
}
b.appendBinData("", len, (BinDataType) subtype, ++p);
p += len;
break;
}
case cdate:
b.appendDate("", (Date_t&) *p);
p += 8;
break;
case cdouble:
b.append("", (double&) *p);
p += sizeof(double);
break;
case cint:
b.append("", static_cast< int >((reinterpret_cast< const PackedDouble& >(*p)).d));
p += sizeof(double);
break;
case clong:
b.append("", static_cast< long long>((reinterpret_cast< const PackedDouble& >(*p)).d));
p += sizeof(double);
break;
case cint64:
b.append("", *reinterpret_cast<const long long *>(p));
p += sizeof(long long);
break;
default:
verify(false);
}
if( (bits & cHASMORE) == 0 )
break;
}
return b.done();
}
static bool receivedQuery(Client& c, DbResponse& dbresponse, Message& m ) {
bool ok = true;
MSGID responseTo = m.header()->id;
DbMessage d(m);
QueryMessage q(d);
auto_ptr< Message > resp( new Message() );
CurOp& op = *(c.curop());
shared_ptr<AssertionException> ex;
try {
dbresponse.exhaust = runQuery(m, q, op, *resp);
assert( !resp->empty() );
}
catch ( SendStaleConfigException& e ){
ex.reset( new SendStaleConfigException( e.getns(), e.getInfo().msg ) );
ok = false;
}
catch ( AssertionException& e ) {
ex.reset( new AssertionException( e.getInfo().msg, e.getCode() ) );
ok = false;
}
if( ex ){
op.debug().exceptionInfo = ex->getInfo();
LOGWITHRATELIMIT {
log() << "assertion " << ex->toString() << " ns:" << q.ns << " query:" <<
(q.query.valid() ? q.query.toString() : "query object is corrupt") << endl;
if( q.ntoskip || q.ntoreturn )
log() << " ntoskip:" << q.ntoskip << " ntoreturn:" << q.ntoreturn << endl;
}
SendStaleConfigException* scex = NULL;
if ( ex->getCode() == SendStaleConfigCode ) scex = static_cast<SendStaleConfigException*>( ex.get() );
BSONObjBuilder err;
ex->getInfo().append( err );
if( scex ) err.append( "ns", scex->getns() );
BSONObj errObj = err.done();
log() << errObj << endl;
BufBuilder b;
b.skip(sizeof(QueryResult));
b.appendBuf((void*) errObj.objdata(), errObj.objsize());
// todo: call replyToQuery() from here instead of this!!! see dbmessage.h
QueryResult * msgdata = (QueryResult *) b.buf();
b.decouple();
QueryResult *qr = msgdata;
qr->_resultFlags() = ResultFlag_ErrSet;
if( scex ) qr->_resultFlags() |= ResultFlag_ShardConfigStale;
qr->len = b.len();
qr->setOperation(opReply);
qr->cursorId = 0;
qr->startingFrom = 0;
qr->nReturned = 1;
resp.reset( new Message() );
resp->setData( msgdata, true );
}
op.debug().responseLength = resp->header()->dataLen();
dbresponse.response = resp.release();
dbresponse.responseTo = responseTo;
return ok;
}
std::string runQuery(OperationContext* txn,
QueryMessage& q,
const NamespaceString& nss,
Message& result) {
CurOp& curop = *CurOp::get(txn);
uassert(ErrorCodes::InvalidNamespace,
str::stream() << "Invalid ns [" << nss.ns() << "]",
nss.isValid());
invariant(!nss.isCommand());
// Set curop information.
beginQueryOp(txn, nss, q.query, q.ntoreturn, q.ntoskip);
// Parse the qm into a CanonicalQuery.
auto statusWithCQ = CanonicalQuery::canonicalize(q, ExtensionsCallbackReal(txn, &nss));
if (!statusWithCQ.isOK()) {
uasserted(
17287,
str::stream() << "Can't canonicalize query: " << statusWithCQ.getStatus().toString());
}
unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
invariant(cq.get());
LOG(5) << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a plan executor.
AutoGetCollectionForRead ctx(txn, nss);
Collection* collection = ctx.getCollection();
const int dbProfilingLevel =
ctx.getDb() ? ctx.getDb()->getProfilingLevel() : serverGlobalParams.defaultProfile;
// We have a parsed query. Time to get the execution plan for it.
std::unique_ptr<PlanExecutor> exec = uassertStatusOK(
getExecutorFind(txn, collection, nss, std::move(cq), PlanExecutor::YIELD_AUTO));
const LiteParsedQuery& pq = exec->getCanonicalQuery()->getParsed();
// If it's actually an explain, do the explain and return rather than falling through
// to the normal query execution loop.
if (pq.isExplain()) {
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder explainBob;
Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob);
// Add the resulting object to the return buffer.
BSONObj explainObj = explainBob.obj();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
// Set query result fields.
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
bool slaveOK = pq.isSlaveOk() || pq.hasReadPref();
Status serveReadsStatus =
repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor(txn, nss, slaveOK);
uassertStatusOK(serveReadsStatus);
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(FindCommon::kInitReplyBufferSize);
bb.skip(sizeof(QueryResult::Value));
// How many results have we obtained from the executor?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
Timestamp slaveReadTill;
BSONObj obj;
PlanExecutor::ExecState state;
// Get summary info about which plan the executor is using.
{
stdx::lock_guard<Client> lk(*txn->getClient());
curop.setPlanSummary_inlock(Explain::getPlanSummary(exec.get()));
}
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
// If we can't fit this result inside the current batch, then we stash it for later.
if (!FindCommon::haveSpaceForNext(obj, numResults, bb.len())) {
exec->enqueue(obj);
break;
}
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.isOplogReplay()) {
BSONElement e = obj["ts"];
if (Date == e.type() || bsonTimestamp == e.type()) {
slaveReadTill = e.timestamp();
}
}
if (FindCommon::enoughForFirstBatch(pq, numResults)) {
LOG(5) << "Enough for first batch, wantMore=" << pq.wantMore()
<< " ntoreturn=" << pq.getNToReturn().value_or(0) << " numResults=" << numResults
<< endl;
break;
}
}
// If we cache the executor later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the executor later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the executor.
exec->deregisterExec();
// Caller expects exceptions thrown in certain cases.
if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) {
error() << "Plan executor error during find: " << PlanExecutor::statestr(state)
<< ", stats: " << Explain::getWinningPlanStats(exec.get());
uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj));
}
// Before saving the cursor, ensure that whatever plan we established happened with the expected
// collection version
auto css = CollectionShardingState::get(txn, nss);
css->checkShardVersionOrThrow(txn);
// Fill out curop based on query results. If we have a cursorid, we will fill out curop with
// this cursorid later.
long long ccId = 0;
if (shouldSaveCursor(txn, collection, state, exec.get())) {
// We won't use the executor until it's getMore'd.
exec->saveState();
exec->detachFromOperationContext();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc =
new ClientCursor(collection->getCursorManager(),
exec.release(),
nss.ns(),
txn->recoveryUnit()->isReadingFromMajorityCommittedSnapshot(),
pq.getOptions(),
pq.getFilter());
ccId = cc->cursorid();
LOG(5) << "caching executor with cursorid " << ccId << " after returning " << numResults
<< " results" << endl;
// TODO document
if (pq.isOplogReplay() && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.isExhaust()) {
curop.debug().exhaust = true;
}
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
endQueryOp(txn, collection, *cc->getExecutor(), dbProfilingLevel, numResults, ccId);
} else {
LOG(5) << "Not caching executor but returning " << numResults << " results.\n";
endQueryOp(txn, collection, *exec, dbProfilingLevel, numResults, ccId);
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult::View qr = result.header().view2ptr();
qr.setCursorId(ccId);
qr.setResultFlagsToOk();
qr.msgdata().setOperation(opReply);
qr.setStartingFrom(0);
qr.setNReturned(numResults);
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? nss.ns() : "";
}
std::string runQuery(OperationContext* opCtx,
QueryMessage& q,
const NamespaceString& nss,
Message& result) {
CurOp& curOp = *CurOp::get(opCtx);
curOp.ensureStarted();
uassert(ErrorCodes::InvalidNamespace,
str::stream() << "Invalid ns [" << nss.ns() << "]",
nss.isValid());
invariant(!nss.isCommand());
// Set CurOp information.
const auto upconvertedQuery = upconvertQueryEntry(q.query, nss, q.ntoreturn, q.ntoskip);
beginQueryOp(opCtx, nss, upconvertedQuery, q.ntoreturn, q.ntoskip);
// Parse the qm into a CanonicalQuery.
const boost::intrusive_ptr<ExpressionContext> expCtx;
auto cq = uassertStatusOKWithContext(
CanonicalQuery::canonicalize(opCtx,
q,
expCtx,
ExtensionsCallbackReal(opCtx, &nss),
MatchExpressionParser::kAllowAllSpecialFeatures),
"Can't canonicalize query");
invariant(cq.get());
LOG(5) << "Running query:\n" << redact(cq->toString());
LOG(2) << "Running query: " << redact(cq->toStringShort());
// Parse, canonicalize, plan, transcribe, and get a plan executor.
AutoGetCollectionForReadCommand ctx(opCtx, nss, AutoGetCollection::ViewMode::kViewsForbidden);
Collection* const collection = ctx.getCollection();
{
const QueryRequest& qr = cq->getQueryRequest();
// Allow the query to run on secondaries if the read preference permits it. If no read
// preference was specified, allow the query to run iff slaveOk has been set.
const bool slaveOK = qr.hasReadPref()
? uassertStatusOK(ReadPreferenceSetting::fromContainingBSON(q.query))
.canRunOnSecondary()
: qr.isSlaveOk();
uassertStatusOK(
repl::ReplicationCoordinator::get(opCtx)->checkCanServeReadsFor(opCtx, nss, slaveOK));
}
// We have a parsed query. Time to get the execution plan for it.
auto exec = uassertStatusOK(getExecutorLegacyFind(opCtx, collection, nss, std::move(cq)));
const QueryRequest& qr = exec->getCanonicalQuery()->getQueryRequest();
// If it's actually an explain, do the explain and return rather than falling through
// to the normal query execution loop.
if (qr.isExplain()) {
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder explainBob;
Explain::explainStages(
exec.get(), collection, ExplainOptions::Verbosity::kExecAllPlans, &explainBob);
// Add the resulting object to the return buffer.
BSONObj explainObj = explainBob.obj();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
// Set query result fields.
QueryResult::View qr = bb.buf();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curOp.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(bb.release());
return "";
}
// Handle query option $maxTimeMS (not used with commands).
if (qr.getMaxTimeMS() > 0) {
uassert(40116,
"Illegal attempt to set operation deadline within DBDirectClient",
!opCtx->getClient()->isInDirectClient());
opCtx->setDeadlineAfterNowBy(Milliseconds{qr.getMaxTimeMS()});
}
opCtx->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(FindCommon::kInitReplyBufferSize);
bb.skip(sizeof(QueryResult::Value));
// How many results have we obtained from the executor?
int numResults = 0;
BSONObj obj;
PlanExecutor::ExecState state;
// Get summary info about which plan the executor is using.
{
stdx::lock_guard<Client> lk(*opCtx->getClient());
curOp.setPlanSummary_inlock(Explain::getPlanSummary(exec.get()));
}
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
// If we can't fit this result inside the current batch, then we stash it for later.
if (!FindCommon::haveSpaceForNext(obj, numResults, bb.len())) {
exec->enqueue(obj);
break;
}
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
if (FindCommon::enoughForFirstBatch(qr, numResults)) {
LOG(5) << "Enough for first batch, wantMore=" << qr.wantMore()
<< " ntoreturn=" << qr.getNToReturn().value_or(0)
<< " numResults=" << numResults;
break;
}
}
// Caller expects exceptions thrown in certain cases.
if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) {
error() << "Plan executor error during find: " << PlanExecutor::statestr(state)
<< ", stats: " << redact(Explain::getWinningPlanStats(exec.get()));
uassertStatusOKWithContext(WorkingSetCommon::getMemberObjectStatus(obj),
"Executor error during OP_QUERY find");
MONGO_UNREACHABLE;
}
// Before saving the cursor, ensure that whatever plan we established happened with the expected
// collection version
auto css = CollectionShardingState::get(opCtx, nss);
css->checkShardVersionOrThrow(opCtx);
// Fill out CurOp based on query results. If we have a cursorid, we will fill out CurOp with
// this cursorid later.
long long ccId = 0;
if (shouldSaveCursor(opCtx, collection, state, exec.get())) {
// We won't use the executor until it's getMore'd.
exec->saveState();
exec->detachFromOperationContext();
// Allocate a new ClientCursor and register it with the cursor manager.
ClientCursorPin pinnedCursor = collection->getCursorManager()->registerCursor(
opCtx,
{std::move(exec),
nss,
AuthorizationSession::get(opCtx->getClient())->getAuthenticatedUserNames(),
opCtx->recoveryUnit()->getReadConcernLevel(),
upconvertedQuery});
ccId = pinnedCursor.getCursor()->cursorid();
LOG(5) << "caching executor with cursorid " << ccId << " after returning " << numResults
<< " results";
// TODO document
if (qr.isExhaust()) {
curOp.debug().exhaust = true;
}
pinnedCursor.getCursor()->setPos(numResults);
// We assume that cursors created through a DBDirectClient are always used from their
// original OperationContext, so we do not need to move time to and from the cursor.
if (!opCtx->getClient()->isInDirectClient()) {
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
pinnedCursor.getCursor()->setLeftoverMaxTimeMicros(opCtx->getRemainingMaxTimeMicros());
}
endQueryOp(opCtx, collection, *pinnedCursor.getCursor()->getExecutor(), numResults, ccId);
} else {
LOG(5) << "Not caching executor but returning " << numResults << " results.";
endQueryOp(opCtx, collection, *exec, numResults, ccId);
}
// Fill out the output buffer's header.
QueryResult::View queryResultView = bb.buf();
queryResultView.setCursorId(ccId);
queryResultView.setResultFlagsToOk();
queryResultView.msgdata().setLen(bb.len());
queryResultView.msgdata().setOperation(opReply);
queryResultView.setStartingFrom(0);
queryResultView.setNReturned(numResults);
// Add the results from the query into the output buffer.
result.setData(bb.release());
// curOp.debug().exhaust is set above.
return curOp.debug().exhaust ? nss.ns() : "";
}
