StatusWith<executor::TaskExecutor::CallbackHandle> TaskExecutorMock::scheduleRemoteCommand( const executor::RemoteCommandRequest& request, const RemoteCommandCallbackFn& cb, const transport::BatonHandle& baton) { if (shouldFailScheduleRemoteCommandRequest(request)) { return Status(ErrorCodes::OperationFailed, "failed to schedule remote command"); } return getExecutor()->scheduleRemoteCommand(request, cb, baton); }
bool TransactionUnloader::RevertTransaction(Transaction* T) { bool Successful = true; if (getExecutor() && T->getModule()) { Successful = getExecutor()->unloadFromJIT(T->getModule(), T->getExeUnloadHandle()) && Successful; // Cleanup the module from unused global values. // if (T->getModule()) { // llvm::ModulePass* globalDCE = llvm::createGlobalDCEPass(); // globalDCE->runOnModule(*T->getModule()); // } Successful = unloadModule(T->getModule()) && Successful; } // Clean up the pending instantiations m_Sema->PendingInstantiations.clear(); m_Sema->PendingLocalImplicitInstantiations.clear(); DeclUnloader DeclU(m_Sema, m_CodeGen, T); Successful = unloadDeclarations(T, DeclU) && Successful; Successful = unloadDeserializedDeclarations(T, DeclU) && Successful; Successful = unloadFromPreprocessor(T, DeclU) && Successful; #ifndef NDEBUG //FIXME: Move the nested transaction marker out of the decl lists and // reenable this assertion. //size_t DeclSize = std::distance(T->decls_begin(), T->decls_end()); //if (T->getCompilationOpts().CodeGenerationForModule) // assert (!DeclSize && "No parsed decls must happen in parse for module"); #endif if (Successful) T->setState(Transaction::kRolledBack); else T->setState(Transaction::kRolledBackWithErrors); return Successful; }
TEST_F(ReporterTest, FailingToScheduleTimeoutShouldMakeReporterInactive) { class TaskExecutorWithFailureInScheduleWorkAt : public unittest::TaskExecutorProxy { public: TaskExecutorWithFailureInScheduleWorkAt(executor::TaskExecutor* executor) : unittest::TaskExecutorProxy(executor) {} virtual StatusWith<executor::TaskExecutor::CallbackHandle> scheduleWorkAt( Date_t when, CallbackFn&&) override { return Status(ErrorCodes::OperationFailed, "failed to schedule work"); } }; TaskExecutorWithFailureInScheduleWorkAt badExecutor(&getExecutor()); _executorProxy->setExecutor(&badExecutor); processNetworkResponse(BSON("ok" << 1)); _executorProxy->setExecutor(&getExecutor()); ASSERT_EQUALS(ErrorCodes::OperationFailed, reporter->join()); assertReporterDone(); }
void Bridge::callFunction( ExecutorToken executorToken, const std::string& moduleId, const std::string& methodId, const folly::dynamic& arguments, const std::string& tracingName) { if (*m_destroyed) { return; } #ifdef WITH_FBSYSTRACE int systraceCookie = m_systraceCookie++; FbSystraceAsyncFlow::begin( TRACE_TAG_REACT_CXX_BRIDGE, tracingName.c_str(), systraceCookie); #endif auto executorMessageQueueThread = getMessageQueueThread(executorToken); if (executorMessageQueueThread == nullptr) { LOG(WARNING) << "Dropping JS call for executor that has been unregistered..."; return; } std::shared_ptr<bool> isDestroyed = m_destroyed; executorMessageQueueThread->runOnQueue([=] () { #ifdef WITH_FBSYSTRACE FbSystraceAsyncFlow::end( TRACE_TAG_REACT_CXX_BRIDGE, tracingName.c_str(), systraceCookie); FbSystraceSection s(TRACE_TAG_REACT_CXX_BRIDGE, tracingName.c_str()); #endif if (*isDestroyed) { return; } JSExecutor *executor = getExecutor(executorToken); if (executor == nullptr) { LOG(WARNING) << "Dropping JS call for executor that has been unregistered..."; return; } // This is safe because we are running on the executor's thread: it won't // destruct until after it's been unregistered (which we check above) and // that will happen on this thread executor->callFunction(moduleId, methodId, arguments); }); }
TEST_F(ReporterTest, KeepAliveTimeoutFailingToScheduleRemoteCommandShouldMakeReporterInactive) { processNetworkResponse(BSON("ok" << 1)); auto until = getExecutor().now() + reporter->getKeepAliveInterval(); ASSERT_EQUALS(until, reporter->getKeepAliveTimeoutWhen_forTest()); ASSERT_TRUE(reporter->isActive()); Status expectedStatus(ErrorCodes::UnknownError, "failed to prepare update command"); prepareReplSetUpdatePositionCommandFn = [expectedStatus]() -> StatusWith<BSONObj> { return expectedStatus; }; runUntil(until); ASSERT_EQUALS(expectedStatus, reporter->join()); assertReporterDone(); }
Status getExecutor(OperationContext* txn, Collection* collection, const std::string& ns, const BSONObj& unparsedQuery, PlanExecutor** out, size_t plannerOptions) { if (!collection) { LOG(2) << "Collection " << ns << " does not exist." << " Using EOF stage: " << unparsedQuery.toString(); EOFStage* eofStage = new EOFStage(); WorkingSet* ws = new WorkingSet(); *out = new PlanExecutor(ws, eofStage, collection); return Status::OK(); } if (!CanonicalQuery::isSimpleIdQuery(unparsedQuery) || !collection->getIndexCatalog()->findIdIndex()) { const WhereCallbackReal whereCallback(collection->ns().db()); CanonicalQuery* cq; Status status = CanonicalQuery::canonicalize( collection->ns(), unparsedQuery, &cq, whereCallback); if (!status.isOK()) return status; // Takes ownership of 'cq'. return getExecutor(txn, collection, cq, out, plannerOptions); } LOG(2) << "Using idhack: " << unparsedQuery.toString(); WorkingSet* ws = new WorkingSet(); PlanStage* root = new IDHackStage(txn, collection, unparsedQuery["_id"].wrap(), ws); // Might have to filter out orphaned docs. if (plannerOptions & QueryPlannerParams::INCLUDE_SHARD_FILTER) { root = new ShardFilterStage(shardingState.getCollectionMetadata(collection->ns()), ws, root); } *out = new PlanExecutor(ws, root, collection); return Status::OK(); }
/* fetch a single object from collection ns that matches query set your db SavedContext first */ RecordId Helpers::findOne(OperationContext* opCtx, Collection* collection, const BSONObj& query, bool requireIndex) { if (!collection) return RecordId(); const ExtensionsCallbackReal extensionsCallback(opCtx, &collection->ns()); auto qr = stdx::make_unique<QueryRequest>(collection->ns()); qr->setFilter(query); const boost::intrusive_ptr<ExpressionContext> expCtx; auto statusWithCQ = CanonicalQuery::canonicalize(opCtx, std::move(qr), expCtx, extensionsCallback, MatchExpressionParser::kAllowAllSpecialFeatures & ~MatchExpressionParser::AllowedFeatures::kExpr); massert(17244, "Could not canonicalize " + query.toString(), statusWithCQ.isOK()); unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue()); size_t options = requireIndex ? QueryPlannerParams::NO_TABLE_SCAN : QueryPlannerParams::DEFAULT; auto exec = uassertStatusOK( getExecutor(opCtx, collection, std::move(cq), PlanExecutor::NO_YIELD, options)); PlanExecutor::ExecState state; BSONObj obj; RecordId loc; if (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, &loc))) { return loc; } massert(34427, "Plan executor error: " + WorkingSetCommon::toStatusString(obj), PlanExecutor::IS_EOF == state); return RecordId(); }
Status getExecutorCount(OperationContext* txn, Collection* collection, const BSONObj& query, const BSONObj& hintObj, PlanExecutor** execOut) { invariant(collection); const WhereCallbackReal whereCallback(collection->ns().db()); CanonicalQuery* cq; uassertStatusOK(CanonicalQuery::canonicalize(collection->ns().ns(), query, BSONObj(), BSONObj(), 0, 0, hintObj, &cq, whereCallback)); // Takes ownership of 'cq'. return getExecutor(txn, collection, cq, execOut, QueryPlannerParams::PRIVATE_IS_COUNT); }
Status FindCmd::explain(OperationContext* txn, const std::string& dbname, const BSONObj& cmdObj, ExplainCommon::Verbosity verbosity, BSONObjBuilder* out) const { const string fullns = parseNs(dbname, cmdObj); // Parse the command BSON to a LiteParsedQuery. LiteParsedQuery* rawLpq; bool isExplain = true; Status lpqStatus = LiteParsedQuery::make(fullns, cmdObj, isExplain, &rawLpq); if (!lpqStatus.isOK()) { return lpqStatus; } auto_ptr<LiteParsedQuery> lpq(rawLpq); const NamespaceString nss(fullns); // Finish the parsing step by using the LiteParsedQuery to create a CanonicalQuery. // This requires a lock on the collection in case we're parsing $where: where-specific // parsing code assumes we have a lock and creates execution machinery that requires it. CanonicalQuery* rawCq; WhereCallbackReal whereCallback(txn, nss.db()); Status canonStatus = CanonicalQuery::canonicalize(lpq.release(), &rawCq, whereCallback); if (!canonStatus.isOK()) { return canonStatus; } auto_ptr<CanonicalQuery> cq(rawCq); AutoGetCollectionForRead ctx(txn, nss); // The collection may be NULL. If so, getExecutor() should handle it by returning // an execution tree with an EOFStage. Collection* collection = ctx.getCollection(); // We have a parsed query. Time to get the execution plan for it. PlanExecutor* rawExec; Status execStatus = Status::OK(); if (cq->getParsed().getOptions().oplogReplay) { execStatus = getOplogStartHack(txn, collection, cq.release(), &rawExec); } else { size_t options = QueryPlannerParams::DEFAULT; if (shardingState.needCollectionMetadata(cq->getParsed().ns())) { options |= QueryPlannerParams::INCLUDE_SHARD_FILTER; } execStatus = getExecutor(txn, collection, cq.release(), PlanExecutor::YIELD_AUTO, &rawExec, options); } if (!execStatus.isOK()) { return execStatus; } scoped_ptr<PlanExecutor> exec(rawExec); // Got the execution tree. Explain it. Explain::explainStages(exec.get(), verbosity, out); return Status::OK(); }
bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) { const string ns = dbname + "." + cmdObj.firstElement().valuestr(); if (!cmdObj["start"].eoo()) { errmsg = "using deprecated 'start' argument to geoNear"; return false; } Client::ReadContext ctx(txn, ns); Database* db = ctx.ctx().db(); if ( !db ) { errmsg = "can't find ns"; return false; } Collection* collection = db->getCollection( txn, ns ); if ( !collection ) { errmsg = "can't find ns"; return false; } IndexCatalog* indexCatalog = collection->getIndexCatalog(); // cout << "raw cmd " << cmdObj.toString() << endl; // We seek to populate this. string nearFieldName; bool using2DIndex = false; if (!getFieldName(txn, collection, indexCatalog, &nearFieldName, &errmsg, &using2DIndex)) { return false; } PointWithCRS point; uassert(17304, "'near' field must be point", GeoParser::parseQueryPoint(cmdObj["near"], &point).isOK()); bool isSpherical = cmdObj["spherical"].trueValue(); if (!using2DIndex) { uassert(17301, "2dsphere index must have spherical: true", isSpherical); } // Build the $near expression for the query. BSONObjBuilder nearBob; if (isSpherical) { nearBob.append("$nearSphere", cmdObj["near"].Obj()); } else { nearBob.append("$near", cmdObj["near"].Obj()); } if (!cmdObj["maxDistance"].eoo()) { uassert(17299, "maxDistance must be a number",cmdObj["maxDistance"].isNumber()); nearBob.append("$maxDistance", cmdObj["maxDistance"].number()); } if (!cmdObj["minDistance"].eoo()) { uassert(17298, "minDistance doesn't work on 2d index", !using2DIndex); uassert(17300, "minDistance must be a number",cmdObj["minDistance"].isNumber()); nearBob.append("$minDistance", cmdObj["minDistance"].number()); } if (!cmdObj["uniqueDocs"].eoo()) { warning() << ns << ": ignoring deprecated uniqueDocs option in geoNear command"; } // And, build the full query expression. BSONObjBuilder queryBob; queryBob.append(nearFieldName, nearBob.obj()); if (!cmdObj["query"].eoo() && cmdObj["query"].isABSONObj()) { queryBob.appendElements(cmdObj["query"].Obj()); } BSONObj rewritten = queryBob.obj(); // cout << "rewritten query: " << rewritten.toString() << endl; int numWanted = 100; const char* limitName = !cmdObj["num"].eoo() ? "num" : "limit"; BSONElement eNumWanted = cmdObj[limitName]; if (!eNumWanted.eoo()) { uassert(17303, "limit must be number", eNumWanted.isNumber()); numWanted = eNumWanted.numberInt(); uassert(17302, "limit must be >=0", numWanted >= 0); } bool includeLocs = false; if (!cmdObj["includeLocs"].eoo()) { includeLocs = cmdObj["includeLocs"].trueValue(); } double distanceMultiplier = 1.0; BSONElement eDistanceMultiplier = cmdObj["distanceMultiplier"]; if (!eDistanceMultiplier.eoo()) { uassert(17296, "distanceMultiplier must be a number", eDistanceMultiplier.isNumber()); distanceMultiplier = eDistanceMultiplier.number(); uassert(17297, "distanceMultiplier must be non-negative", distanceMultiplier >= 0); } BSONObj projObj = BSON("$pt" << BSON("$meta" << LiteParsedQuery::metaGeoNearPoint) << "$dis" << BSON("$meta" << LiteParsedQuery::metaGeoNearDistance)); CanonicalQuery* cq; const NamespaceString nss(dbname); const WhereCallbackReal whereCallback(txn, nss.db()); if (!CanonicalQuery::canonicalize(ns, rewritten, BSONObj(), projObj, 0, numWanted, BSONObj(), &cq, whereCallback).isOK()) { errmsg = "Can't parse filter / create query"; return false; } PlanExecutor* rawExec; if (!getExecutor(txn, collection, cq, &rawExec, 0).isOK()) { errmsg = "can't get query runner"; return false; } auto_ptr<PlanExecutor> exec(rawExec); const ScopedExecutorRegistration safety(exec.get()); double totalDistance = 0; BSONObjBuilder resultBuilder(result.subarrayStart("results")); double farthestDist = 0; BSONObj currObj; int results = 0; while ((results < numWanted) && PlanExecutor::ADVANCED == exec->getNext(&currObj, NULL)) { // Come up with the correct distance. double dist = currObj["$dis"].number() * distanceMultiplier; totalDistance += dist; if (dist > farthestDist) { farthestDist = dist; } // Strip out '$dis' and '$pt' from the result obj. The rest gets added as 'obj' // in the command result. BSONObjIterator resIt(currObj); BSONObjBuilder resBob; while (resIt.more()) { BSONElement elt = resIt.next(); if (!mongoutils::str::equals("$pt", elt.fieldName()) && !mongoutils::str::equals("$dis", elt.fieldName())) { resBob.append(elt); } } BSONObj resObj = resBob.obj(); // Don't make a too-big result object. if (resultBuilder.len() + resObj.objsize()> BSONObjMaxUserSize) { warning() << "Too many geoNear results for query " << rewritten.toString() << ", truncating output."; break; } // Add the next result to the result builder. BSONObjBuilder oneResultBuilder( resultBuilder.subobjStart(BSONObjBuilder::numStr(results))); oneResultBuilder.append("dis", dist); if (includeLocs) { oneResultBuilder.appendAs(currObj["$pt"], "loc"); } oneResultBuilder.append("obj", resObj); oneResultBuilder.done(); ++results; } resultBuilder.done(); // Fill out the stats subobj. BSONObjBuilder stats(result.subobjStart("stats")); // Fill in nscanned from the explain. PlanSummaryStats summary; Explain::getSummaryStats(exec.get(), &summary); stats.appendNumber("nscanned", summary.totalKeysExamined); stats.appendNumber("objectsLoaded", summary.totalDocsExamined); stats.append("avgDistance", totalDistance / results); stats.append("maxDistance", farthestDist); stats.append("time", txn->getCurOp()->elapsedMillis()); stats.done(); return true; }
std::shared_ptr<IOExecutor> getIOExecutor() { return getExecutor( globalIOExecutor, globalIOThreadPool, globalIOExecutorLock); }
/* * Runs the command object cmdobj on the db with name dbname and puts result in result. * @param dbname, name of db * @param cmdobj, object that contains entire command * @param options * @param errmsg, reference to error message * @param result, reference to builder for result * @param fromRepl * @return true if successful, false otherwise */ bool FTSCommand::_run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int cmdOptions, const string& ns, const string& searchString, string language, // "" for not-set int limit, BSONObj& filter, BSONObj& projection, string& errmsg, BSONObjBuilder& result ) { Timer comm; // Rewrite the cmd as a normal query. BSONObjBuilder queryBob; queryBob.appendElements(filter); BSONObjBuilder textBob; textBob.append("$search", searchString); if (!language.empty()) { textBob.append("$language", language); } queryBob.append("$text", textBob.obj()); // This is the query we exec. BSONObj queryObj = queryBob.obj(); // We sort by the score. BSONObj sortSpec = BSON("$s" << BSON("$meta" << LiteParsedQuery::metaTextScore)); // We also project the score into the document and strip it out later during the reformatting // of the results. BSONObjBuilder projBob; projBob.appendElements(projection); projBob.appendElements(sortSpec); BSONObj projObj = projBob.obj(); AutoGetCollectionForRead ctx(txn, ns); CanonicalQuery* cq; Status canonicalizeStatus = CanonicalQuery::canonicalize(ns, queryObj, sortSpec, projObj, 0, limit, BSONObj(), &cq, WhereCallbackReal(txn, dbname)); if (!canonicalizeStatus.isOK()) { errmsg = canonicalizeStatus.reason(); return false; } PlanExecutor* rawExec; Status getExecStatus = getExecutor(txn, ctx.getCollection(), cq, &rawExec); if (!getExecStatus.isOK()) { errmsg = getExecStatus.reason(); return false; } auto_ptr<PlanExecutor> exec(rawExec); BSONArrayBuilder resultBuilder(result.subarrayStart("results")); // Quoth: "leave a mb for other things" int resultSize = 1024 * 1024; int numReturned = 0; BSONObj obj; while (PlanExecutor::ADVANCED == exec->getNext(&obj, NULL)) { if ((resultSize + obj.objsize()) >= BSONObjMaxUserSize) { break; } // We return an array of results. Add another element. BSONObjBuilder oneResultBuilder(resultBuilder.subobjStart()); oneResultBuilder.append("score", obj["$s"].number()); // Strip out the score from the returned obj. BSONObjIterator resIt(obj); BSONObjBuilder resBob; while (resIt.more()) { BSONElement elt = resIt.next(); if (!mongoutils::str::equals("$s", elt.fieldName())) { resBob.append(elt); } } oneResultBuilder.append("obj", resBob.obj()); BSONObj addedArrayObj = oneResultBuilder.done(); resultSize += addedArrayObj.objsize(); numReturned++; } resultBuilder.done(); // returns some stats to the user BSONObjBuilder stats(result.subobjStart("stats")); // Fill in nscanned from the explain. PlanSummaryStats summary; Explain::getSummaryStats(exec.get(), &summary); stats.appendNumber("nscanned", summary.totalKeysExamined); stats.appendNumber("nscannedObjects", summary.totalDocsExamined); stats.appendNumber( "n" , numReturned ); stats.append( "timeMicros", (int)comm.micros() ); stats.done(); return true; }
static bool runImpl(OperationContext* txn, const string& dbname, const string& ns, const BSONObj& query, const BSONObj& fields, const BSONObj& update, const BSONObj& sort, bool upsert, bool returnNew, bool remove , BSONObjBuilder& result, string& errmsg) { AutoGetOrCreateDb autoDb(txn, dbname, MODE_IX); Lock::CollectionLock collLock(txn->lockState(), ns, MODE_IX); Client::Context ctx(txn, ns, autoDb.getDb(), autoDb.justCreated()); if (!repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(dbname)) { return appendCommandStatus(result, Status(ErrorCodes::NotMaster, str::stream() << "Not primary while running findAndModify in " << ns)); } Collection* collection = ctx.db()->getCollection(ns); const WhereCallbackReal whereCallback(txn, StringData(ns)); if ( !collection ) { if ( !upsert ) { // no collectio and no upsert, so can't possible do anything _appendHelper( result, BSONObj(), false, fields, whereCallback ); return true; } // no collection, but upsert, so we want to create it // problem is we only have IX on db and collection :( // so we tell our caller who can do it errmsg = "no-collection"; return false; } Snapshotted<BSONObj> snapshotDoc; RecordId loc; bool found = false; { CanonicalQuery* cq; const BSONObj projection; const long long skip = 0; const long long limit = -1; // 1 document requested; negative indicates hard limit. uassertStatusOK(CanonicalQuery::canonicalize(ns, query, sort, projection, skip, limit, &cq, whereCallback)); PlanExecutor* rawExec; uassertStatusOK(getExecutor(txn, collection, cq, PlanExecutor::YIELD_AUTO, &rawExec, QueryPlannerParams::DEFAULT)); scoped_ptr<PlanExecutor> exec(rawExec); PlanExecutor::ExecState state = exec->getNextSnapshotted(&snapshotDoc, &loc); if (PlanExecutor::ADVANCED == state) { found = true; } else if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { if (PlanExecutor::FAILURE == state && WorkingSetCommon::isValidStatusMemberObject(snapshotDoc.value())) { const Status errorStatus = WorkingSetCommon::getMemberObjectStatus(snapshotDoc.value()); invariant(!errorStatus.isOK()); uasserted(errorStatus.code(), errorStatus.reason()); } uasserted(ErrorCodes::OperationFailed, str::stream() << "executor returned " << PlanExecutor::statestr(state) << " while finding document to update"); } else { invariant(PlanExecutor::IS_EOF == state); } } WriteUnitOfWork wuow(txn); if (found) { // We found a doc, but it might not be associated with the active snapshot. // If the doc has changed or is no longer in the collection, we will throw a // write conflict exception and start again from the beginning. if (txn->recoveryUnit()->getSnapshotId() != snapshotDoc.snapshotId()) { BSONObj oldObj = snapshotDoc.value(); if (!collection->findDoc(txn, loc, &snapshotDoc)) { // Got deleted in the new snapshot. throw WriteConflictException(); } if (!oldObj.binaryEqual(snapshotDoc.value())) { // Got updated in the new snapshot. throw WriteConflictException(); } } // If we get here without throwing, then we should have the copy of the doc from // the latest snapshot. invariant(txn->recoveryUnit()->getSnapshotId() == snapshotDoc.snapshotId()); } BSONObj doc = snapshotDoc.value(); BSONObj queryModified = query; if (found && !doc["_id"].eoo() && !CanonicalQuery::isSimpleIdQuery(query)) { // we're going to re-write the query to be more efficient // we have to be a little careful because of positional operators // maybe we can pass this all through eventually, but right now isn't an easy way bool hasPositionalUpdate = false; { // if the update has a positional piece ($) // then we need to pull all query parts in // so here we check for $ // a little hacky BSONObjIterator i( update ); while ( i.more() ) { const BSONElement& elem = i.next(); if ( elem.fieldName()[0] != '$' || elem.type() != Object ) continue; BSONObjIterator j( elem.Obj() ); while ( j.more() ) { if ( str::contains( j.next().fieldName(), ".$" ) ) { hasPositionalUpdate = true; break; } } } } BSONObjBuilder b(query.objsize() + 10); b.append( doc["_id"] ); bool addedAtomic = false; BSONObjIterator i(query); while ( i.more() ) { const BSONElement& elem = i.next(); if ( str::equals( "_id" , elem.fieldName() ) ) { // we already do _id continue; } if ( ! hasPositionalUpdate ) { // if there is a dotted field, accept we may need more query parts continue; } if ( ! addedAtomic ) { b.appendBool( "$atomic" , true ); addedAtomic = true; } b.append( elem ); } queryModified = b.obj(); } if ( remove ) { _appendHelper(result, doc, found, fields, whereCallback); if ( found ) { deleteObjects(txn, ctx.db(), ns, queryModified, PlanExecutor::YIELD_MANUAL, true, true); BSONObjBuilder le( result.subobjStart( "lastErrorObject" ) ); le.appendNumber( "n" , 1 ); le.done(); } } else { // update if ( ! found && ! upsert ) { // didn't have it, and am not upserting _appendHelper(result, doc, found, fields, whereCallback); } else { // we found it or we're updating if ( ! returnNew ) { _appendHelper(result, doc, found, fields, whereCallback); } const NamespaceString requestNs(ns); UpdateRequest request(requestNs); request.setQuery(queryModified); request.setUpdates(update); request.setUpsert(upsert); request.setUpdateOpLog(); request.setStoreResultDoc(returnNew); request.setYieldPolicy(PlanExecutor::YIELD_MANUAL); // TODO(greg) We need to send if we are ignoring // the shard version below, but for now no UpdateLifecycleImpl updateLifecycle(false, requestNs); request.setLifecycle(&updateLifecycle); UpdateResult res = mongo::update(txn, ctx.db(), request, &txn->getCurOp()->debug()); if (!found && res.existing) { // No match was found during the read part of this find and modify, which // means that we're here doing an upsert. But the update also told us that // we modified an *already existing* document. This probably means that // the query reported EOF based on an out-of-date snapshot. This should be // a rare event, so we handle it by throwing a write conflict. throw WriteConflictException(); } if ( !collection ) { // collection created by an upsert collection = ctx.db()->getCollection(ns); } LOG(3) << "update result: " << res ; if (returnNew) { dassert(!res.newObj.isEmpty()); _appendHelper(result, res.newObj, true, fields, whereCallback); } BSONObjBuilder le( result.subobjStart( "lastErrorObject" ) ); le.appendBool( "updatedExisting" , res.existing ); le.appendNumber( "n" , res.numMatched ); if ( !res.upserted.isEmpty() ) { le.append( res.upserted[kUpsertedFieldName] ); } le.done(); } } // Committing the WUOW can close the current snapshot. Until this happens, the // snapshot id should not have changed. if (found) { invariant(txn->recoveryUnit()->getSnapshotId() == snapshotDoc.snapshotId()); } wuow.commit(); return true; }
bool run(OperationContext* txn, const string& dbname, BSONObj& jsobj, int, string& errmsg, BSONObjBuilder& result) { const std::string ns = parseNs(dbname, jsobj); md5digest d; md5_state_t st; md5_init(&st); int n = 0; bool partialOk = jsobj["partialOk"].trueValue(); if (partialOk) { // WARNING: This code depends on the binary layout of md5_state. It will not be // compatible with different md5 libraries or work correctly in an environment with // mongod's of different endians. It is ok for mongos to be a different endian since // it just passes the buffer through to another mongod. BSONElement stateElem = jsobj["md5state"]; if (!stateElem.eoo()) { int len; const char* data = stateElem.binDataClean(len); massert(16247, "md5 state not correct size", len == sizeof(st)); memcpy(&st, data, sizeof(st)); } n = jsobj["startAt"].numberInt(); } BSONObj query = BSON("files_id" << jsobj["filemd5"] << "n" << GTE << n); BSONObj sort = BSON("files_id" << 1 << "n" << 1); MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { CanonicalQuery* cq; if (!CanonicalQuery::canonicalize(ns, query, sort, BSONObj(), &cq).isOK()) { uasserted(17240, "Can't canonicalize query " + query.toString()); return 0; } // Check shard version at startup. // This will throw before we've done any work if shard version is outdated // We drop and re-acquire these locks every document because md5'ing is expensive unique_ptr<AutoGetCollectionForRead> ctx(new AutoGetCollectionForRead(txn, ns)); Collection* coll = ctx->getCollection(); PlanExecutor* rawExec; if (!getExecutor(txn, coll, cq, PlanExecutor::YIELD_MANUAL, &rawExec, QueryPlannerParams::NO_TABLE_SCAN).isOK()) { uasserted(17241, "Can't get executor for query " + query.toString()); return 0; } unique_ptr<PlanExecutor> exec(rawExec); // Process notifications when the lock is released/reacquired in the loop below exec->registerExec(); BSONObj obj; PlanExecutor::ExecState state; while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) { BSONElement ne = obj["n"]; verify(ne.isNumber()); int myn = ne.numberInt(); if (n != myn) { if (partialOk) { break; // skipped chunk is probably on another shard } log() << "should have chunk: " << n << " have:" << myn << endl; dumpChunks(txn, ns, query, sort); uassert(10040, "chunks out of order", n == myn); } // make a copy of obj since we access data in it while yielding locks BSONObj owned = obj.getOwned(); exec->saveState(); // UNLOCKED ctx.reset(); int len; const char* data = owned["data"].binDataClean(len); // This is potentially an expensive operation, so do it out of the lock md5_append(&st, (const md5_byte_t*)(data), len); n++; try { // RELOCKED ctx.reset(new AutoGetCollectionForRead(txn, ns)); } catch (const SendStaleConfigException& ex) { LOG(1) << "chunk metadata changed during filemd5, will retarget and continue"; break; } // Have the lock again. See if we were killed. if (!exec->restoreState(txn)) { if (!partialOk) { uasserted(13281, "File deleted during filemd5 command"); } } } if (partialOk) result.appendBinData("md5state", sizeof(st), BinDataGeneral, &st); // This must be *after* the capture of md5state since it mutates st md5_finish(&st, d); result.append("numChunks", n); result.append("md5", digestToString(d)); } MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "filemd5", dbname); return true; }
bool group( OperationContext* txn, Database* db, const std::string& ns, const BSONObj& query, BSONObj keyPattern, const std::string& keyFunctionCode, const std::string& reduceCode, const char * reduceScope, BSONObj initial, const std::string& finalize, string& errmsg, BSONObjBuilder& result ) { const string userToken = ClientBasic::getCurrent()->getAuthorizationSession() ->getAuthenticatedUserNamesToken(); auto_ptr<Scope> s = globalScriptEngine->getPooledScope(db->name(), "group" + userToken); if ( reduceScope ) s->init( reduceScope ); s->setObject( "$initial" , initial , true ); s->exec( "$reduce = " + reduceCode , "$group reduce setup" , false , true , true , 100 ); s->exec( "$arr = [];" , "$group reduce setup 2" , false , true , true , 100 ); ScriptingFunction f = s->createFunction( "function(){ " " if ( $arr[n] == null ){ " " next = {}; " " Object.extend( next , $key ); " " Object.extend( next , $initial , true ); " " $arr[n] = next; " " next = null; " " } " " $reduce( obj , $arr[n] ); " "}" ); ScriptingFunction keyFunction = 0; if ( keyFunctionCode.size() ) { keyFunction = s->createFunction( keyFunctionCode.c_str() ); } double keysize = keyPattern.objsize() * 3; double keynum = 1; Collection* collection = db->getCollection( txn, ns ); const WhereCallbackReal whereCallback(txn, StringData(db->name())); map<BSONObj,int,BSONObjCmp> map; list<BSONObj> blah; if (collection) { CanonicalQuery* cq; if (!CanonicalQuery::canonicalize(ns, query, &cq, whereCallback).isOK()) { uasserted(17212, "Can't canonicalize query " + query.toString()); return 0; } PlanExecutor* rawExec; if (!getExecutor(txn,collection, cq, &rawExec).isOK()) { uasserted(17213, "Can't get executor for query " + query.toString()); return 0; } auto_ptr<PlanExecutor> exec(rawExec); const ScopedExecutorRegistration safety(exec.get()); BSONObj obj; PlanExecutor::ExecState state; while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) { BSONObj key = getKey(obj , keyPattern , keyFunction , keysize / keynum, s.get() ); keysize += key.objsize(); keynum++; int& n = map[key]; if ( n == 0 ) { n = map.size(); s->setObject( "$key" , key , true ); uassert(17203, "group() can't handle more than 20000 unique keys", n <= 20000 ); } s->setObject( "obj" , obj , true ); s->setNumber( "n" , n - 1 ); if ( s->invoke( f , 0, 0 , 0 , true ) ) { throw UserException(17214, (string)"reduce invoke failed: " + s->getError()); } } } if (!finalize.empty()) { s->exec( "$finalize = " + finalize , "$group finalize define" , false , true , true , 100 ); ScriptingFunction g = s->createFunction( "function(){ " " for(var i=0; i < $arr.length; i++){ " " var ret = $finalize($arr[i]); " " if (ret !== undefined) " " $arr[i] = ret; " " } " "}" ); s->invoke( g , 0, 0 , 0 , true ); } result.appendArray( "retval" , s->getObject( "$arr" ) ); result.append( "count" , keynum - 1 ); result.append( "keys" , (int)(map.size()) ); s->exec( "$arr = [];" , "$group reduce setup 2" , false , true , true , 100 ); s->gc(); return true; }
Status getOplogStartHack(OperationContext* txn, Collection* collection, CanonicalQuery* cq, PlanExecutor** execOut) { invariant(cq); auto_ptr<CanonicalQuery> autoCq(cq); if ( collection == NULL ) return Status(ErrorCodes::InternalError, "getOplogStartHack called with a NULL collection" ); // A query can only do oplog start finding if it has a top-level $gt or $gte predicate over // the "ts" field (the operation's timestamp). Find that predicate and pass it to // the OplogStart stage. MatchExpression* tsExpr = NULL; if (MatchExpression::AND == cq->root()->matchType()) { // The query has an AND at the top-level. See if any of the children // of the AND are $gt or $gte predicates over 'ts'. for (size_t i = 0; i < cq->root()->numChildren(); ++i) { MatchExpression* me = cq->root()->getChild(i); if (isOplogTsPred(me)) { tsExpr = me; break; } } } else if (isOplogTsPred(cq->root())) { // The root of the tree is a $gt or $gte predicate over 'ts'. tsExpr = cq->root(); } if (NULL == tsExpr) { return Status(ErrorCodes::OplogOperationUnsupported, "OplogReplay query does not contain top-level " "$gt or $gte over the 'ts' field."); } DiskLoc startLoc = DiskLoc().setInvalid(); // See if the RecordStore supports the oplogStartHack const BSONElement tsElem = extractOplogTsOptime(tsExpr); if (tsElem.type() == Timestamp) { StatusWith<DiskLoc> goal = oploghack::keyForOptime(tsElem._opTime()); if (goal.isOK()) { startLoc = collection->getRecordStore()->oplogStartHack(txn, goal.getValue()); } } if (startLoc.isValid()) { LOG(3) << "Using direct oplog seek"; } else { LOG(3) << "Using OplogStart stage"; // Fallback to trying the OplogStart stage. WorkingSet* oplogws = new WorkingSet(); OplogStart* stage = new OplogStart(txn, collection, tsExpr, oplogws); PlanExecutor* rawExec; // Takes ownership of oplogws and stage. Status execStatus = PlanExecutor::make(txn, oplogws, stage, collection, PlanExecutor::YIELD_AUTO, &rawExec); invariant(execStatus.isOK()); scoped_ptr<PlanExecutor> exec(rawExec); // The stage returns a DiskLoc of where to start. PlanExecutor::ExecState state = exec->getNext(NULL, &startLoc); // This is normal. The start of the oplog is the beginning of the collection. if (PlanExecutor::IS_EOF == state) { return getExecutor(txn, collection, autoCq.release(), PlanExecutor::YIELD_AUTO, execOut); } // This is not normal. An error was encountered. if (PlanExecutor::ADVANCED != state) { return Status(ErrorCodes::InternalError, "quick oplog start location had error...?"); } } // cout << "diskloc is " << startLoc.toString() << endl; // Build our collection scan... CollectionScanParams params; params.collection = collection; params.start = startLoc; params.direction = CollectionScanParams::FORWARD; params.tailable = cq->getParsed().getOptions().tailable; WorkingSet* ws = new WorkingSet(); CollectionScan* cs = new CollectionScan(txn, params, ws, cq->root()); // Takes ownership of 'ws', 'cs', and 'cq'. return PlanExecutor::make(txn, ws, cs, autoCq.release(), collection, PlanExecutor::YIELD_AUTO, execOut); }
std::string newRunQuery(OperationContext* txn, Message& m, QueryMessage& q, CurOp& curop, Message &result, bool fromDBDirectClient) { // Validate the namespace. const char *ns = q.ns; uassert(16332, "can't have an empty ns", ns[0]); const NamespaceString nsString(ns); uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid()); // Set curop information. curop.debug().ns = ns; curop.debug().ntoreturn = q.ntoreturn; curop.debug().query = q.query; curop.setQuery(q.query); // If the query is really a command, run it. if (nsString.isCommand()) { int nToReturn = q.ntoreturn; uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn << ") for $cmd type ns - can only be 1 or -1", nToReturn == 1 || nToReturn == -1); curop.markCommand(); BufBuilder bb; bb.skip(sizeof(QueryResult::Value)); BSONObjBuilder cmdResBuf; if (!runCommands(txn, ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) { uasserted(13530, "bad or malformed command request?"); } curop.debug().iscommand = true; // TODO: Does this get overwritten/do we really need to set this twice? curop.debug().query = q.query; QueryResult::View qr = bb.buf(); bb.decouple(); qr.setResultFlagsToOk(); qr.msgdata().setLen(bb.len()); curop.debug().responseLength = bb.len(); qr.msgdata().setOperation(opReply); qr.setCursorId(0); qr.setStartingFrom(0); qr.setNReturned(1); result.setData(qr.view2ptr(), true); return ""; } const NamespaceString nss(q.ns); // Parse the qm into a CanonicalQuery. CanonicalQuery* cq; Status canonStatus = CanonicalQuery::canonicalize( q, &cq, WhereCallbackReal(txn, StringData(nss.db()))); if (!canonStatus.isOK()) { uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString()); } QLOG() << "Running query:\n" << cq->toString(); LOG(2) << "Running query: " << cq->toStringShort(); // Parse, canonicalize, plan, transcribe, and get a plan executor. PlanExecutor* rawExec = NULL; // We use this a lot below. const LiteParsedQuery& pq = cq->getParsed(); AutoGetCollectionForRead ctx(txn, nss); const int dbProfilingLevel = (ctx.getDb() != NULL) ? ctx.getDb()->getProfilingLevel() : serverGlobalParams.defaultProfile; Collection* collection = ctx.getCollection(); // We'll now try to get the query executor that will execute this query for us. There // are a few cases in which we know upfront which executor we should get and, therefore, // we shortcut the selection process here. // // (a) If the query is over a collection that doesn't exist, we use an EOFStage. // // (b) if the query is a replication's initial sync one, we use a specifically designed // stage that skips extents faster (see details in exec/oplogstart.h). // // Otherwise we go through the selection of which executor is most suited to the // query + run-time context at hand. Status status = Status::OK(); if (NULL != collection && pq.getOptions().oplogReplay) { // Takes ownership of 'cq'. status = getOplogStartHack(txn, collection, cq, &rawExec); } else { size_t options = QueryPlannerParams::DEFAULT; if (shardingState.needCollectionMetadata(pq.ns())) { options |= QueryPlannerParams::INCLUDE_SHARD_FILTER; } // Takes ownership of 'cq'. status = getExecutor(txn, collection, cq, PlanExecutor::YIELD_AUTO, &rawExec, options); } if (!status.isOK()) { // NOTE: Do not access cq as getExecutor has deleted it. uasserted(17007, "Unable to execute query: " + status.reason()); } verify(NULL != rawExec); auto_ptr<PlanExecutor> exec(rawExec); // If it's actually an explain, do the explain and return rather than falling through // to the normal query execution loop. if (pq.isExplain()) { BufBuilder bb; bb.skip(sizeof(QueryResult::Value)); BSONObjBuilder explainBob; Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob); // Add the resulting object to the return buffer. BSONObj explainObj = explainBob.obj(); bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize()); curop.debug().iscommand = true; // TODO: Does this get overwritten/do we really need to set this twice? curop.debug().query = q.query; // Set query result fields. QueryResult::View qr = bb.buf(); bb.decouple(); qr.setResultFlagsToOk(); qr.msgdata().setLen(bb.len()); curop.debug().responseLength = bb.len(); qr.msgdata().setOperation(opReply); qr.setCursorId(0); qr.setStartingFrom(0); qr.setNReturned(1); result.setData(qr.view2ptr(), true); return ""; } // We freak out later if this changes before we're done with the query. const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns()); // Handle query option $maxTimeMS (not used with commands). curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000); txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. // uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set. bool slaveOK = pq.getOptions().slaveOk || pq.hasReadPref(); status = repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor( txn, NamespaceString(cq->ns()), slaveOK); uassertStatusOK(status); // If this exists, the collection is sharded. // If it doesn't exist, we can assume we're not sharded. // If we're sharded, we might encounter data that is not consistent with our sharding state. // We must ignore this data. CollectionMetadataPtr collMetadata; if (!shardingState.needCollectionMetadata(pq.ns())) { collMetadata = CollectionMetadataPtr(); } else { collMetadata = shardingState.getCollectionMetadata(pq.ns()); } // Run the query. // bb is used to hold query results // this buffer should contain either requested documents per query or // explain information, but not both BufBuilder bb(32768); bb.skip(sizeof(QueryResult::Value)); // How many results have we obtained from the executor? int numResults = 0; // If we're replaying the oplog, we save the last time that we read. OpTime slaveReadTill; // Do we save the PlanExecutor in a ClientCursor for getMore calls later? bool saveClientCursor = false; BSONObj obj; PlanExecutor::ExecState state; // uint64_t numMisplacedDocs = 0; // Get summary info about which plan the executor is using. curop.debug().planSummary = Explain::getPlanSummary(exec.get()); while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) { // Add result to output buffer. bb.appendBuf((void*)obj.objdata(), obj.objsize()); // Count the result. ++numResults; // Possibly note slave's position in the oplog. if (pq.getOptions().oplogReplay) { BSONElement e = obj["ts"]; if (Date == e.type() || Timestamp == e.type()) { slaveReadTill = e._opTime(); } } // TODO: only one type of 2d search doesn't support this. We need a way to pull it out // of CanonicalQuery. :( const bool supportsGetMore = true; if (!supportsGetMore && (enough(pq, numResults) || bb.len() >= MaxBytesToReturnToClientAtOnce)) { break; } else if (enoughForFirstBatch(pq, numResults, bb.len())) { QLOG() << "Enough for first batch, wantMore=" << pq.wantMore() << " numToReturn=" << pq.getNumToReturn() << " numResults=" << numResults << endl; // If only one result requested assume it's a findOne() and don't save the cursor. if (pq.wantMore() && 1 != pq.getNumToReturn()) { QLOG() << " executor EOF=" << exec->isEOF() << endl; saveClientCursor = !exec->isEOF(); } break; } } // If we cache the executor later, we want to deregister it as it receives notifications // anyway by virtue of being cached. // // If we don't cache the executor later, we are deleting it, so it must be deregistered. // // So, no matter what, deregister the executor. exec->deregisterExec(); // Caller expects exceptions thrown in certain cases. if (PlanExecutor::EXEC_ERROR == state) { scoped_ptr<PlanStageStats> stats(exec->getStats()); error() << "Plan executor error, stats: " << Explain::statsToBSON(*stats); uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj)); } // Why save a dead executor? if (PlanExecutor::DEAD == state) { saveClientCursor = false; } else if (pq.getOptions().tailable) { // If we're tailing a capped collection, we don't bother saving the cursor if the // collection is empty. Otherwise, the semantics of the tailable cursor is that the // client will keep trying to read from it. So we'll keep it around. if (collection && collection->numRecords(txn) != 0 && pq.getNumToReturn() != 1) { saveClientCursor = true; } } // TODO(greg): This will go away soon. if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) { // if the version changed during the query we might be missing some data and its safe to // send this as mongos can resend at this point throw SendStaleConfigException(pq.ns(), "version changed during initial query", shardingVersionAtStart, shardingState.getVersion(pq.ns())); } const logger::LogComponent queryLogComponent = logger::LogComponent::kQuery; const logger::LogSeverity logLevelOne = logger::LogSeverity::Debug(1); PlanSummaryStats summaryStats; Explain::getSummaryStats(exec.get(), &summaryStats); curop.debug().ntoskip = pq.getSkip(); curop.debug().nreturned = numResults; curop.debug().scanAndOrder = summaryStats.hasSortStage; curop.debug().nscanned = summaryStats.totalKeysExamined; curop.debug().nscannedObjects = summaryStats.totalDocsExamined; curop.debug().idhack = summaryStats.isIdhack; // Set debug information for consumption by the profiler. if (dbProfilingLevel > 0 || curop.elapsedMillis() > serverGlobalParams.slowMS || logger::globalLogDomain()->shouldLog(queryLogComponent, logLevelOne)) { // Get BSON stats. scoped_ptr<PlanStageStats> execStats(exec->getStats()); BSONObjBuilder statsBob; Explain::statsToBSON(*execStats, &statsBob); curop.debug().execStats.set(statsBob.obj()); // Replace exec stats with plan summary if stats cannot fit into CachedBSONObj. if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) { BSONObjBuilder bob; bob.append("summary", curop.debug().planSummary.toString()); curop.debug().execStats.set(bob.done()); } } long long ccId = 0; if (saveClientCursor) { // We won't use the executor until it's getMore'd. exec->saveState(); // Allocate a new ClientCursor. We don't have to worry about leaking it as it's // inserted into a global map by its ctor. ClientCursor* cc = new ClientCursor(collection, exec.get(), cq->getParsed().getOptions().toInt(), cq->getParsed().getFilter()); ccId = cc->cursorid(); if (fromDBDirectClient) { cc->setUnownedRecoveryUnit(txn->recoveryUnit()); } else if (state == PlanExecutor::IS_EOF && pq.getOptions().tailable) { // Don't stash the RU for tailable cursors at EOF, let them get a new RU on their // next getMore. } else { // We stash away the RecoveryUnit in the ClientCursor. It's used for subsequent // getMore requests. The calling OpCtx gets a fresh RecoveryUnit. cc->setOwnedRecoveryUnit(txn->releaseRecoveryUnit()); StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine(); txn->setRecoveryUnit(storageEngine->newRecoveryUnit(txn)); } QLOG() << "caching executor with cursorid " << ccId << " after returning " << numResults << " results" << endl; // ClientCursor takes ownership of executor. Release to make sure it's not deleted. exec.release(); // TODO document if (pq.getOptions().oplogReplay && !slaveReadTill.isNull()) { cc->slaveReadTill(slaveReadTill); } // TODO document if (pq.getOptions().exhaust) { curop.debug().exhaust = true; } // Set attributes for getMore. cc->setCollMetadata(collMetadata); cc->setPos(numResults); // If the query had a time limit, remaining time is "rolled over" to the cursor (for // use by future getmore ops). cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros()); } else { QLOG() << "Not caching executor but returning " << numResults << " results.\n"; } // Add the results from the query into the output buffer. result.appendData(bb.buf(), bb.len()); bb.decouple(); // Fill out the output buffer's header. QueryResult::View qr = result.header().view2ptr(); qr.setCursorId(ccId); curop.debug().cursorid = (0 == ccId ? -1 : ccId); qr.setResultFlagsToOk(); qr.msgdata().setOperation(opReply); qr.setStartingFrom(0); qr.setNReturned(numResults); // curop.debug().exhaust is set above. return curop.debug().exhaust ? pq.ns() : ""; }
Status getExecutorDistinct(OperationContext* txn, Collection* collection, const BSONObj& query, const std::string& field, PlanExecutor** out) { // This should'a been checked by the distinct command. invariant(collection); // TODO: check for idhack here? // When can we do a fast distinct hack? // 1. There is a plan with just one leaf and that leaf is an ixscan. // 2. The ixscan indexes the field we're interested in. // 2a: We are correct if the index contains the field but for now we look for prefix. // 3. The query is covered/no fetch. // // We go through normal planning (with limited parameters) to see if we can produce // a soln with the above properties. QueryPlannerParams plannerParams; plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN; IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false); while (ii.more()) { const IndexDescriptor* desc = ii.next(); // The distinct hack can work if any field is in the index but it's not always clear // if it's a win unless it's the first field. if (desc->keyPattern().firstElement().fieldName() == field) { plannerParams.indices.push_back(IndexEntry(desc->keyPattern(), desc->getAccessMethodName(), desc->isMultikey(), desc->isSparse(), desc->indexName(), desc->infoObj())); } } const WhereCallbackReal whereCallback(collection->ns().db()); // If there are no suitable indices for the distinct hack bail out now into regular planning // with no projection. if (plannerParams.indices.empty()) { CanonicalQuery* cq; Status status = CanonicalQuery::canonicalize( collection->ns().ns(), query, &cq, whereCallback); if (!status.isOK()) { return status; } // Takes ownership of 'cq'. return getExecutor(txn, collection, cq, out); } // // If we're here, we have an index prefixed by the field we're distinct-ing over. // // Applying a projection allows the planner to try to give us covered plans that we can turn // into the projection hack. getDistinctProjection deals with .find() projection semantics // (ie _id:1 being implied by default). BSONObj projection = getDistinctProjection(field); // Apply a projection of the key. Empty BSONObj() is for the sort. CanonicalQuery* cq; Status status = CanonicalQuery::canonicalize(collection->ns().ns(), query, BSONObj(), projection, &cq, whereCallback); if (!status.isOK()) { return status; } auto_ptr<CanonicalQuery> autoCq(cq); // If there's no query, we can just distinct-scan one of the indices. // Not every index in plannerParams.indices may be suitable. Refer to // getDistinctNodeIndex(). size_t distinctNodeIndex = 0; if (query.isEmpty() && getDistinctNodeIndex(plannerParams.indices, field, &distinctNodeIndex)) { DistinctNode* dn = new DistinctNode(); dn->indexKeyPattern = plannerParams.indices[distinctNodeIndex].keyPattern; dn->direction = 1; IndexBoundsBuilder::allValuesBounds(dn->indexKeyPattern, &dn->bounds); dn->fieldNo = 0; QueryPlannerParams params; // Takes ownership of 'dn'. QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*cq, params, dn); invariant(soln); LOG(2) << "Using fast distinct: " << cq->toStringShort() << ", planSummary: " << getPlanSummary(*soln); WorkingSet* ws = new WorkingSet(); PlanStage* root; verify(StageBuilder::build(txn, collection, *soln, ws, &root)); // Takes ownership of its arguments (except for 'collection'). *out = new PlanExecutor(ws, root, soln, autoCq.release(), collection); return Status::OK(); } // See if we can answer the query in a fast-distinct compatible fashion. vector<QuerySolution*> solutions; status = QueryPlanner::plan(*cq, plannerParams, &solutions); if (!status.isOK()) { return getExecutor(txn, collection, autoCq.release(), out); } // We look for a solution that has an ixscan we can turn into a distinctixscan for (size_t i = 0; i < solutions.size(); ++i) { if (turnIxscanIntoDistinctIxscan(solutions[i], field)) { // Great, we can use solutions[i]. Clean up the other QuerySolution(s). for (size_t j = 0; j < solutions.size(); ++j) { if (j != i) { delete solutions[j]; } } LOG(2) << "Using fast distinct: " << cq->toStringShort() << ", planSummary: " << getPlanSummary(*solutions[i]); // Build and return the SSR over solutions[i]. WorkingSet* ws = new WorkingSet(); PlanStage* root; verify(StageBuilder::build(txn, collection, *solutions[i], ws, &root)); // Takes ownership of its arguments (except for 'collection'). *out = new PlanExecutor(ws, root, solutions[i], autoCq.release(), collection); return Status::OK(); } } // If we're here, the planner made a soln with the restricted index set but we couldn't // translate any of them into a distinct-compatible soln. So, delete the solutions and just // go through normal planning. for (size_t i = 0; i < solutions.size(); ++i) { delete solutions[i]; } // We drop the projection from the 'cq'. Unfortunately this is not trivial. status = CanonicalQuery::canonicalize(collection->ns().ns(), query, &cq, whereCallback); if (!status.isOK()) { return status; } autoCq.reset(cq); // Takes ownership of 'autoCq'. return getExecutor(txn, collection, autoCq.release(), out); }
std::shared_ptr<Executor> getCPUExecutor() { return getExecutor( globalCPUExecutor, globalInlineExecutor, globalCPUExecutorLock); }
void Mechanoid::enterBuilding(detail::MapBuilding *bld) { if (!bld) return; auto mmb = bld->getModificationMapBuilding(); if (!mmb) return; building = mmb; if (!isPlayer()) { // handle bot's building visit return; } // player is now officially in the building // we need to run scripts, setup texts etc. auto e = getEngine(); // set building, mechanoid auto bm = e->getBuildingMenu(); bm->setCurrentBuilding(mmb); bm->setCurrentMechanoid(this); bm->clearText(); // now run scripts auto se = mmb->map->modification->getScriptEngine(); path script_file = path("maps") / mmb->map->script_dir.toString() / mmb->script_name.toString(); auto s = se->getScript(script_file.string()); // set player visit auto iter = player->buildings.find_if([mmb](const auto &vb) { return vb->building.get() == mmb; }); if (iter == player->buildings.end()) { auto vb = e->getStorage()->modificationPlayerBuildings.createAtEnd(); vb->player = player; vb->building = mmb; vb->know_location = true; vb->visited = true; player->buildings.insert(vb); } else { (*iter)->know_location = true; (*iter)->visited = true; } // set script data s->data.script = s; s->data.player = player; s->data.building = mmb; // set script for possible callbacks from building menu bm->setCurrentScriptCallback([s](const auto &fn, const auto ¶ms) mutable { s->call(fn, params); }); // register available building quests s->RegisterQuests(); // main script call s->OnEnterBuilding(); // update building menu bm->refresh(); e->ShowBuildingMenu(); // workaround { static Executor e; getExecutor(&e); } // do async save to not freeze the game // the ideal algorithm here is: // 1. freeze the game // 2. async({ save(); unfreeze(); }); getExecutor().push([e]() { e->saveAuto(); }); }