StatusWith<executor::TaskExecutor::CallbackHandle> TaskExecutorMock::scheduleRemoteCommand(
const executor::RemoteCommandRequest& request,
const RemoteCommandCallbackFn& cb,
const transport::BatonHandle& baton) {
if (shouldFailScheduleRemoteCommandRequest(request)) {
return Status(ErrorCodes::OperationFailed, "failed to schedule remote command");
}
return getExecutor()->scheduleRemoteCommand(request, cb, baton);
}
bool TransactionUnloader::RevertTransaction(Transaction* T) {
bool Successful = true;
if (getExecutor() && T->getModule()) {
Successful = getExecutor()->unloadFromJIT(T->getModule(),
T->getExeUnloadHandle())
&& Successful;
// Cleanup the module from unused global values.
// if (T->getModule()) {
// llvm::ModulePass* globalDCE = llvm::createGlobalDCEPass();
// globalDCE->runOnModule(*T->getModule());
// }
Successful = unloadModule(T->getModule()) && Successful;
}
// Clean up the pending instantiations
m_Sema->PendingInstantiations.clear();
m_Sema->PendingLocalImplicitInstantiations.clear();
DeclUnloader DeclU(m_Sema, m_CodeGen, T);
Successful = unloadDeclarations(T, DeclU) && Successful;
Successful = unloadDeserializedDeclarations(T, DeclU) && Successful;
Successful = unloadFromPreprocessor(T, DeclU) && Successful;
#ifndef NDEBUG
//FIXME: Move the nested transaction marker out of the decl lists and
// reenable this assertion.
//size_t DeclSize = std::distance(T->decls_begin(), T->decls_end());
//if (T->getCompilationOpts().CodeGenerationForModule)
// assert (!DeclSize && "No parsed decls must happen in parse for module");
#endif
if (Successful)
T->setState(Transaction::kRolledBack);
else
T->setState(Transaction::kRolledBackWithErrors);
return Successful;
}
TEST_F(ReporterTest, FailingToScheduleTimeoutShouldMakeReporterInactive) {
class TaskExecutorWithFailureInScheduleWorkAt : public unittest::TaskExecutorProxy {
public:
TaskExecutorWithFailureInScheduleWorkAt(executor::TaskExecutor* executor)
: unittest::TaskExecutorProxy(executor) {}
virtual StatusWith<executor::TaskExecutor::CallbackHandle> scheduleWorkAt(
Date_t when, CallbackFn&&) override {
return Status(ErrorCodes::OperationFailed, "failed to schedule work");
}
};
TaskExecutorWithFailureInScheduleWorkAt badExecutor(&getExecutor());
_executorProxy->setExecutor(&badExecutor);
processNetworkResponse(BSON("ok" << 1));
_executorProxy->setExecutor(&getExecutor());
ASSERT_EQUALS(ErrorCodes::OperationFailed, reporter->join());
assertReporterDone();
}
void Bridge::callFunction(
ExecutorToken executorToken,
const std::string& moduleId,
const std::string& methodId,
const folly::dynamic& arguments,
const std::string& tracingName) {
if (*m_destroyed) {
return;
}
#ifdef WITH_FBSYSTRACE
int systraceCookie = m_systraceCookie++;
FbSystraceAsyncFlow::begin(
TRACE_TAG_REACT_CXX_BRIDGE,
tracingName.c_str(),
systraceCookie);
#endif
auto executorMessageQueueThread = getMessageQueueThread(executorToken);
if (executorMessageQueueThread == nullptr) {
LOG(WARNING) << "Dropping JS call for executor that has been unregistered...";
return;
}
std::shared_ptr<bool> isDestroyed = m_destroyed;
executorMessageQueueThread->runOnQueue([=] () {
#ifdef WITH_FBSYSTRACE
FbSystraceAsyncFlow::end(
TRACE_TAG_REACT_CXX_BRIDGE,
tracingName.c_str(),
systraceCookie);
FbSystraceSection s(TRACE_TAG_REACT_CXX_BRIDGE, tracingName.c_str());
#endif
if (*isDestroyed) {
return;
}
JSExecutor *executor = getExecutor(executorToken);
if (executor == nullptr) {
LOG(WARNING) << "Dropping JS call for executor that has been unregistered...";
return;
}
// This is safe because we are running on the executor's thread: it won't
// destruct until after it's been unregistered (which we check above) and
// that will happen on this thread
executor->callFunction(moduleId, methodId, arguments);
});
}
TEST_F(ReporterTest, KeepAliveTimeoutFailingToScheduleRemoteCommandShouldMakeReporterInactive) {
processNetworkResponse(BSON("ok" << 1));
auto until = getExecutor().now() + reporter->getKeepAliveInterval();
ASSERT_EQUALS(until, reporter->getKeepAliveTimeoutWhen_forTest());
ASSERT_TRUE(reporter->isActive());
Status expectedStatus(ErrorCodes::UnknownError, "failed to prepare update command");
prepareReplSetUpdatePositionCommandFn = [expectedStatus]() -> StatusWith<BSONObj> {
return expectedStatus;
};
runUntil(until);
ASSERT_EQUALS(expectedStatus, reporter->join());
assertReporterDone();
}
Status getExecutor(OperationContext* txn,
Collection* collection,
const std::string& ns,
const BSONObj& unparsedQuery,
PlanExecutor** out,
size_t plannerOptions) {
if (!collection) {
LOG(2) << "Collection " << ns << " does not exist."
<< " Using EOF stage: " << unparsedQuery.toString();
EOFStage* eofStage = new EOFStage();
WorkingSet* ws = new WorkingSet();
*out = new PlanExecutor(ws, eofStage, collection);
return Status::OK();
}
if (!CanonicalQuery::isSimpleIdQuery(unparsedQuery) ||
!collection->getIndexCatalog()->findIdIndex()) {
const WhereCallbackReal whereCallback(collection->ns().db());
CanonicalQuery* cq;
Status status = CanonicalQuery::canonicalize(
collection->ns(), unparsedQuery, &cq, whereCallback);
if (!status.isOK())
return status;
// Takes ownership of 'cq'.
return getExecutor(txn, collection, cq, out, plannerOptions);
}
LOG(2) << "Using idhack: " << unparsedQuery.toString();
WorkingSet* ws = new WorkingSet();
PlanStage* root = new IDHackStage(txn, collection, unparsedQuery["_id"].wrap(), ws);
// Might have to filter out orphaned docs.
if (plannerOptions & QueryPlannerParams::INCLUDE_SHARD_FILTER) {
root = new ShardFilterStage(shardingState.getCollectionMetadata(collection->ns()),
ws, root);
}
*out = new PlanExecutor(ws, root, collection);
return Status::OK();
}
/* fetch a single object from collection ns that matches query
set your db SavedContext first
*/
RecordId Helpers::findOne(OperationContext* opCtx,
Collection* collection,
const BSONObj& query,
bool requireIndex) {
if (!collection)
return RecordId();
const ExtensionsCallbackReal extensionsCallback(opCtx, &collection->ns());
auto qr = stdx::make_unique<QueryRequest>(collection->ns());
qr->setFilter(query);
const boost::intrusive_ptr<ExpressionContext> expCtx;
auto statusWithCQ =
CanonicalQuery::canonicalize(opCtx,
std::move(qr),
expCtx,
extensionsCallback,
MatchExpressionParser::kAllowAllSpecialFeatures &
~MatchExpressionParser::AllowedFeatures::kExpr);
massert(17244, "Could not canonicalize " + query.toString(), statusWithCQ.isOK());
unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
size_t options = requireIndex ? QueryPlannerParams::NO_TABLE_SCAN : QueryPlannerParams::DEFAULT;
auto exec = uassertStatusOK(
getExecutor(opCtx, collection, std::move(cq), PlanExecutor::NO_YIELD, options));
PlanExecutor::ExecState state;
BSONObj obj;
RecordId loc;
if (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, &loc))) {
return loc;
}
massert(34427,
"Plan executor error: " + WorkingSetCommon::toStatusString(obj),
PlanExecutor::IS_EOF == state);
return RecordId();
}
Status getExecutorCount(OperationContext* txn,
Collection* collection,
const BSONObj& query,
const BSONObj& hintObj,
PlanExecutor** execOut) {
invariant(collection);
const WhereCallbackReal whereCallback(collection->ns().db());
CanonicalQuery* cq;
uassertStatusOK(CanonicalQuery::canonicalize(collection->ns().ns(),
query,
BSONObj(),
BSONObj(),
0,
0,
hintObj,
&cq,
whereCallback));
// Takes ownership of 'cq'.
return getExecutor(txn, collection, cq, execOut, QueryPlannerParams::PRIVATE_IS_COUNT);
}
Status FindCmd::explain(OperationContext* txn,
const std::string& dbname,
const BSONObj& cmdObj,
ExplainCommon::Verbosity verbosity,
BSONObjBuilder* out) const {
const string fullns = parseNs(dbname, cmdObj);
// Parse the command BSON to a LiteParsedQuery.
LiteParsedQuery* rawLpq;
bool isExplain = true;
Status lpqStatus = LiteParsedQuery::make(fullns, cmdObj, isExplain, &rawLpq);
if (!lpqStatus.isOK()) {
return lpqStatus;
}
auto_ptr<LiteParsedQuery> lpq(rawLpq);
const NamespaceString nss(fullns);
// Finish the parsing step by using the LiteParsedQuery to create a CanonicalQuery.
// This requires a lock on the collection in case we're parsing $where: where-specific
// parsing code assumes we have a lock and creates execution machinery that requires it.
CanonicalQuery* rawCq;
WhereCallbackReal whereCallback(txn, nss.db());
Status canonStatus = CanonicalQuery::canonicalize(lpq.release(), &rawCq, whereCallback);
if (!canonStatus.isOK()) {
return canonStatus;
}
auto_ptr<CanonicalQuery> cq(rawCq);
AutoGetCollectionForRead ctx(txn, nss);
// The collection may be NULL. If so, getExecutor() should handle it by returning
// an execution tree with an EOFStage.
Collection* collection = ctx.getCollection();
// We have a parsed query. Time to get the execution plan for it.
PlanExecutor* rawExec;
Status execStatus = Status::OK();
if (cq->getParsed().getOptions().oplogReplay) {
execStatus = getOplogStartHack(txn, collection, cq.release(), &rawExec);
}
else {
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(cq->getParsed().ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
execStatus = getExecutor(txn,
collection,
cq.release(),
PlanExecutor::YIELD_AUTO,
&rawExec,
options);
}
if (!execStatus.isOK()) {
return execStatus;
}
scoped_ptr<PlanExecutor> exec(rawExec);
// Got the execution tree. Explain it.
Explain::explainStages(exec.get(), verbosity, out);
return Status::OK();
}
bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
const string ns = dbname + "." + cmdObj.firstElement().valuestr();
if (!cmdObj["start"].eoo()) {
errmsg = "using deprecated 'start' argument to geoNear";
return false;
}
Client::ReadContext ctx(txn, ns);
Database* db = ctx.ctx().db();
if ( !db ) {
errmsg = "can't find ns";
return false;
}
Collection* collection = db->getCollection( txn, ns );
if ( !collection ) {
errmsg = "can't find ns";
return false;
}
IndexCatalog* indexCatalog = collection->getIndexCatalog();
// cout << "raw cmd " << cmdObj.toString() << endl;
// We seek to populate this.
string nearFieldName;
bool using2DIndex = false;
if (!getFieldName(txn, collection, indexCatalog, &nearFieldName, &errmsg, &using2DIndex)) {
return false;
}
PointWithCRS point;
uassert(17304, "'near' field must be point",
GeoParser::parseQueryPoint(cmdObj["near"], &point).isOK());
bool isSpherical = cmdObj["spherical"].trueValue();
if (!using2DIndex) {
uassert(17301, "2dsphere index must have spherical: true", isSpherical);
}
// Build the $near expression for the query.
BSONObjBuilder nearBob;
if (isSpherical) {
nearBob.append("$nearSphere", cmdObj["near"].Obj());
}
else {
nearBob.append("$near", cmdObj["near"].Obj());
}
if (!cmdObj["maxDistance"].eoo()) {
uassert(17299, "maxDistance must be a number",cmdObj["maxDistance"].isNumber());
nearBob.append("$maxDistance", cmdObj["maxDistance"].number());
}
if (!cmdObj["minDistance"].eoo()) {
uassert(17298, "minDistance doesn't work on 2d index", !using2DIndex);
uassert(17300, "minDistance must be a number",cmdObj["minDistance"].isNumber());
nearBob.append("$minDistance", cmdObj["minDistance"].number());
}
if (!cmdObj["uniqueDocs"].eoo()) {
warning() << ns << ": ignoring deprecated uniqueDocs option in geoNear command";
}
// And, build the full query expression.
BSONObjBuilder queryBob;
queryBob.append(nearFieldName, nearBob.obj());
if (!cmdObj["query"].eoo() && cmdObj["query"].isABSONObj()) {
queryBob.appendElements(cmdObj["query"].Obj());
}
BSONObj rewritten = queryBob.obj();
// cout << "rewritten query: " << rewritten.toString() << endl;
int numWanted = 100;
const char* limitName = !cmdObj["num"].eoo() ? "num" : "limit";
BSONElement eNumWanted = cmdObj[limitName];
if (!eNumWanted.eoo()) {
uassert(17303, "limit must be number", eNumWanted.isNumber());
numWanted = eNumWanted.numberInt();
uassert(17302, "limit must be >=0", numWanted >= 0);
}
bool includeLocs = false;
if (!cmdObj["includeLocs"].eoo()) {
includeLocs = cmdObj["includeLocs"].trueValue();
}
double distanceMultiplier = 1.0;
BSONElement eDistanceMultiplier = cmdObj["distanceMultiplier"];
if (!eDistanceMultiplier.eoo()) {
uassert(17296, "distanceMultiplier must be a number", eDistanceMultiplier.isNumber());
distanceMultiplier = eDistanceMultiplier.number();
uassert(17297, "distanceMultiplier must be non-negative", distanceMultiplier >= 0);
}
BSONObj projObj = BSON("$pt" << BSON("$meta" << LiteParsedQuery::metaGeoNearPoint) <<
"$dis" << BSON("$meta" << LiteParsedQuery::metaGeoNearDistance));
CanonicalQuery* cq;
const NamespaceString nss(dbname);
const WhereCallbackReal whereCallback(txn, nss.db());
if (!CanonicalQuery::canonicalize(ns,
rewritten,
BSONObj(),
projObj,
0,
numWanted,
BSONObj(),
&cq,
whereCallback).isOK()) {
errmsg = "Can't parse filter / create query";
return false;
}
PlanExecutor* rawExec;
if (!getExecutor(txn, collection, cq, &rawExec, 0).isOK()) {
errmsg = "can't get query runner";
return false;
}
auto_ptr<PlanExecutor> exec(rawExec);
const ScopedExecutorRegistration safety(exec.get());
double totalDistance = 0;
BSONObjBuilder resultBuilder(result.subarrayStart("results"));
double farthestDist = 0;
BSONObj currObj;
int results = 0;
while ((results < numWanted) && PlanExecutor::ADVANCED == exec->getNext(&currObj, NULL)) {
// Come up with the correct distance.
double dist = currObj["$dis"].number() * distanceMultiplier;
totalDistance += dist;
if (dist > farthestDist) { farthestDist = dist; }
// Strip out '$dis' and '$pt' from the result obj. The rest gets added as 'obj'
// in the command result.
BSONObjIterator resIt(currObj);
BSONObjBuilder resBob;
while (resIt.more()) {
BSONElement elt = resIt.next();
if (!mongoutils::str::equals("$pt", elt.fieldName())
&& !mongoutils::str::equals("$dis", elt.fieldName())) {
resBob.append(elt);
}
}
BSONObj resObj = resBob.obj();
// Don't make a too-big result object.
if (resultBuilder.len() + resObj.objsize()> BSONObjMaxUserSize) {
warning() << "Too many geoNear results for query " << rewritten.toString()
<< ", truncating output.";
break;
}
// Add the next result to the result builder.
BSONObjBuilder oneResultBuilder(
resultBuilder.subobjStart(BSONObjBuilder::numStr(results)));
oneResultBuilder.append("dis", dist);
if (includeLocs) {
oneResultBuilder.appendAs(currObj["$pt"], "loc");
}
oneResultBuilder.append("obj", resObj);
oneResultBuilder.done();
++results;
}
resultBuilder.done();
// Fill out the stats subobj.
BSONObjBuilder stats(result.subobjStart("stats"));
// Fill in nscanned from the explain.
PlanSummaryStats summary;
Explain::getSummaryStats(exec.get(), &summary);
stats.appendNumber("nscanned", summary.totalKeysExamined);
stats.appendNumber("objectsLoaded", summary.totalDocsExamined);
stats.append("avgDistance", totalDistance / results);
stats.append("maxDistance", farthestDist);
stats.append("time", txn->getCurOp()->elapsedMillis());
stats.done();
return true;
}
std::shared_ptr<IOExecutor> getIOExecutor() {
return getExecutor(
globalIOExecutor,
globalIOThreadPool,
globalIOExecutorLock);
}
/*
* Runs the command object cmdobj on the db with name dbname and puts result in result.
* @param dbname, name of db
* @param cmdobj, object that contains entire command
* @param options
* @param errmsg, reference to error message
* @param result, reference to builder for result
* @param fromRepl
* @return true if successful, false otherwise
*/
bool FTSCommand::_run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int cmdOptions,
const string& ns,
const string& searchString,
string language, // "" for not-set
int limit,
BSONObj& filter,
BSONObj& projection,
string& errmsg,
BSONObjBuilder& result ) {
Timer comm;
// Rewrite the cmd as a normal query.
BSONObjBuilder queryBob;
queryBob.appendElements(filter);
BSONObjBuilder textBob;
textBob.append("$search", searchString);
if (!language.empty()) {
textBob.append("$language", language);
}
queryBob.append("$text", textBob.obj());
// This is the query we exec.
BSONObj queryObj = queryBob.obj();
// We sort by the score.
BSONObj sortSpec = BSON("$s" << BSON("$meta" << LiteParsedQuery::metaTextScore));
// We also project the score into the document and strip it out later during the reformatting
// of the results.
BSONObjBuilder projBob;
projBob.appendElements(projection);
projBob.appendElements(sortSpec);
BSONObj projObj = projBob.obj();
AutoGetCollectionForRead ctx(txn, ns);
CanonicalQuery* cq;
Status canonicalizeStatus =
CanonicalQuery::canonicalize(ns,
queryObj,
sortSpec,
projObj,
0,
limit,
BSONObj(),
&cq,
WhereCallbackReal(txn, dbname));
if (!canonicalizeStatus.isOK()) {
errmsg = canonicalizeStatus.reason();
return false;
}
PlanExecutor* rawExec;
Status getExecStatus = getExecutor(txn, ctx.getCollection(), cq, &rawExec);
if (!getExecStatus.isOK()) {
errmsg = getExecStatus.reason();
return false;
}
auto_ptr<PlanExecutor> exec(rawExec);
BSONArrayBuilder resultBuilder(result.subarrayStart("results"));
// Quoth: "leave a mb for other things"
int resultSize = 1024 * 1024;
int numReturned = 0;
BSONObj obj;
while (PlanExecutor::ADVANCED == exec->getNext(&obj, NULL)) {
if ((resultSize + obj.objsize()) >= BSONObjMaxUserSize) {
break;
}
// We return an array of results. Add another element.
BSONObjBuilder oneResultBuilder(resultBuilder.subobjStart());
oneResultBuilder.append("score", obj["$s"].number());
// Strip out the score from the returned obj.
BSONObjIterator resIt(obj);
BSONObjBuilder resBob;
while (resIt.more()) {
BSONElement elt = resIt.next();
if (!mongoutils::str::equals("$s", elt.fieldName())) {
resBob.append(elt);
}
}
oneResultBuilder.append("obj", resBob.obj());
BSONObj addedArrayObj = oneResultBuilder.done();
resultSize += addedArrayObj.objsize();
numReturned++;
}
resultBuilder.done();
// returns some stats to the user
BSONObjBuilder stats(result.subobjStart("stats"));
// Fill in nscanned from the explain.
PlanSummaryStats summary;
Explain::getSummaryStats(exec.get(), &summary);
stats.appendNumber("nscanned", summary.totalKeysExamined);
stats.appendNumber("nscannedObjects", summary.totalDocsExamined);
stats.appendNumber( "n" , numReturned );
stats.append( "timeMicros", (int)comm.micros() );
stats.done();
return true;
}
static bool runImpl(OperationContext* txn,
const string& dbname,
const string& ns,
const BSONObj& query,
const BSONObj& fields,
const BSONObj& update,
const BSONObj& sort,
bool upsert,
bool returnNew,
bool remove ,
BSONObjBuilder& result,
string& errmsg) {
AutoGetOrCreateDb autoDb(txn, dbname, MODE_IX);
Lock::CollectionLock collLock(txn->lockState(), ns, MODE_IX);
Client::Context ctx(txn, ns, autoDb.getDb(), autoDb.justCreated());
if (!repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(dbname)) {
return appendCommandStatus(result, Status(ErrorCodes::NotMaster, str::stream()
<< "Not primary while running findAndModify in " << ns));
}
Collection* collection = ctx.db()->getCollection(ns);
const WhereCallbackReal whereCallback(txn, StringData(ns));
if ( !collection ) {
if ( !upsert ) {
// no collectio and no upsert, so can't possible do anything
_appendHelper( result, BSONObj(), false, fields, whereCallback );
return true;
}
// no collection, but upsert, so we want to create it
// problem is we only have IX on db and collection :(
// so we tell our caller who can do it
errmsg = "no-collection";
return false;
}
Snapshotted<BSONObj> snapshotDoc;
RecordId loc;
bool found = false;
{
CanonicalQuery* cq;
const BSONObj projection;
const long long skip = 0;
const long long limit = -1; // 1 document requested; negative indicates hard limit.
uassertStatusOK(CanonicalQuery::canonicalize(ns,
query,
sort,
projection,
skip,
limit,
&cq,
whereCallback));
PlanExecutor* rawExec;
uassertStatusOK(getExecutor(txn,
collection,
cq,
PlanExecutor::YIELD_AUTO,
&rawExec,
QueryPlannerParams::DEFAULT));
scoped_ptr<PlanExecutor> exec(rawExec);
PlanExecutor::ExecState state = exec->getNextSnapshotted(&snapshotDoc, &loc);
if (PlanExecutor::ADVANCED == state) {
found = true;
}
else if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) {
if (PlanExecutor::FAILURE == state &&
WorkingSetCommon::isValidStatusMemberObject(snapshotDoc.value())) {
const Status errorStatus =
WorkingSetCommon::getMemberObjectStatus(snapshotDoc.value());
invariant(!errorStatus.isOK());
uasserted(errorStatus.code(), errorStatus.reason());
}
uasserted(ErrorCodes::OperationFailed,
str::stream() << "executor returned " << PlanExecutor::statestr(state)
<< " while finding document to update");
}
else {
invariant(PlanExecutor::IS_EOF == state);
}
}
WriteUnitOfWork wuow(txn);
if (found) {
// We found a doc, but it might not be associated with the active snapshot.
// If the doc has changed or is no longer in the collection, we will throw a
// write conflict exception and start again from the beginning.
if (txn->recoveryUnit()->getSnapshotId() != snapshotDoc.snapshotId()) {
BSONObj oldObj = snapshotDoc.value();
if (!collection->findDoc(txn, loc, &snapshotDoc)) {
// Got deleted in the new snapshot.
throw WriteConflictException();
}
if (!oldObj.binaryEqual(snapshotDoc.value())) {
// Got updated in the new snapshot.
throw WriteConflictException();
}
}
// If we get here without throwing, then we should have the copy of the doc from
// the latest snapshot.
invariant(txn->recoveryUnit()->getSnapshotId() == snapshotDoc.snapshotId());
}
BSONObj doc = snapshotDoc.value();
BSONObj queryModified = query;
if (found && !doc["_id"].eoo() && !CanonicalQuery::isSimpleIdQuery(query)) {
// we're going to re-write the query to be more efficient
// we have to be a little careful because of positional operators
// maybe we can pass this all through eventually, but right now isn't an easy way
bool hasPositionalUpdate = false;
{
// if the update has a positional piece ($)
// then we need to pull all query parts in
// so here we check for $
// a little hacky
BSONObjIterator i( update );
while ( i.more() ) {
const BSONElement& elem = i.next();
if ( elem.fieldName()[0] != '$' || elem.type() != Object )
continue;
BSONObjIterator j( elem.Obj() );
while ( j.more() ) {
if ( str::contains( j.next().fieldName(), ".$" ) ) {
hasPositionalUpdate = true;
break;
}
}
}
}
BSONObjBuilder b(query.objsize() + 10);
b.append( doc["_id"] );
bool addedAtomic = false;
BSONObjIterator i(query);
while ( i.more() ) {
const BSONElement& elem = i.next();
if ( str::equals( "_id" , elem.fieldName() ) ) {
// we already do _id
continue;
}
if ( ! hasPositionalUpdate ) {
// if there is a dotted field, accept we may need more query parts
continue;
}
if ( ! addedAtomic ) {
b.appendBool( "$atomic" , true );
addedAtomic = true;
}
b.append( elem );
}
queryModified = b.obj();
}
if ( remove ) {
_appendHelper(result, doc, found, fields, whereCallback);
if ( found ) {
deleteObjects(txn, ctx.db(), ns, queryModified, PlanExecutor::YIELD_MANUAL,
true, true);
BSONObjBuilder le( result.subobjStart( "lastErrorObject" ) );
le.appendNumber( "n" , 1 );
le.done();
}
}
else {
// update
if ( ! found && ! upsert ) {
// didn't have it, and am not upserting
_appendHelper(result, doc, found, fields, whereCallback);
}
else {
// we found it or we're updating
if ( ! returnNew ) {
_appendHelper(result, doc, found, fields, whereCallback);
}
const NamespaceString requestNs(ns);
UpdateRequest request(requestNs);
request.setQuery(queryModified);
request.setUpdates(update);
request.setUpsert(upsert);
request.setUpdateOpLog();
request.setStoreResultDoc(returnNew);
request.setYieldPolicy(PlanExecutor::YIELD_MANUAL);
// TODO(greg) We need to send if we are ignoring
// the shard version below, but for now no
UpdateLifecycleImpl updateLifecycle(false, requestNs);
request.setLifecycle(&updateLifecycle);
UpdateResult res = mongo::update(txn,
ctx.db(),
request,
&txn->getCurOp()->debug());
if (!found && res.existing) {
// No match was found during the read part of this find and modify, which
// means that we're here doing an upsert. But the update also told us that
// we modified an *already existing* document. This probably means that
// the query reported EOF based on an out-of-date snapshot. This should be
// a rare event, so we handle it by throwing a write conflict.
throw WriteConflictException();
}
if ( !collection ) {
// collection created by an upsert
collection = ctx.db()->getCollection(ns);
}
LOG(3) << "update result: " << res ;
if (returnNew) {
dassert(!res.newObj.isEmpty());
_appendHelper(result, res.newObj, true, fields, whereCallback);
}
BSONObjBuilder le( result.subobjStart( "lastErrorObject" ) );
le.appendBool( "updatedExisting" , res.existing );
le.appendNumber( "n" , res.numMatched );
if ( !res.upserted.isEmpty() ) {
le.append( res.upserted[kUpsertedFieldName] );
}
le.done();
}
}
// Committing the WUOW can close the current snapshot. Until this happens, the
// snapshot id should not have changed.
if (found) {
invariant(txn->recoveryUnit()->getSnapshotId() == snapshotDoc.snapshotId());
}
wuow.commit();
return true;
}
bool run(OperationContext* txn,
const string& dbname,
BSONObj& jsobj,
int,
string& errmsg,
BSONObjBuilder& result) {
const std::string ns = parseNs(dbname, jsobj);
md5digest d;
md5_state_t st;
md5_init(&st);
int n = 0;
bool partialOk = jsobj["partialOk"].trueValue();
if (partialOk) {
// WARNING: This code depends on the binary layout of md5_state. It will not be
// compatible with different md5 libraries or work correctly in an environment with
// mongod's of different endians. It is ok for mongos to be a different endian since
// it just passes the buffer through to another mongod.
BSONElement stateElem = jsobj["md5state"];
if (!stateElem.eoo()) {
int len;
const char* data = stateElem.binDataClean(len);
massert(16247, "md5 state not correct size", len == sizeof(st));
memcpy(&st, data, sizeof(st));
}
n = jsobj["startAt"].numberInt();
}
BSONObj query = BSON("files_id" << jsobj["filemd5"] << "n" << GTE << n);
BSONObj sort = BSON("files_id" << 1 << "n" << 1);
MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN {
CanonicalQuery* cq;
if (!CanonicalQuery::canonicalize(ns, query, sort, BSONObj(), &cq).isOK()) {
uasserted(17240, "Can't canonicalize query " + query.toString());
return 0;
}
// Check shard version at startup.
// This will throw before we've done any work if shard version is outdated
// We drop and re-acquire these locks every document because md5'ing is expensive
unique_ptr<AutoGetCollectionForRead> ctx(new AutoGetCollectionForRead(txn, ns));
Collection* coll = ctx->getCollection();
PlanExecutor* rawExec;
if (!getExecutor(txn,
coll,
cq,
PlanExecutor::YIELD_MANUAL,
&rawExec,
QueryPlannerParams::NO_TABLE_SCAN).isOK()) {
uasserted(17241, "Can't get executor for query " + query.toString());
return 0;
}
unique_ptr<PlanExecutor> exec(rawExec);
// Process notifications when the lock is released/reacquired in the loop below
exec->registerExec();
BSONObj obj;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
BSONElement ne = obj["n"];
verify(ne.isNumber());
int myn = ne.numberInt();
if (n != myn) {
if (partialOk) {
break; // skipped chunk is probably on another shard
}
log() << "should have chunk: " << n << " have:" << myn << endl;
dumpChunks(txn, ns, query, sort);
uassert(10040, "chunks out of order", n == myn);
}
// make a copy of obj since we access data in it while yielding locks
BSONObj owned = obj.getOwned();
exec->saveState();
// UNLOCKED
ctx.reset();
int len;
const char* data = owned["data"].binDataClean(len);
// This is potentially an expensive operation, so do it out of the lock
md5_append(&st, (const md5_byte_t*)(data), len);
n++;
try {
// RELOCKED
ctx.reset(new AutoGetCollectionForRead(txn, ns));
} catch (const SendStaleConfigException& ex) {
LOG(1) << "chunk metadata changed during filemd5, will retarget and continue";
break;
}
// Have the lock again. See if we were killed.
if (!exec->restoreState(txn)) {
if (!partialOk) {
uasserted(13281, "File deleted during filemd5 command");
}
}
}
if (partialOk)
result.appendBinData("md5state", sizeof(st), BinDataGeneral, &st);
// This must be *after* the capture of md5state since it mutates st
md5_finish(&st, d);
result.append("numChunks", n);
result.append("md5", digestToString(d));
}
MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "filemd5", dbname);
return true;
}
bool group( OperationContext* txn,
Database* db,
const std::string& ns,
const BSONObj& query,
BSONObj keyPattern,
const std::string& keyFunctionCode,
const std::string& reduceCode,
const char * reduceScope,
BSONObj initial,
const std::string& finalize,
string& errmsg,
BSONObjBuilder& result ) {
const string userToken = ClientBasic::getCurrent()->getAuthorizationSession()
->getAuthenticatedUserNamesToken();
auto_ptr<Scope> s = globalScriptEngine->getPooledScope(db->name(), "group" + userToken);
if ( reduceScope )
s->init( reduceScope );
s->setObject( "$initial" , initial , true );
s->exec( "$reduce = " + reduceCode , "$group reduce setup" , false , true , true , 100 );
s->exec( "$arr = [];" , "$group reduce setup 2" , false , true , true , 100 );
ScriptingFunction f = s->createFunction(
"function(){ "
" if ( $arr[n] == null ){ "
" next = {}; "
" Object.extend( next , $key ); "
" Object.extend( next , $initial , true ); "
" $arr[n] = next; "
" next = null; "
" } "
" $reduce( obj , $arr[n] ); "
"}" );
ScriptingFunction keyFunction = 0;
if ( keyFunctionCode.size() ) {
keyFunction = s->createFunction( keyFunctionCode.c_str() );
}
double keysize = keyPattern.objsize() * 3;
double keynum = 1;
Collection* collection = db->getCollection( txn, ns );
const WhereCallbackReal whereCallback(txn, StringData(db->name()));
map<BSONObj,int,BSONObjCmp> map;
list<BSONObj> blah;
if (collection) {
CanonicalQuery* cq;
if (!CanonicalQuery::canonicalize(ns, query, &cq, whereCallback).isOK()) {
uasserted(17212, "Can't canonicalize query " + query.toString());
return 0;
}
PlanExecutor* rawExec;
if (!getExecutor(txn,collection, cq, &rawExec).isOK()) {
uasserted(17213, "Can't get executor for query " + query.toString());
return 0;
}
auto_ptr<PlanExecutor> exec(rawExec);
const ScopedExecutorRegistration safety(exec.get());
BSONObj obj;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
BSONObj key = getKey(obj , keyPattern , keyFunction , keysize / keynum,
s.get() );
keysize += key.objsize();
keynum++;
int& n = map[key];
if ( n == 0 ) {
n = map.size();
s->setObject( "$key" , key , true );
uassert(17203, "group() can't handle more than 20000 unique keys",
n <= 20000 );
}
s->setObject( "obj" , obj , true );
s->setNumber( "n" , n - 1 );
if ( s->invoke( f , 0, 0 , 0 , true ) ) {
throw UserException(17214,
(string)"reduce invoke failed: " + s->getError());
}
}
}
if (!finalize.empty()) {
s->exec( "$finalize = " + finalize , "$group finalize define" ,
false , true , true , 100 );
ScriptingFunction g = s->createFunction(
"function(){ "
" for(var i=0; i < $arr.length; i++){ "
" var ret = $finalize($arr[i]); "
" if (ret !== undefined) "
" $arr[i] = ret; "
" } "
"}" );
s->invoke( g , 0, 0 , 0 , true );
}
result.appendArray( "retval" , s->getObject( "$arr" ) );
result.append( "count" , keynum - 1 );
result.append( "keys" , (int)(map.size()) );
s->exec( "$arr = [];" , "$group reduce setup 2" , false , true , true , 100 );
s->gc();
return true;
}
Status getOplogStartHack(OperationContext* txn,
Collection* collection,
CanonicalQuery* cq,
PlanExecutor** execOut) {
invariant(cq);
auto_ptr<CanonicalQuery> autoCq(cq);
if ( collection == NULL )
return Status(ErrorCodes::InternalError,
"getOplogStartHack called with a NULL collection" );
// A query can only do oplog start finding if it has a top-level $gt or $gte predicate over
// the "ts" field (the operation's timestamp). Find that predicate and pass it to
// the OplogStart stage.
MatchExpression* tsExpr = NULL;
if (MatchExpression::AND == cq->root()->matchType()) {
// The query has an AND at the top-level. See if any of the children
// of the AND are $gt or $gte predicates over 'ts'.
for (size_t i = 0; i < cq->root()->numChildren(); ++i) {
MatchExpression* me = cq->root()->getChild(i);
if (isOplogTsPred(me)) {
tsExpr = me;
break;
}
}
}
else if (isOplogTsPred(cq->root())) {
// The root of the tree is a $gt or $gte predicate over 'ts'.
tsExpr = cq->root();
}
if (NULL == tsExpr) {
return Status(ErrorCodes::OplogOperationUnsupported,
"OplogReplay query does not contain top-level "
"$gt or $gte over the 'ts' field.");
}
DiskLoc startLoc = DiskLoc().setInvalid();
// See if the RecordStore supports the oplogStartHack
const BSONElement tsElem = extractOplogTsOptime(tsExpr);
if (tsElem.type() == Timestamp) {
StatusWith<DiskLoc> goal = oploghack::keyForOptime(tsElem._opTime());
if (goal.isOK()) {
startLoc = collection->getRecordStore()->oplogStartHack(txn, goal.getValue());
}
}
if (startLoc.isValid()) {
LOG(3) << "Using direct oplog seek";
}
else {
LOG(3) << "Using OplogStart stage";
// Fallback to trying the OplogStart stage.
WorkingSet* oplogws = new WorkingSet();
OplogStart* stage = new OplogStart(txn, collection, tsExpr, oplogws);
PlanExecutor* rawExec;
// Takes ownership of oplogws and stage.
Status execStatus = PlanExecutor::make(txn, oplogws, stage, collection,
PlanExecutor::YIELD_AUTO, &rawExec);
invariant(execStatus.isOK());
scoped_ptr<PlanExecutor> exec(rawExec);
// The stage returns a DiskLoc of where to start.
PlanExecutor::ExecState state = exec->getNext(NULL, &startLoc);
// This is normal. The start of the oplog is the beginning of the collection.
if (PlanExecutor::IS_EOF == state) {
return getExecutor(txn, collection, autoCq.release(), PlanExecutor::YIELD_AUTO,
execOut);
}
// This is not normal. An error was encountered.
if (PlanExecutor::ADVANCED != state) {
return Status(ErrorCodes::InternalError,
"quick oplog start location had error...?");
}
}
// cout << "diskloc is " << startLoc.toString() << endl;
// Build our collection scan...
CollectionScanParams params;
params.collection = collection;
params.start = startLoc;
params.direction = CollectionScanParams::FORWARD;
params.tailable = cq->getParsed().getOptions().tailable;
WorkingSet* ws = new WorkingSet();
CollectionScan* cs = new CollectionScan(txn, params, ws, cq->root());
// Takes ownership of 'ws', 'cs', and 'cq'.
return PlanExecutor::make(txn, ws, cs, autoCq.release(), collection,
PlanExecutor::YIELD_AUTO, execOut);
}
std::string newRunQuery(OperationContext* txn,
Message& m,
QueryMessage& q,
CurOp& curop,
Message &result,
bool fromDBDirectClient) {
// Validate the namespace.
const char *ns = q.ns;
uassert(16332, "can't have an empty ns", ns[0]);
const NamespaceString nsString(ns);
uassert(16256, str::stream() << "Invalid ns [" << ns << "]", nsString.isValid());
// Set curop information.
curop.debug().ns = ns;
curop.debug().ntoreturn = q.ntoreturn;
curop.debug().query = q.query;
curop.setQuery(q.query);
// If the query is really a command, run it.
if (nsString.isCommand()) {
int nToReturn = q.ntoreturn;
uassert(16979, str::stream() << "bad numberToReturn (" << nToReturn
<< ") for $cmd type ns - can only be 1 or -1",
nToReturn == 1 || nToReturn == -1);
curop.markCommand();
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder cmdResBuf;
if (!runCommands(txn, ns, q.query, curop, bb, cmdResBuf, false, q.queryOptions)) {
uasserted(13530, "bad or malformed command request?");
}
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
const NamespaceString nss(q.ns);
// Parse the qm into a CanonicalQuery.
CanonicalQuery* cq;
Status canonStatus = CanonicalQuery::canonicalize(
q, &cq, WhereCallbackReal(txn, StringData(nss.db())));
if (!canonStatus.isOK()) {
uasserted(17287, str::stream() << "Can't canonicalize query: " << canonStatus.toString());
}
QLOG() << "Running query:\n" << cq->toString();
LOG(2) << "Running query: " << cq->toStringShort();
// Parse, canonicalize, plan, transcribe, and get a plan executor.
PlanExecutor* rawExec = NULL;
// We use this a lot below.
const LiteParsedQuery& pq = cq->getParsed();
AutoGetCollectionForRead ctx(txn, nss);
const int dbProfilingLevel = (ctx.getDb() != NULL) ? ctx.getDb()->getProfilingLevel() :
serverGlobalParams.defaultProfile;
Collection* collection = ctx.getCollection();
// We'll now try to get the query executor that will execute this query for us. There
// are a few cases in which we know upfront which executor we should get and, therefore,
// we shortcut the selection process here.
//
// (a) If the query is over a collection that doesn't exist, we use an EOFStage.
//
// (b) if the query is a replication's initial sync one, we use a specifically designed
// stage that skips extents faster (see details in exec/oplogstart.h).
//
// Otherwise we go through the selection of which executor is most suited to the
// query + run-time context at hand.
Status status = Status::OK();
if (NULL != collection && pq.getOptions().oplogReplay) {
// Takes ownership of 'cq'.
status = getOplogStartHack(txn, collection, cq, &rawExec);
}
else {
size_t options = QueryPlannerParams::DEFAULT;
if (shardingState.needCollectionMetadata(pq.ns())) {
options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
}
// Takes ownership of 'cq'.
status = getExecutor(txn, collection, cq, PlanExecutor::YIELD_AUTO, &rawExec, options);
}
if (!status.isOK()) {
// NOTE: Do not access cq as getExecutor has deleted it.
uasserted(17007, "Unable to execute query: " + status.reason());
}
verify(NULL != rawExec);
auto_ptr<PlanExecutor> exec(rawExec);
// If it's actually an explain, do the explain and return rather than falling through
// to the normal query execution loop.
if (pq.isExplain()) {
BufBuilder bb;
bb.skip(sizeof(QueryResult::Value));
BSONObjBuilder explainBob;
Explain::explainStages(exec.get(), ExplainCommon::EXEC_ALL_PLANS, &explainBob);
// Add the resulting object to the return buffer.
BSONObj explainObj = explainBob.obj();
bb.appendBuf((void*)explainObj.objdata(), explainObj.objsize());
curop.debug().iscommand = true;
// TODO: Does this get overwritten/do we really need to set this twice?
curop.debug().query = q.query;
// Set query result fields.
QueryResult::View qr = bb.buf();
bb.decouple();
qr.setResultFlagsToOk();
qr.msgdata().setLen(bb.len());
curop.debug().responseLength = bb.len();
qr.msgdata().setOperation(opReply);
qr.setCursorId(0);
qr.setStartingFrom(0);
qr.setNReturned(1);
result.setData(qr.view2ptr(), true);
return "";
}
// We freak out later if this changes before we're done with the query.
const ChunkVersion shardingVersionAtStart = shardingState.getVersion(cq->ns());
// Handle query option $maxTimeMS (not used with commands).
curop.setMaxTimeMicros(static_cast<unsigned long long>(pq.getMaxTimeMS()) * 1000);
txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point.
// uassert if we are not on a primary, and not a secondary with SlaveOk query parameter set.
bool slaveOK = pq.getOptions().slaveOk || pq.hasReadPref();
status = repl::getGlobalReplicationCoordinator()->checkCanServeReadsFor(
txn,
NamespaceString(cq->ns()),
slaveOK);
uassertStatusOK(status);
// If this exists, the collection is sharded.
// If it doesn't exist, we can assume we're not sharded.
// If we're sharded, we might encounter data that is not consistent with our sharding state.
// We must ignore this data.
CollectionMetadataPtr collMetadata;
if (!shardingState.needCollectionMetadata(pq.ns())) {
collMetadata = CollectionMetadataPtr();
}
else {
collMetadata = shardingState.getCollectionMetadata(pq.ns());
}
// Run the query.
// bb is used to hold query results
// this buffer should contain either requested documents per query or
// explain information, but not both
BufBuilder bb(32768);
bb.skip(sizeof(QueryResult::Value));
// How many results have we obtained from the executor?
int numResults = 0;
// If we're replaying the oplog, we save the last time that we read.
OpTime slaveReadTill;
// Do we save the PlanExecutor in a ClientCursor for getMore calls later?
bool saveClientCursor = false;
BSONObj obj;
PlanExecutor::ExecState state;
// uint64_t numMisplacedDocs = 0;
// Get summary info about which plan the executor is using.
curop.debug().planSummary = Explain::getPlanSummary(exec.get());
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
// Add result to output buffer.
bb.appendBuf((void*)obj.objdata(), obj.objsize());
// Count the result.
++numResults;
// Possibly note slave's position in the oplog.
if (pq.getOptions().oplogReplay) {
BSONElement e = obj["ts"];
if (Date == e.type() || Timestamp == e.type()) {
slaveReadTill = e._opTime();
}
}
// TODO: only one type of 2d search doesn't support this. We need a way to pull it out
// of CanonicalQuery. :(
const bool supportsGetMore = true;
if (!supportsGetMore && (enough(pq, numResults)
|| bb.len() >= MaxBytesToReturnToClientAtOnce)) {
break;
}
else if (enoughForFirstBatch(pq, numResults, bb.len())) {
QLOG() << "Enough for first batch, wantMore=" << pq.wantMore()
<< " numToReturn=" << pq.getNumToReturn()
<< " numResults=" << numResults
<< endl;
// If only one result requested assume it's a findOne() and don't save the cursor.
if (pq.wantMore() && 1 != pq.getNumToReturn()) {
QLOG() << " executor EOF=" << exec->isEOF() << endl;
saveClientCursor = !exec->isEOF();
}
break;
}
}
// If we cache the executor later, we want to deregister it as it receives notifications
// anyway by virtue of being cached.
//
// If we don't cache the executor later, we are deleting it, so it must be deregistered.
//
// So, no matter what, deregister the executor.
exec->deregisterExec();
// Caller expects exceptions thrown in certain cases.
if (PlanExecutor::EXEC_ERROR == state) {
scoped_ptr<PlanStageStats> stats(exec->getStats());
error() << "Plan executor error, stats: "
<< Explain::statsToBSON(*stats);
uasserted(17144, "Executor error: " + WorkingSetCommon::toStatusString(obj));
}
// Why save a dead executor?
if (PlanExecutor::DEAD == state) {
saveClientCursor = false;
}
else if (pq.getOptions().tailable) {
// If we're tailing a capped collection, we don't bother saving the cursor if the
// collection is empty. Otherwise, the semantics of the tailable cursor is that the
// client will keep trying to read from it. So we'll keep it around.
if (collection && collection->numRecords(txn) != 0 && pq.getNumToReturn() != 1) {
saveClientCursor = true;
}
}
// TODO(greg): This will go away soon.
if (!shardingState.getVersion(pq.ns()).isWriteCompatibleWith(shardingVersionAtStart)) {
// if the version changed during the query we might be missing some data and its safe to
// send this as mongos can resend at this point
throw SendStaleConfigException(pq.ns(), "version changed during initial query",
shardingVersionAtStart,
shardingState.getVersion(pq.ns()));
}
const logger::LogComponent queryLogComponent = logger::LogComponent::kQuery;
const logger::LogSeverity logLevelOne = logger::LogSeverity::Debug(1);
PlanSummaryStats summaryStats;
Explain::getSummaryStats(exec.get(), &summaryStats);
curop.debug().ntoskip = pq.getSkip();
curop.debug().nreturned = numResults;
curop.debug().scanAndOrder = summaryStats.hasSortStage;
curop.debug().nscanned = summaryStats.totalKeysExamined;
curop.debug().nscannedObjects = summaryStats.totalDocsExamined;
curop.debug().idhack = summaryStats.isIdhack;
// Set debug information for consumption by the profiler.
if (dbProfilingLevel > 0 ||
curop.elapsedMillis() > serverGlobalParams.slowMS ||
logger::globalLogDomain()->shouldLog(queryLogComponent, logLevelOne)) {
// Get BSON stats.
scoped_ptr<PlanStageStats> execStats(exec->getStats());
BSONObjBuilder statsBob;
Explain::statsToBSON(*execStats, &statsBob);
curop.debug().execStats.set(statsBob.obj());
// Replace exec stats with plan summary if stats cannot fit into CachedBSONObj.
if (curop.debug().execStats.tooBig() && !curop.debug().planSummary.empty()) {
BSONObjBuilder bob;
bob.append("summary", curop.debug().planSummary.toString());
curop.debug().execStats.set(bob.done());
}
}
long long ccId = 0;
if (saveClientCursor) {
// We won't use the executor until it's getMore'd.
exec->saveState();
// Allocate a new ClientCursor. We don't have to worry about leaking it as it's
// inserted into a global map by its ctor.
ClientCursor* cc = new ClientCursor(collection, exec.get(),
cq->getParsed().getOptions().toInt(),
cq->getParsed().getFilter());
ccId = cc->cursorid();
if (fromDBDirectClient) {
cc->setUnownedRecoveryUnit(txn->recoveryUnit());
}
else if (state == PlanExecutor::IS_EOF && pq.getOptions().tailable) {
// Don't stash the RU for tailable cursors at EOF, let them get a new RU on their
// next getMore.
}
else {
// We stash away the RecoveryUnit in the ClientCursor. It's used for subsequent
// getMore requests. The calling OpCtx gets a fresh RecoveryUnit.
cc->setOwnedRecoveryUnit(txn->releaseRecoveryUnit());
StorageEngine* storageEngine = getGlobalEnvironment()->getGlobalStorageEngine();
txn->setRecoveryUnit(storageEngine->newRecoveryUnit(txn));
}
QLOG() << "caching executor with cursorid " << ccId
<< " after returning " << numResults << " results" << endl;
// ClientCursor takes ownership of executor. Release to make sure it's not deleted.
exec.release();
// TODO document
if (pq.getOptions().oplogReplay && !slaveReadTill.isNull()) {
cc->slaveReadTill(slaveReadTill);
}
// TODO document
if (pq.getOptions().exhaust) {
curop.debug().exhaust = true;
}
// Set attributes for getMore.
cc->setCollMetadata(collMetadata);
cc->setPos(numResults);
// If the query had a time limit, remaining time is "rolled over" to the cursor (for
// use by future getmore ops).
cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
}
else {
QLOG() << "Not caching executor but returning " << numResults << " results.\n";
}
// Add the results from the query into the output buffer.
result.appendData(bb.buf(), bb.len());
bb.decouple();
// Fill out the output buffer's header.
QueryResult::View qr = result.header().view2ptr();
qr.setCursorId(ccId);
curop.debug().cursorid = (0 == ccId ? -1 : ccId);
qr.setResultFlagsToOk();
qr.msgdata().setOperation(opReply);
qr.setStartingFrom(0);
qr.setNReturned(numResults);
// curop.debug().exhaust is set above.
return curop.debug().exhaust ? pq.ns() : "";
}
Status getExecutorDistinct(OperationContext* txn,
Collection* collection,
const BSONObj& query,
const std::string& field,
PlanExecutor** out) {
// This should'a been checked by the distinct command.
invariant(collection);
// TODO: check for idhack here?
// When can we do a fast distinct hack?
// 1. There is a plan with just one leaf and that leaf is an ixscan.
// 2. The ixscan indexes the field we're interested in.
// 2a: We are correct if the index contains the field but for now we look for prefix.
// 3. The query is covered/no fetch.
//
// We go through normal planning (with limited parameters) to see if we can produce
// a soln with the above properties.
QueryPlannerParams plannerParams;
plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN;
IndexCatalog::IndexIterator ii = collection->getIndexCatalog()->getIndexIterator(false);
while (ii.more()) {
const IndexDescriptor* desc = ii.next();
// The distinct hack can work if any field is in the index but it's not always clear
// if it's a win unless it's the first field.
if (desc->keyPattern().firstElement().fieldName() == field) {
plannerParams.indices.push_back(IndexEntry(desc->keyPattern(),
desc->getAccessMethodName(),
desc->isMultikey(),
desc->isSparse(),
desc->indexName(),
desc->infoObj()));
}
}
const WhereCallbackReal whereCallback(collection->ns().db());
// If there are no suitable indices for the distinct hack bail out now into regular planning
// with no projection.
if (plannerParams.indices.empty()) {
CanonicalQuery* cq;
Status status = CanonicalQuery::canonicalize(
collection->ns().ns(), query, &cq, whereCallback);
if (!status.isOK()) {
return status;
}
// Takes ownership of 'cq'.
return getExecutor(txn, collection, cq, out);
}
//
// If we're here, we have an index prefixed by the field we're distinct-ing over.
//
// Applying a projection allows the planner to try to give us covered plans that we can turn
// into the projection hack. getDistinctProjection deals with .find() projection semantics
// (ie _id:1 being implied by default).
BSONObj projection = getDistinctProjection(field);
// Apply a projection of the key. Empty BSONObj() is for the sort.
CanonicalQuery* cq;
Status status = CanonicalQuery::canonicalize(collection->ns().ns(),
query,
BSONObj(),
projection,
&cq,
whereCallback);
if (!status.isOK()) {
return status;
}
auto_ptr<CanonicalQuery> autoCq(cq);
// If there's no query, we can just distinct-scan one of the indices.
// Not every index in plannerParams.indices may be suitable. Refer to
// getDistinctNodeIndex().
size_t distinctNodeIndex = 0;
if (query.isEmpty() &&
getDistinctNodeIndex(plannerParams.indices, field, &distinctNodeIndex)) {
DistinctNode* dn = new DistinctNode();
dn->indexKeyPattern = plannerParams.indices[distinctNodeIndex].keyPattern;
dn->direction = 1;
IndexBoundsBuilder::allValuesBounds(dn->indexKeyPattern, &dn->bounds);
dn->fieldNo = 0;
QueryPlannerParams params;
// Takes ownership of 'dn'.
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(*cq, params, dn);
invariant(soln);
LOG(2) << "Using fast distinct: " << cq->toStringShort()
<< ", planSummary: " << getPlanSummary(*soln);
WorkingSet* ws = new WorkingSet();
PlanStage* root;
verify(StageBuilder::build(txn, collection, *soln, ws, &root));
// Takes ownership of its arguments (except for 'collection').
*out = new PlanExecutor(ws, root, soln, autoCq.release(), collection);
return Status::OK();
}
// See if we can answer the query in a fast-distinct compatible fashion.
vector<QuerySolution*> solutions;
status = QueryPlanner::plan(*cq, plannerParams, &solutions);
if (!status.isOK()) {
return getExecutor(txn, collection, autoCq.release(), out);
}
// We look for a solution that has an ixscan we can turn into a distinctixscan
for (size_t i = 0; i < solutions.size(); ++i) {
if (turnIxscanIntoDistinctIxscan(solutions[i], field)) {
// Great, we can use solutions[i]. Clean up the other QuerySolution(s).
for (size_t j = 0; j < solutions.size(); ++j) {
if (j != i) {
delete solutions[j];
}
}
LOG(2) << "Using fast distinct: " << cq->toStringShort()
<< ", planSummary: " << getPlanSummary(*solutions[i]);
// Build and return the SSR over solutions[i].
WorkingSet* ws = new WorkingSet();
PlanStage* root;
verify(StageBuilder::build(txn, collection, *solutions[i], ws, &root));
// Takes ownership of its arguments (except for 'collection').
*out = new PlanExecutor(ws, root, solutions[i], autoCq.release(), collection);
return Status::OK();
}
}
// If we're here, the planner made a soln with the restricted index set but we couldn't
// translate any of them into a distinct-compatible soln. So, delete the solutions and just
// go through normal planning.
for (size_t i = 0; i < solutions.size(); ++i) {
delete solutions[i];
}
// We drop the projection from the 'cq'. Unfortunately this is not trivial.
status = CanonicalQuery::canonicalize(collection->ns().ns(), query, &cq, whereCallback);
if (!status.isOK()) {
return status;
}
autoCq.reset(cq);
// Takes ownership of 'autoCq'.
return getExecutor(txn, collection, autoCq.release(), out);
}
std::shared_ptr<Executor> getCPUExecutor() {
return getExecutor(
globalCPUExecutor,
globalInlineExecutor,
globalCPUExecutorLock);
}
void Mechanoid::enterBuilding(detail::MapBuilding *bld)
{
if (!bld)
return;
auto mmb = bld->getModificationMapBuilding();
if (!mmb)
return;
building = mmb;
if (!isPlayer())
{
// handle bot's building visit
return;
}
// player is now officially in the building
// we need to run scripts, setup texts etc.
auto e = getEngine();
// set building, mechanoid
auto bm = e->getBuildingMenu();
bm->setCurrentBuilding(mmb);
bm->setCurrentMechanoid(this);
bm->clearText();
// now run scripts
auto se = mmb->map->modification->getScriptEngine();
path script_file = path("maps") / mmb->map->script_dir.toString() / mmb->script_name.toString();
auto s = se->getScript(script_file.string());
// set player visit
auto iter = player->buildings.find_if([mmb](const auto &vb)
{
return vb->building.get() == mmb;
});
if (iter == player->buildings.end())
{
auto vb = e->getStorage()->modificationPlayerBuildings.createAtEnd();
vb->player = player;
vb->building = mmb;
vb->know_location = true;
vb->visited = true;
player->buildings.insert(vb);
}
else
{
(*iter)->know_location = true;
(*iter)->visited = true;
}
// set script data
s->data.script = s;
s->data.player = player;
s->data.building = mmb;
// set script for possible callbacks from building menu
bm->setCurrentScriptCallback([s](const auto &fn, const auto ¶ms) mutable { s->call(fn, params); });
// register available building quests
s->RegisterQuests();
// main script call
s->OnEnterBuilding();
// update building menu
bm->refresh();
e->ShowBuildingMenu();
// workaround
{
static Executor e;
getExecutor(&e);
}
// do async save to not freeze the game
// the ideal algorithm here is:
// 1. freeze the game
// 2. async({ save(); unfreeze(); });
getExecutor().push([e]()
{
e->saveAuto();
});
}
