BSONObj ResolvedViewDefinition::asExpandedViewAggregation(const AggregationRequest& request) { BSONObjBuilder aggregationBuilder; // Perform the aggregation on the resolved namespace. aggregationBuilder.append("aggregate", collectionNss.coll()); // The new pipeline consists of two parts: first, 'pipeline' in this ResolvedViewDefinition; // then, the pipeline in 'request'. BSONArrayBuilder pipelineBuilder(aggregationBuilder.subarrayStart("pipeline")); for (auto&& item : pipeline) { pipelineBuilder.append(item); } for (auto&& item : request.getPipeline()) { pipelineBuilder.append(item); } pipelineBuilder.doneFast(); // The cursor option is always specified regardless of the presence of batchSize. if (request.getBatchSize()) { BSONObjBuilder batchSizeBuilder(aggregationBuilder.subobjStart("cursor")); batchSizeBuilder.append(AggregationRequest::kBatchSizeName, *request.getBatchSize()); batchSizeBuilder.doneFast(); } else { aggregationBuilder.append("cursor", BSONObj()); } if (request.isExplain()) aggregationBuilder.append("explain", true); return aggregationBuilder.obj(); }
/** * Round trips the pipeline through serialization by calling serialize(), then Pipeline::parse(). * fasserts if it fails to parse after being serialized. */ boost::intrusive_ptr<Pipeline> reparsePipeline( const boost::intrusive_ptr<Pipeline>& pipeline, const AggregationRequest& request, const boost::intrusive_ptr<ExpressionContext>& expCtx) { auto serialized = pipeline->serialize(); // Convert vector<Value> to vector<BSONObj>. std::vector<BSONObj> parseableSerialization; parseableSerialization.reserve(serialized.size()); for (auto&& serializedStage : serialized) { invariant(serializedStage.getType() == BSONType::Object); parseableSerialization.push_back(serializedStage.getDocument().toBson()); } auto reparsedPipeline = Pipeline::parse(parseableSerialization, expCtx); if (!reparsedPipeline.isOK()) { error() << "Aggregation command did not round trip through parsing and serialization " "correctly. Input pipeline: " << Value(request.getPipeline()).toString() << ", serialized pipeline: " << Value(serialized).toString(); fassertFailedWithStatusNoTrace(40175, reparsedPipeline.getStatus()); } reparsedPipeline.getValue()->injectExpressionContext(expCtx); reparsedPipeline.getValue()->optimizePipeline(); return reparsedPipeline.getValue(); }
bool runParsed(OperationContext* txn, const NamespaceString& origNss, const AggregationRequest& request, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result) { // For operations on views, this will be the underlying namespace. const NamespaceString& nss = request.getNamespaceString(); // Set up the ExpressionContext. intrusive_ptr<ExpressionContext> expCtx = new ExpressionContext(txn, request); expCtx->tempDir = storageGlobalParams.dbpath + "/_tmp"; // Parse the pipeline. auto statusWithPipeline = Pipeline::parse(request.getPipeline(), expCtx); if (!statusWithPipeline.isOK()) { return appendCommandStatus(result, statusWithPipeline.getStatus()); } auto pipeline = std::move(statusWithPipeline.getValue()); auto resolvedNamespaces = resolveInvolvedNamespaces(txn, pipeline, expCtx); if (!resolvedNamespaces.isOK()) { return appendCommandStatus(result, resolvedNamespaces.getStatus()); } expCtx->resolvedNamespaces = std::move(resolvedNamespaces.getValue()); unique_ptr<ClientCursorPin> pin; // either this OR the exec will be non-null unique_ptr<PlanExecutor> exec; auto curOp = CurOp::get(txn); { // This will throw if the sharding version for this connection is out of date. If the // namespace is a view, the lock will be released before re-running the aggregation. // Otherwise, the lock must be held continuously from now until we have we created both // the output ClientCursor and the input executor. This ensures that both are using the // same sharding version that we synchronize on here. This is also why we always need to // create a ClientCursor even when we aren't outputting to a cursor. See the comment on // ShardFilterStage for more details. AutoGetCollectionOrViewForRead ctx(txn, nss); Collection* collection = ctx.getCollection(); // If running $collStats on a view, we do not resolve the view since we want stats // on this view namespace. auto startsWithCollStats = [&pipeline]() { const Pipeline::SourceContainer& sources = pipeline->getSources(); return !sources.empty() && dynamic_cast<DocumentSourceCollStats*>(sources.front().get()); }; // If this is a view, resolve it by finding the underlying collection and stitching view // pipelines and this request's pipeline together. We then release our locks before // recursively calling run, which will re-acquire locks on the underlying collection. // (The lock must be released because recursively acquiring locks on the database will // prohibit yielding.) auto view = ctx.getView(); if (view && !startsWithCollStats()) { auto viewDefinition = ViewShardingCheck::getResolvedViewIfSharded(txn, ctx.getDb(), view); if (!viewDefinition.isOK()) { return appendCommandStatus(result, viewDefinition.getStatus()); } if (!viewDefinition.getValue().isEmpty()) { ViewShardingCheck::appendShardedViewStatus(viewDefinition.getValue(), &result); return false; } auto resolvedView = ctx.getDb()->getViewCatalog()->resolveView(txn, nss); if (!resolvedView.isOK()) { return appendCommandStatus(result, resolvedView.getStatus()); } // With the view resolved, we can relinquish locks. ctx.releaseLocksForView(); // Parse the resolved view into a new aggregation request. auto newCmd = resolvedView.getValue().asExpandedViewAggregation(request); if (!newCmd.isOK()) { return appendCommandStatus(result, newCmd.getStatus()); } auto newNss = resolvedView.getValue().getNamespace(); auto newRequest = AggregationRequest::parseFromBSON(newNss, newCmd.getValue()); if (!newRequest.isOK()) { return appendCommandStatus(result, newRequest.getStatus()); } bool status = runParsed( txn, origNss, newRequest.getValue(), newCmd.getValue(), errmsg, result); { // Set the namespace of the curop back to the view namespace so ctx records // stats on this view namespace on destruction. stdx::lock_guard<Client>(*txn->getClient()); curOp->setNS_inlock(nss.ns()); } return status; } // If the pipeline does not have a user-specified collation, set it from the collection // default. if (request.getCollation().isEmpty() && collection && collection->getDefaultCollator()) { invariant(!expCtx->getCollator()); expCtx->setCollator(collection->getDefaultCollator()->clone()); } // Propagate the ExpressionContext throughout all of the pipeline's stages and // expressions. pipeline->injectExpressionContext(expCtx); // The pipeline must be optimized after the correct collator has been set on it (by // injecting the ExpressionContext containing the collator). This is necessary because // optimization may make string comparisons, e.g. optimizing {$eq: [<str1>, <str2>]} to // a constant. pipeline->optimizePipeline(); if (kDebugBuild && !expCtx->isExplain && !expCtx->inShard) { // Make sure all operations round-trip through Pipeline::serialize() correctly by // re-parsing every command in debug builds. This is important because sharded // aggregations rely on this ability. Skipping when inShard because this has // already been through the transformation (and this un-sets expCtx->inShard). pipeline = reparsePipeline(pipeline, request, expCtx); } // This does mongod-specific stuff like creating the input PlanExecutor and adding // it to the front of the pipeline if needed. PipelineD::prepareCursorSource(collection, pipeline); // Create the PlanExecutor which returns results from the pipeline. The WorkingSet // ('ws') and the PipelineProxyStage ('proxy') will be owned by the created // PlanExecutor. auto ws = make_unique<WorkingSet>(); auto proxy = make_unique<PipelineProxyStage>(txn, pipeline, ws.get()); auto statusWithPlanExecutor = (NULL == collection) ? PlanExecutor::make( txn, std::move(ws), std::move(proxy), nss.ns(), PlanExecutor::YIELD_MANUAL) : PlanExecutor::make( txn, std::move(ws), std::move(proxy), collection, PlanExecutor::YIELD_MANUAL); invariant(statusWithPlanExecutor.isOK()); exec = std::move(statusWithPlanExecutor.getValue()); { auto planSummary = Explain::getPlanSummary(exec.get()); stdx::lock_guard<Client>(*txn->getClient()); curOp->setPlanSummary_inlock(std::move(planSummary)); } if (collection) { PlanSummaryStats stats; Explain::getSummaryStats(*exec, &stats); collection->infoCache()->notifyOfQuery(txn, stats.indexesUsed); } if (collection) { const bool isAggCursor = true; // enable special locking behavior ClientCursor* cursor = new ClientCursor(collection->getCursorManager(), exec.release(), nss.ns(), txn->recoveryUnit()->isReadingFromMajorityCommittedSnapshot(), 0, cmdObj.getOwned(), isAggCursor); pin.reset(new ClientCursorPin(collection->getCursorManager(), cursor->cursorid())); // Don't add any code between here and the start of the try block. } // At this point, it is safe to release the collection lock. // - In the case where we have a collection: we will need to reacquire the // collection lock later when cleaning up our ClientCursorPin. // - In the case where we don't have a collection: our PlanExecutor won't be // registered, so it will be safe to clean it up outside the lock. invariant(!exec || !collection); } try { // Unless set to true, the ClientCursor created above will be deleted on block exit. bool keepCursor = false; // Use of the aggregate command without specifying to use a cursor is deprecated. // Applications should migrate to using cursors. Cursors are strictly more useful than // outputting the results as a single document, since results that fit inside a single // BSONObj will also fit inside a single batch. // // We occasionally log a deprecation warning. if (!request.isCursorCommand()) { RARELY { warning() << "Use of the aggregate command without the 'cursor' " "option is deprecated. See " "http://dochub.mongodb.org/core/aggregate-without-cursor-deprecation."; } } // If both explain and cursor are specified, explain wins. if (expCtx->isExplain) { result << "stages" << Value(pipeline->writeExplainOps()); } else if (request.isCursorCommand()) { keepCursor = handleCursorCommand(txn, origNss.ns(), pin.get(), pin ? pin->c()->getExecutor() : exec.get(), request, result); } else { pipeline->run(result); } if (!expCtx->isExplain) { PlanSummaryStats stats; Explain::getSummaryStats(pin ? *pin->c()->getExecutor() : *exec.get(), &stats); curOp->debug().setPlanSummaryMetrics(stats); curOp->debug().nreturned = stats.nReturned; } // Clean up our ClientCursorPin, if needed. We must reacquire the collection lock // in order to do so. if (pin) { // We acquire locks here with DBLock and CollectionLock instead of using // AutoGetCollectionForRead. AutoGetCollectionForRead will throw if the // sharding version is out of date, and we don't care if the sharding version // has changed. Lock::DBLock dbLock(txn->lockState(), nss.db(), MODE_IS); Lock::CollectionLock collLock(txn->lockState(), nss.ns(), MODE_IS); if (keepCursor) { pin->release(); } else { pin->deleteUnderlying(); } } } catch (...) {
Status runAggregate(OperationContext* opCtx, const NamespaceString& origNss, const AggregationRequest& request, const BSONObj& cmdObj, BSONObjBuilder& result) { // For operations on views, this will be the underlying namespace. NamespaceString nss = request.getNamespaceString(); // The collation to use for this aggregation. boost::optional to distinguish between the case // where the collation has not yet been resolved, and where it has been resolved to nullptr. boost::optional<std::unique_ptr<CollatorInterface>> collatorToUse; unique_ptr<PlanExecutor, PlanExecutor::Deleter> exec; boost::intrusive_ptr<ExpressionContext> expCtx; Pipeline* unownedPipeline; auto curOp = CurOp::get(opCtx); { const LiteParsedPipeline liteParsedPipeline(request); // Check whether the parsed pipeline supports the given read concern. liteParsedPipeline.assertSupportsReadConcern(opCtx, request.getExplain()); if (liteParsedPipeline.hasChangeStream()) { nss = NamespaceString::kRsOplogNamespace; // If the read concern is not specified, upgrade to 'majority' and wait to make sure we // have a snapshot available. if (!repl::ReadConcernArgs::get(opCtx).hasLevel()) { const repl::ReadConcernArgs readConcern( repl::ReadConcernLevel::kMajorityReadConcern); uassertStatusOK(waitForReadConcern(opCtx, readConcern, true)); } if (!origNss.isCollectionlessAggregateNS()) { // AutoGetCollectionForReadCommand will raise an error if 'origNss' is a view. AutoGetCollectionForReadCommand origNssCtx(opCtx, origNss); // Resolve the collator to either the user-specified collation or the default // collation of the collection on which $changeStream was invoked, so that we do not // end up resolving the collation on the oplog. invariant(!collatorToUse); Collection* origColl = origNssCtx.getCollection(); collatorToUse.emplace(resolveCollator(opCtx, request, origColl)); } } const auto& pipelineInvolvedNamespaces = liteParsedPipeline.getInvolvedNamespaces(); // If emplaced, AutoGetCollectionForReadCommand will throw if the sharding version for this // connection is out of date. If the namespace is a view, the lock will be released before // re-running the expanded aggregation. boost::optional<AutoGetCollectionForReadCommand> ctx; // If this is a collectionless aggregation, we won't create 'ctx' but will still need an // AutoStatsTracker to record CurOp and Top entries. boost::optional<AutoStatsTracker> statsTracker; // If this is a collectionless aggregation with no foreign namespaces, we don't want to // acquire any locks. Otherwise, lock the collection or view. if (nss.isCollectionlessAggregateNS() && pipelineInvolvedNamespaces.empty()) { statsTracker.emplace(opCtx, nss, Top::LockType::NotLocked, 0); } else { ctx.emplace(opCtx, nss, AutoGetCollection::ViewMode::kViewsPermitted); } Collection* collection = ctx ? ctx->getCollection() : nullptr; // The collator may already have been set if this is a $changeStream pipeline. If not, // resolve the collator to either the user-specified collation or the collection default. if (!collatorToUse) { collatorToUse.emplace(resolveCollator(opCtx, request, collection)); } // If this is a view, resolve it by finding the underlying collection and stitching view // pipelines and this request's pipeline together. We then release our locks before // recursively calling runAggregate(), which will re-acquire locks on the underlying // collection. (The lock must be released because recursively acquiring locks on the // database will prohibit yielding.) if (ctx && ctx->getView() && !liteParsedPipeline.startsWithCollStats()) { invariant(nss != NamespaceString::kRsOplogNamespace); invariant(!nss.isCollectionlessAggregateNS()); // Check that the default collation of 'view' is compatible with the operation's // collation. The check is skipped if the request did not specify a collation. if (!request.getCollation().isEmpty()) { invariant(collatorToUse); // Should already be resolved at this point. if (!CollatorInterface::collatorsMatch(ctx->getView()->defaultCollator(), collatorToUse->get())) { return {ErrorCodes::OptionNotSupportedOnView, "Cannot override a view's default collation"}; } } ViewShardingCheck::throwResolvedViewIfSharded(opCtx, ctx->getDb(), ctx->getView()); auto resolvedView = ctx->getDb()->getViewCatalog()->resolveView(opCtx, nss); if (!resolvedView.isOK()) { return resolvedView.getStatus(); } // With the view & collation resolved, we can relinquish locks. ctx.reset(); // Parse the resolved view into a new aggregation request. auto newRequest = resolvedView.getValue().asExpandedViewAggregation(request); auto newCmd = newRequest.serializeToCommandObj().toBson(); auto status = runAggregate(opCtx, origNss, newRequest, newCmd, result); { // Set the namespace of the curop back to the view namespace so ctx records // stats on this view namespace on destruction. stdx::lock_guard<Client> lk(*opCtx->getClient()); curOp->setNS_inlock(nss.ns()); } return status; } invariant(collatorToUse); expCtx.reset( new ExpressionContext(opCtx, request, std::move(*collatorToUse), std::make_shared<PipelineD::MongoDInterface>(opCtx), uassertStatusOK(resolveInvolvedNamespaces(opCtx, request)))); expCtx->tempDir = storageGlobalParams.dbpath + "/_tmp"; auto session = OperationContextSession::get(opCtx); expCtx->inSnapshotReadOrMultiDocumentTransaction = session && session->inSnapshotReadOrMultiDocumentTransaction(); auto pipeline = uassertStatusOK(Pipeline::parse(request.getPipeline(), expCtx)); // Check that the view's collation matches the collation of any views involved in the // pipeline. if (!pipelineInvolvedNamespaces.empty()) { invariant(ctx); auto pipelineCollationStatus = collatorCompatibleWithPipeline( opCtx, ctx->getDb(), expCtx->getCollator(), pipeline.get()); if (!pipelineCollationStatus.isOK()) { return pipelineCollationStatus; } } pipeline->optimizePipeline(); if (kDebugBuild && !expCtx->explain && !expCtx->fromMongos) { // Make sure all operations round-trip through Pipeline::serialize() correctly by // re-parsing every command in debug builds. This is important because sharded // aggregations rely on this ability. Skipping when fromMongos because this has // already been through the transformation (and this un-sets expCtx->fromMongos). pipeline = reparsePipeline(pipeline.get(), request, expCtx); } // Prepare a PlanExecutor to provide input into the pipeline, if needed. if (liteParsedPipeline.hasChangeStream()) { // If we are using a change stream, the cursor stage should have a simple collation, // regardless of what the user's collation was. std::unique_ptr<CollatorInterface> collatorForCursor = nullptr; auto collatorStash = expCtx->temporarilyChangeCollator(std::move(collatorForCursor)); PipelineD::prepareCursorSource(collection, nss, &request, pipeline.get()); } else { PipelineD::prepareCursorSource(collection, nss, &request, pipeline.get()); } // Optimize again, since there may be additional optimizations that can be done after adding // the initial cursor stage. Note this has to be done outside the above blocks to ensure // this process uses the correct collation if it does any string comparisons. pipeline->optimizePipeline(); // Transfer ownership of the Pipeline to the PipelineProxyStage. unownedPipeline = pipeline.get(); auto ws = make_unique<WorkingSet>(); auto proxy = make_unique<PipelineProxyStage>(opCtx, std::move(pipeline), ws.get()); // This PlanExecutor will simply forward requests to the Pipeline, so does not need to // yield or to be registered with any collection's CursorManager to receive invalidations. // The Pipeline may contain PlanExecutors which *are* yielding PlanExecutors and which *are* // registered with their respective collection's CursorManager auto statusWithPlanExecutor = PlanExecutor::make(opCtx, std::move(ws), std::move(proxy), nss, PlanExecutor::NO_YIELD); invariant(statusWithPlanExecutor.isOK()); exec = std::move(statusWithPlanExecutor.getValue()); { auto planSummary = Explain::getPlanSummary(exec.get()); stdx::lock_guard<Client> lk(*opCtx->getClient()); curOp->setPlanSummary_inlock(std::move(planSummary)); } } // Having released the collection lock, we can now create a cursor that returns results from the // pipeline. This cursor owns no collection state, and thus we register it with the global // cursor manager. The global cursor manager does not deliver invalidations or kill // notifications; the underlying PlanExecutor(s) used by the pipeline will be receiving // invalidations and kill notifications themselves, not the cursor we create here. ClientCursorParams cursorParams( std::move(exec), origNss, AuthorizationSession::get(opCtx->getClient())->getAuthenticatedUserNames(), opCtx->recoveryUnit()->getReadConcernLevel(), cmdObj); if (expCtx->tailableMode == TailableModeEnum::kTailableAndAwaitData) { cursorParams.setTailable(true); cursorParams.setAwaitData(true); } auto pin = CursorManager::getGlobalCursorManager()->registerCursor(opCtx, std::move(cursorParams)); ScopeGuard cursorFreer = MakeGuard(&ClientCursorPin::deleteUnderlying, &pin); // If both explain and cursor are specified, explain wins. if (expCtx->explain) { Explain::explainPipelineExecutor( pin.getCursor()->getExecutor(), *(expCtx->explain), &result); } else { // Cursor must be specified, if explain is not. const bool keepCursor = handleCursorCommand(opCtx, origNss, pin.getCursor(), request, result); if (keepCursor) { cursorFreer.Dismiss(); } } if (!expCtx->explain) { PlanSummaryStats stats; Explain::getSummaryStats(*(pin.getCursor()->getExecutor()), &stats); curOp->debug().setPlanSummaryMetrics(stats); curOp->debug().nreturned = stats.nReturned; } // Any code that needs the cursor pinned must be inside the try block, above. return Status::OK(); }