TEST_F(DocumentSourceFacetTest, MultipleFacetsShouldSeeTheSameDocuments) { auto ctx = getExpCtx(); auto firstDummy = DocumentSourcePassthrough::create(); auto firstPipeline = uassertStatusOK(Pipeline::create({firstDummy}, ctx)); auto secondDummy = DocumentSourcePassthrough::create(); auto secondPipeline = uassertStatusOK(Pipeline::create({secondDummy}, ctx)); auto facetStage = DocumentSourceFacet::create({{"first", firstPipeline}, {"second", secondPipeline}}, ctx); deque<DocumentSource::GetNextResult> inputs = { Document{{"_id", 0}}, Document{{"_id", 1}}, Document{{"_id", 2}}}; auto mock = DocumentSourceMock::create(inputs); facetStage->setSource(mock.get()); auto output = facetStage->getNext(); // The output fields are in no guaranteed order. vector<Value> expectedOutputs; for (auto&& input : inputs) { expectedOutputs.emplace_back(input.releaseDocument()); } ASSERT(output.isAdvanced()); ASSERT_EQ(output.getDocument().size(), 2UL); ASSERT_VALUE_EQ(output.getDocument()["first"], Value(expectedOutputs)); ASSERT_VALUE_EQ(output.getDocument()["second"], Value(expectedOutputs)); // Should be exhausted now. ASSERT(facetStage->getNext().isEOF()); ASSERT(facetStage->getNext().isEOF()); ASSERT(facetStage->getNext().isEOF()); }
TEST_F(DocumentSourceFacetTest, SingleFacetShouldReceiveAllDocuments) { auto ctx = getExpCtx(); auto dummy = DocumentSourcePassthrough::create(); auto statusWithPipeline = Pipeline::create({dummy}, ctx); ASSERT_OK(statusWithPipeline.getStatus()); auto pipeline = std::move(statusWithPipeline.getValue()); auto facetStage = DocumentSourceFacet::create({{"results", pipeline}}, ctx); deque<DocumentSource::GetNextResult> inputs = { Document{{"_id", 0}}, Document{{"_id", 1}}, Document{{"_id", 2}}}; auto mock = DocumentSourceMock::create(inputs); facetStage->setSource(mock.get()); auto output = facetStage->getNext(); ASSERT(output.isAdvanced()); ASSERT_DOCUMENT_EQ(output.getDocument(), Document(fromjson("{results: [{_id: 0}, {_id: 1}, {_id: 2}]}"))); // Should be exhausted now. ASSERT(facetStage->getNext().isEOF()); ASSERT(facetStage->getNext().isEOF()); ASSERT(facetStage->getNext().isEOF()); }
DocumentSource::GetNextResult DocumentSourceOut::getNext() { pExpCtx->checkForInterrupt(); if (_done) { return GetNextResult::makeEOF(); } if (!_initialized) { initialize(); } // Insert all documents into temp collection, batching to perform vectored inserts. vector<BSONObj> bufferedObjects; int bufferedBytes = 0; auto nextInput = pSource->getNext(); for (; nextInput.isAdvanced(); nextInput = pSource->getNext()) { BSONObj toInsert = nextInput.releaseDocument().toBson(); bufferedBytes += toInsert.objsize(); if (!bufferedObjects.empty() && (bufferedBytes > BSONObjMaxUserSize || bufferedObjects.size() >= write_ops::kMaxWriteBatchSize)) { spill(bufferedObjects); bufferedObjects.clear(); bufferedBytes = toInsert.objsize(); } bufferedObjects.push_back(toInsert); } if (!bufferedObjects.empty()) spill(bufferedObjects); switch (nextInput.getStatus()) { case GetNextResult::ReturnStatus::kAdvanced: { MONGO_UNREACHABLE; // We consumed all advances above. } case GetNextResult::ReturnStatus::kPauseExecution: { return nextInput; // Propagate the pause. } case GetNextResult::ReturnStatus::kEOF: { auto renameCommandObj = BSON("renameCollection" << _tempNs.ns() << "to" << _outputNs.ns() << "dropTarget" << true); auto status = pExpCtx->mongoProcessInterface->renameIfOptionsAndIndexesHaveNotChanged( pExpCtx->opCtx, renameCommandObj, _outputNs, _originalOutOptions, _originalIndexes); uassert(16997, str::stream() << "$out failed: " << status.reason(), status.isOK()); // We don't need to drop the temp collection in our destructor if the rename succeeded. _tempNs = {}; _done = true; // $out doesn't currently produce any outputs. return nextInput; } } MONGO_UNREACHABLE; }
DocumentSource::GetNextResult Exchange::ExchangeBuffer::getNext() { invariant(!_buffer.empty()); auto result = std::move(_buffer.front()); _buffer.pop_front(); if (result.isAdvanced()) { _bytesInBuffer -= result.getDocument().getApproximateSize(); } return result; }
DocumentSource::GetNextResult DocumentSourceCheckInvalidate::getNext() { pExpCtx->checkForInterrupt(); invariant(!pExpCtx->inMongos); if (_queuedInvalidate) { const auto res = DocumentSource::GetNextResult(std::move(_queuedInvalidate.get())); _queuedInvalidate.reset(); return res; } auto nextInput = pSource->getNext(); if (!nextInput.isAdvanced()) return nextInput; auto doc = nextInput.getDocument(); const auto& kOperationTypeField = DSCS::kOperationTypeField; DSCS::checkValueType(doc[kOperationTypeField], kOperationTypeField, BSONType::String); auto operationType = doc[kOperationTypeField].getString(); // If this command should invalidate the stream, generate an invalidate entry and queue it up // to be returned after the notification of this command. The new entry will have a nearly // identical resume token to the notification for the command, except with an extra flag // indicating that the token is from an invalidate. This flag is necessary to disambiguate // the two tokens, and thus preserve a total ordering on the stream. // As a special case, if a client receives an invalidate like this one and then wants to // start a new stream after the invalidate, they can use the "startAfter" option, in which // case '_ignoreFirstInvalidate' will be set, and we should ignore (AKA not generate) the // very first invalidation. if (isInvalidatingCommand(pExpCtx, operationType) && !_ignoreFirstInvalidate) { auto resumeTokenData = ResumeToken::parse(doc[DSCS::kIdField].getDocument()).getData(); resumeTokenData.fromInvalidate = ResumeTokenData::FromInvalidate::kFromInvalidate; MutableDocument result(Document{{DSCS::kIdField, ResumeToken(resumeTokenData).toDocument()}, {DSCS::kOperationTypeField, DSCS::kInvalidateOpType}, {DSCS::kClusterTimeField, doc[DSCS::kClusterTimeField]}}); // If we're in a sharded environment, we'll need to merge the results by their sort key, so // add that as metadata. result.copyMetaDataFrom(doc); _queuedInvalidate = result.freeze(); } // Regardless of whether the first document we see is an invalidating command, we only skip the // first invalidate for streams with the 'startAfter' option, so we should not skip any // invalidates that come after the first one. _ignoreFirstInvalidate = false; return nextInput; }
size_t Exchange::loadNextBatch() { auto input = _pipeline->getSources().back()->getNext(); for (; input.isAdvanced(); input = _pipeline->getSources().back()->getNext()) { // We have a document and we will deliver it to a consumer(s) based on the policy. switch (_policy) { case ExchangePolicyEnum::kBroadcast: { bool full = false; // The document is sent to all consumers. for (auto& c : _consumers) { full = c->appendDocument(input, _maxBufferSize); } if (full) return 0; } break; case ExchangePolicyEnum::kRoundRobin: { size_t target = _roundRobinCounter; _roundRobinCounter = (_roundRobinCounter + 1) % _consumers.size(); if (_consumers[target]->appendDocument(std::move(input), _maxBufferSize)) return target; } break; case ExchangePolicyEnum::kKeyRange: { size_t target = getTargetConsumer(input.getDocument()); bool full = _consumers[target]->appendDocument(std::move(input), _maxBufferSize); if (full && _orderPreserving) { // TODO send the high watermark here. } if (full) return target; } break; default: MONGO_UNREACHABLE; } } invariant(input.isEOF()); // We have reached the end so send EOS to all consumers. for (auto& c : _consumers) { c->appendDocument(input, _maxBufferSize); } return kInvalidThreadId; }
TEST_F(DocumentSourceFacetTest, ShouldBeAbleToEvaluateMultipleStagesWithinOneSubPipeline) { auto ctx = getExpCtx(); auto firstDummy = DocumentSourcePassthrough::create(); auto secondDummy = DocumentSourcePassthrough::create(); auto pipeline = uassertStatusOK(Pipeline::create({firstDummy, secondDummy}, ctx)); auto facetStage = DocumentSourceFacet::create({{"subPipe", pipeline}}, ctx); deque<DocumentSource::GetNextResult> inputs = {Document{{"_id", 0}}, Document{{"_id", 1}}}; auto mock = DocumentSourceMock::create(inputs); facetStage->setSource(mock.get()); auto output = facetStage->getNext(); ASSERT(output.isAdvanced()); ASSERT_DOCUMENT_EQ(output.getDocument(), Document(fromjson("{subPipe: [{_id: 0}, {_id: 1}]}"))); }
DocumentSource::GetNextResult DocumentSourceCloseCursor::getNext() { pExpCtx->checkForInterrupt(); // Close cursor if we have returned an invalidate entry. if (_shouldCloseCursor) { uasserted(ErrorCodes::CloseChangeStream, "Change stream has been invalidated"); } if (_queuedInvalidate) { _shouldCloseCursor = true; return DocumentSource::GetNextResult(std::move(_queuedInvalidate.get())); } auto nextInput = pSource->getNext(); if (!nextInput.isAdvanced()) return nextInput; auto doc = nextInput.getDocument(); const auto& kOperationTypeField = DocumentSourceChangeStream::kOperationTypeField; DocumentSourceChangeStream::checkValueType( doc[kOperationTypeField], kOperationTypeField, BSONType::String); auto operationType = doc[kOperationTypeField].getString(); if (operationType == DocumentSourceChangeStream::kInvalidateOpType) { // Pass the invalidation forward, so that it can be included in the results, or // filtered/transformed by further stages in the pipeline, then throw an exception // to close the cursor on the next call to getNext(). _shouldCloseCursor = true; } // Check if this is an invalidating command and the next entry should be an "invalidate". if (isInvalidatingCommand(pExpCtx, operationType)) { _queuedInvalidate = Document{ {DocumentSourceChangeStream::kIdField, doc[DocumentSourceChangeStream::kIdField]}, {DocumentSourceChangeStream::kClusterTimeField, doc[DocumentSourceChangeStream::kClusterTimeField]}, {DocumentSourceChangeStream::kOperationTypeField, "invalidate"_sd}}; } return nextInput; }
TEST_F(DocumentSourceMatchTest, ShouldCorrectlyJoinWithSubsequentMatch) { const auto match = DocumentSourceMatch::create(BSON("a" << 1), getExpCtx()); const auto secondMatch = DocumentSourceMatch::create(BSON("b" << 1), getExpCtx()); match->joinMatchWith(secondMatch); const auto mock = DocumentSourceMock::create({Document{{"a", 1}, {"b", 1}}, Document{{"a", 2}, {"b", 1}}, Document{{"a", 1}, {"b", 2}}, Document{{"a", 2}, {"b", 2}}}); match->setSource(mock.get()); // The first result should match. auto next = match->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.releaseDocument(), (Document{{"a", 1}, {"b", 1}})); // The rest should not match. ASSERT_TRUE(match->getNext().isEOF()); ASSERT_TRUE(match->getNext().isEOF()); ASSERT_TRUE(match->getNext().isEOF()); }
TEST_F(DocumentSourceFacetTest, ShouldCorrectlyHandleSubPipelinesYieldingDifferentNumbersOfResults) { auto ctx = getExpCtx(); auto passthrough = DocumentSourcePassthrough::create(); auto passthroughPipe = uassertStatusOK(Pipeline::create({passthrough}, ctx)); auto limit = DocumentSourceLimit::create(ctx, 1); auto limitedPipe = uassertStatusOK(Pipeline::create({limit}, ctx)); auto facetStage = DocumentSourceFacet::create({{"all", passthroughPipe}, {"first", limitedPipe}}, ctx); deque<DocumentSource::GetNextResult> inputs = { Document{{"_id", 0}}, Document{{"_id", 1}}, Document{{"_id", 2}}, Document{{"_id", 3}}}; auto mock = DocumentSourceMock::create(inputs); facetStage->setSource(mock.get()); vector<Value> expectedPassthroughOutput; for (auto&& input : inputs) { expectedPassthroughOutput.emplace_back(input.getDocument()); } auto output = facetStage->getNext(); // The output fields are in no guaranteed order. ASSERT(output.isAdvanced()); ASSERT_EQ(output.getDocument().size(), 2UL); ASSERT_VALUE_EQ(output.getDocument()["all"], Value(expectedPassthroughOutput)); ASSERT_VALUE_EQ(output.getDocument()["first"], Value(vector<Value>{Value(expectedPassthroughOutput.front())})); // Should be exhausted now. ASSERT(facetStage->getNext().isEOF()); ASSERT(facetStage->getNext().isEOF()); ASSERT(facetStage->getNext().isEOF()); }