TEST_F(DocumentSourceFacetTest, MultipleFacetsShouldSeeTheSameDocuments) {
auto ctx = getExpCtx();
auto firstDummy = DocumentSourcePassthrough::create();
auto firstPipeline = uassertStatusOK(Pipeline::create({firstDummy}, ctx));
auto secondDummy = DocumentSourcePassthrough::create();
auto secondPipeline = uassertStatusOK(Pipeline::create({secondDummy}, ctx));
auto facetStage =
DocumentSourceFacet::create({{"first", firstPipeline}, {"second", secondPipeline}}, ctx);
deque<DocumentSource::GetNextResult> inputs = {
Document{{"_id", 0}}, Document{{"_id", 1}}, Document{{"_id", 2}}};
auto mock = DocumentSourceMock::create(inputs);
facetStage->setSource(mock.get());
auto output = facetStage->getNext();
// The output fields are in no guaranteed order.
vector<Value> expectedOutputs;
for (auto&& input : inputs) {
expectedOutputs.emplace_back(input.releaseDocument());
}
ASSERT(output.isAdvanced());
ASSERT_EQ(output.getDocument().size(), 2UL);
ASSERT_VALUE_EQ(output.getDocument()["first"], Value(expectedOutputs));
ASSERT_VALUE_EQ(output.getDocument()["second"], Value(expectedOutputs));
// Should be exhausted now.
ASSERT(facetStage->getNext().isEOF());
ASSERT(facetStage->getNext().isEOF());
ASSERT(facetStage->getNext().isEOF());
}
TEST_F(DocumentSourceFacetTest, SingleFacetShouldReceiveAllDocuments) {
auto ctx = getExpCtx();
auto dummy = DocumentSourcePassthrough::create();
auto statusWithPipeline = Pipeline::create({dummy}, ctx);
ASSERT_OK(statusWithPipeline.getStatus());
auto pipeline = std::move(statusWithPipeline.getValue());
auto facetStage = DocumentSourceFacet::create({{"results", pipeline}}, ctx);
deque<DocumentSource::GetNextResult> inputs = {
Document{{"_id", 0}}, Document{{"_id", 1}}, Document{{"_id", 2}}};
auto mock = DocumentSourceMock::create(inputs);
facetStage->setSource(mock.get());
auto output = facetStage->getNext();
ASSERT(output.isAdvanced());
ASSERT_DOCUMENT_EQ(output.getDocument(),
Document(fromjson("{results: [{_id: 0}, {_id: 1}, {_id: 2}]}")));
// Should be exhausted now.
ASSERT(facetStage->getNext().isEOF());
ASSERT(facetStage->getNext().isEOF());
ASSERT(facetStage->getNext().isEOF());
}
DocumentSource::GetNextResult DocumentSourceOut::getNext() {
pExpCtx->checkForInterrupt();
if (_done) {
return GetNextResult::makeEOF();
}
if (!_initialized) {
initialize();
}
// Insert all documents into temp collection, batching to perform vectored inserts.
vector<BSONObj> bufferedObjects;
int bufferedBytes = 0;
auto nextInput = pSource->getNext();
for (; nextInput.isAdvanced(); nextInput = pSource->getNext()) {
BSONObj toInsert = nextInput.releaseDocument().toBson();
bufferedBytes += toInsert.objsize();
if (!bufferedObjects.empty() && (bufferedBytes > BSONObjMaxUserSize ||
bufferedObjects.size() >= write_ops::kMaxWriteBatchSize)) {
spill(bufferedObjects);
bufferedObjects.clear();
bufferedBytes = toInsert.objsize();
}
bufferedObjects.push_back(toInsert);
}
if (!bufferedObjects.empty())
spill(bufferedObjects);
switch (nextInput.getStatus()) {
case GetNextResult::ReturnStatus::kAdvanced: {
MONGO_UNREACHABLE; // We consumed all advances above.
}
case GetNextResult::ReturnStatus::kPauseExecution: {
return nextInput; // Propagate the pause.
}
case GetNextResult::ReturnStatus::kEOF: {
auto renameCommandObj =
BSON("renameCollection" << _tempNs.ns() << "to" << _outputNs.ns() << "dropTarget"
<< true);
auto status = pExpCtx->mongoProcessInterface->renameIfOptionsAndIndexesHaveNotChanged(
pExpCtx->opCtx, renameCommandObj, _outputNs, _originalOutOptions, _originalIndexes);
uassert(16997, str::stream() << "$out failed: " << status.reason(), status.isOK());
// We don't need to drop the temp collection in our destructor if the rename succeeded.
_tempNs = {};
_done = true;
// $out doesn't currently produce any outputs.
return nextInput;
}
}
MONGO_UNREACHABLE;
}
DocumentSource::GetNextResult Exchange::ExchangeBuffer::getNext() {
invariant(!_buffer.empty());
auto result = std::move(_buffer.front());
_buffer.pop_front();
if (result.isAdvanced()) {
_bytesInBuffer -= result.getDocument().getApproximateSize();
}
return result;
}
DocumentSource::GetNextResult DocumentSourceCheckInvalidate::getNext() {
pExpCtx->checkForInterrupt();
invariant(!pExpCtx->inMongos);
if (_queuedInvalidate) {
const auto res = DocumentSource::GetNextResult(std::move(_queuedInvalidate.get()));
_queuedInvalidate.reset();
return res;
}
auto nextInput = pSource->getNext();
if (!nextInput.isAdvanced())
return nextInput;
auto doc = nextInput.getDocument();
const auto& kOperationTypeField = DSCS::kOperationTypeField;
DSCS::checkValueType(doc[kOperationTypeField], kOperationTypeField, BSONType::String);
auto operationType = doc[kOperationTypeField].getString();
// If this command should invalidate the stream, generate an invalidate entry and queue it up
// to be returned after the notification of this command. The new entry will have a nearly
// identical resume token to the notification for the command, except with an extra flag
// indicating that the token is from an invalidate. This flag is necessary to disambiguate
// the two tokens, and thus preserve a total ordering on the stream.
// As a special case, if a client receives an invalidate like this one and then wants to
// start a new stream after the invalidate, they can use the "startAfter" option, in which
// case '_ignoreFirstInvalidate' will be set, and we should ignore (AKA not generate) the
// very first invalidation.
if (isInvalidatingCommand(pExpCtx, operationType) && !_ignoreFirstInvalidate) {
auto resumeTokenData = ResumeToken::parse(doc[DSCS::kIdField].getDocument()).getData();
resumeTokenData.fromInvalidate = ResumeTokenData::FromInvalidate::kFromInvalidate;
MutableDocument result(Document{{DSCS::kIdField, ResumeToken(resumeTokenData).toDocument()},
{DSCS::kOperationTypeField, DSCS::kInvalidateOpType},
{DSCS::kClusterTimeField, doc[DSCS::kClusterTimeField]}});
// If we're in a sharded environment, we'll need to merge the results by their sort key, so
// add that as metadata.
result.copyMetaDataFrom(doc);
_queuedInvalidate = result.freeze();
}
// Regardless of whether the first document we see is an invalidating command, we only skip the
// first invalidate for streams with the 'startAfter' option, so we should not skip any
// invalidates that come after the first one.
_ignoreFirstInvalidate = false;
return nextInput;
}
size_t Exchange::loadNextBatch() {
auto input = _pipeline->getSources().back()->getNext();
for (; input.isAdvanced(); input = _pipeline->getSources().back()->getNext()) {
// We have a document and we will deliver it to a consumer(s) based on the policy.
switch (_policy) {
case ExchangePolicyEnum::kBroadcast: {
bool full = false;
// The document is sent to all consumers.
for (auto& c : _consumers) {
full = c->appendDocument(input, _maxBufferSize);
}
if (full)
return 0;
} break;
case ExchangePolicyEnum::kRoundRobin: {
size_t target = _roundRobinCounter;
_roundRobinCounter = (_roundRobinCounter + 1) % _consumers.size();
if (_consumers[target]->appendDocument(std::move(input), _maxBufferSize))
return target;
} break;
case ExchangePolicyEnum::kKeyRange: {
size_t target = getTargetConsumer(input.getDocument());
bool full = _consumers[target]->appendDocument(std::move(input), _maxBufferSize);
if (full && _orderPreserving) {
// TODO send the high watermark here.
}
if (full)
return target;
} break;
default:
MONGO_UNREACHABLE;
}
}
invariant(input.isEOF());
// We have reached the end so send EOS to all consumers.
for (auto& c : _consumers) {
c->appendDocument(input, _maxBufferSize);
}
return kInvalidThreadId;
}
TEST_F(DocumentSourceFacetTest, ShouldBeAbleToEvaluateMultipleStagesWithinOneSubPipeline) {
auto ctx = getExpCtx();
auto firstDummy = DocumentSourcePassthrough::create();
auto secondDummy = DocumentSourcePassthrough::create();
auto pipeline = uassertStatusOK(Pipeline::create({firstDummy, secondDummy}, ctx));
auto facetStage = DocumentSourceFacet::create({{"subPipe", pipeline}}, ctx);
deque<DocumentSource::GetNextResult> inputs = {Document{{"_id", 0}}, Document{{"_id", 1}}};
auto mock = DocumentSourceMock::create(inputs);
facetStage->setSource(mock.get());
auto output = facetStage->getNext();
ASSERT(output.isAdvanced());
ASSERT_DOCUMENT_EQ(output.getDocument(), Document(fromjson("{subPipe: [{_id: 0}, {_id: 1}]}")));
}
DocumentSource::GetNextResult DocumentSourceCloseCursor::getNext() {
pExpCtx->checkForInterrupt();
// Close cursor if we have returned an invalidate entry.
if (_shouldCloseCursor) {
uasserted(ErrorCodes::CloseChangeStream, "Change stream has been invalidated");
}
if (_queuedInvalidate) {
_shouldCloseCursor = true;
return DocumentSource::GetNextResult(std::move(_queuedInvalidate.get()));
}
auto nextInput = pSource->getNext();
if (!nextInput.isAdvanced())
return nextInput;
auto doc = nextInput.getDocument();
const auto& kOperationTypeField = DocumentSourceChangeStream::kOperationTypeField;
DocumentSourceChangeStream::checkValueType(
doc[kOperationTypeField], kOperationTypeField, BSONType::String);
auto operationType = doc[kOperationTypeField].getString();
if (operationType == DocumentSourceChangeStream::kInvalidateOpType) {
// Pass the invalidation forward, so that it can be included in the results, or
// filtered/transformed by further stages in the pipeline, then throw an exception
// to close the cursor on the next call to getNext().
_shouldCloseCursor = true;
}
// Check if this is an invalidating command and the next entry should be an "invalidate".
if (isInvalidatingCommand(pExpCtx, operationType)) {
_queuedInvalidate = Document{
{DocumentSourceChangeStream::kIdField, doc[DocumentSourceChangeStream::kIdField]},
{DocumentSourceChangeStream::kClusterTimeField,
doc[DocumentSourceChangeStream::kClusterTimeField]},
{DocumentSourceChangeStream::kOperationTypeField, "invalidate"_sd}};
}
return nextInput;
}
TEST_F(DocumentSourceMatchTest, ShouldCorrectlyJoinWithSubsequentMatch) {
const auto match = DocumentSourceMatch::create(BSON("a" << 1), getExpCtx());
const auto secondMatch = DocumentSourceMatch::create(BSON("b" << 1), getExpCtx());
match->joinMatchWith(secondMatch);
const auto mock = DocumentSourceMock::create({Document{{"a", 1}, {"b", 1}},
Document{{"a", 2}, {"b", 1}},
Document{{"a", 1}, {"b", 2}},
Document{{"a", 2}, {"b", 2}}});
match->setSource(mock.get());
// The first result should match.
auto next = match->getNext();
ASSERT_TRUE(next.isAdvanced());
ASSERT_DOCUMENT_EQ(next.releaseDocument(), (Document{{"a", 1}, {"b", 1}}));
// The rest should not match.
ASSERT_TRUE(match->getNext().isEOF());
ASSERT_TRUE(match->getNext().isEOF());
ASSERT_TRUE(match->getNext().isEOF());
}
TEST_F(DocumentSourceFacetTest,
ShouldCorrectlyHandleSubPipelinesYieldingDifferentNumbersOfResults) {
auto ctx = getExpCtx();
auto passthrough = DocumentSourcePassthrough::create();
auto passthroughPipe = uassertStatusOK(Pipeline::create({passthrough}, ctx));
auto limit = DocumentSourceLimit::create(ctx, 1);
auto limitedPipe = uassertStatusOK(Pipeline::create({limit}, ctx));
auto facetStage =
DocumentSourceFacet::create({{"all", passthroughPipe}, {"first", limitedPipe}}, ctx);
deque<DocumentSource::GetNextResult> inputs = {
Document{{"_id", 0}}, Document{{"_id", 1}}, Document{{"_id", 2}}, Document{{"_id", 3}}};
auto mock = DocumentSourceMock::create(inputs);
facetStage->setSource(mock.get());
vector<Value> expectedPassthroughOutput;
for (auto&& input : inputs) {
expectedPassthroughOutput.emplace_back(input.getDocument());
}
auto output = facetStage->getNext();
// The output fields are in no guaranteed order.
ASSERT(output.isAdvanced());
ASSERT_EQ(output.getDocument().size(), 2UL);
ASSERT_VALUE_EQ(output.getDocument()["all"], Value(expectedPassthroughOutput));
ASSERT_VALUE_EQ(output.getDocument()["first"],
Value(vector<Value>{Value(expectedPassthroughOutput.front())}));
// Should be exhausted now.
ASSERT(facetStage->getNext().isEOF());
ASSERT(facetStage->getNext().isEOF());
ASSERT(facetStage->getNext().isEOF());
}
