intrusive_ptr<Pipeline> Pipeline::splitForSharded() {
/* create an initialize the shard spec we'll return */
intrusive_ptr<Pipeline> pShardPipeline(new Pipeline(pCtx));
pShardPipeline->collectionName = collectionName;
pShardPipeline->explain = explain;
// We will be removing from the front so reverse for now. undone later
// TODO: maybe sourceVector should be a deque
reverse(sourceVector.begin(), sourceVector.end());
/*
Run through the pipeline, looking for points to split it into
shard pipelines, and the rest.
*/
while(!sourceVector.empty()) {
// pop the first source
intrusive_ptr<DocumentSource> pSource = sourceVector.back();
sourceVector.pop_back();
// Check if this source is splittable
SplittableDocumentSource* splittable=
dynamic_cast<SplittableDocumentSource *>(pSource.get());
if (!splittable){
// move the source from the router sourceVector to the shard sourceVector
pShardPipeline->sourceVector.push_back(pSource);
}
else {
// split into Router and Shard sources
intrusive_ptr<DocumentSource> shardSource = splittable->getShardSource();
intrusive_ptr<DocumentSource> routerSource = splittable->getRouterSource();
if (shardSource) pShardPipeline->sourceVector.push_back(shardSource);
if (routerSource) this->sourceVector.push_back(routerSource);
// put the sourceVector back in the correct order and exit the loop
reverse(sourceVector.begin(), sourceVector.end());
break;
}
}
return pShardPipeline;
}
intrusive_ptr<Pipeline> Pipeline::splitForSharded() {
/* create an initialize the shard spec we'll return */
intrusive_ptr<Pipeline> pShardPipeline(new Pipeline(pCtx));
pShardPipeline->collectionName = collectionName;
pShardPipeline->explain = explain;
/* put the source list aside */
SourceVector tempVector(sourceVector);
sourceVector.clear();
/*
Run through the pipeline, looking for points to split it into
shard pipelines, and the rest.
*/
while(!tempVector.empty()) {
intrusive_ptr<DocumentSource> &pSource = tempVector.front();
/* hang on to this in advance, in case it is a group */
DocumentSourceGroup *pGroup =
dynamic_cast<DocumentSourceGroup *>(pSource.get());
/* move the source from the tempVector to the shard sourceVector */
pShardPipeline->sourceVector.push_back(pSource);
tempVector.erase(tempVector.begin());
/*
If we found a group, that's a split point.
*/
if (pGroup) {
/* start this pipeline with the group merger */
sourceVector.push_back(pGroup->createMerger());
/* and then add everything that remains and quit */
for(size_t tempn = tempVector.size(), tempi = 0;
tempi < tempn; ++tempi)
sourceVector.push_back(tempVector[tempi]);
break;
}
}
return pShardPipeline;
}
intrusive_ptr<Pipeline> Pipeline::splitForSharded() {
/* create an initialize the shard spec we'll return */
intrusive_ptr<Pipeline> pShardPipeline(new Pipeline(pCtx));
pShardPipeline->collectionName = collectionName;
pShardPipeline->explain = explain;
/*
Run through the pipeline, looking for points to split it into
shard pipelines, and the rest.
*/
while (!sources.empty()) {
// pop the first source
intrusive_ptr<DocumentSource> pSource = sources.front();
sources.pop_front();
// Check if this source is splittable
SplittableDocumentSource* splittable=
dynamic_cast<SplittableDocumentSource *>(pSource.get());
if (!splittable){
// move the source from the router sources to the shard sources
pShardPipeline->sources.push_back(pSource);
}
else {
// split into Router and Shard sources
intrusive_ptr<DocumentSource> shardSource = splittable->getShardSource();
intrusive_ptr<DocumentSource> routerSource = splittable->getRouterSource();
if (shardSource) pShardPipeline->sources.push_back(shardSource);
if (routerSource) this->sources.push_front(routerSource);
break;
}
}
return pShardPipeline;
}
bool PipelineCommand::executePipeline(
BSONObjBuilder &result, string &errmsg, const string &ns,
intrusive_ptr<Pipeline> &pPipeline,
intrusive_ptr<DocumentSourceCursor> &pSource,
intrusive_ptr<ExpressionContext> &pCtx) {
/* this is the normal non-debug path */
if (!pPipeline->getSplitMongodPipeline())
return pPipeline->run(result, errmsg, pSource);
/* setup as if we're in the router */
pCtx->setInRouter(true);
/*
Here, we'll split the pipeline in the same way we would for sharding,
for testing purposes.
Run the shard pipeline first, then feed the results into the remains
of the existing pipeline.
Start by splitting the pipeline.
*/
intrusive_ptr<Pipeline> pShardSplit(
pPipeline->splitForSharded());
/*
Write the split pipeline as we would in order to transmit it to
the shard servers.
*/
BSONObjBuilder shardBuilder;
pShardSplit->toBson(&shardBuilder);
BSONObj shardBson(shardBuilder.done());
DEV (log() << "\n---- shardBson\n" <<
shardBson.jsonString(Strict, 1) << "\n----\n").flush();
/* for debugging purposes, show what the pipeline now looks like */
DEV {
BSONObjBuilder pipelineBuilder;
pPipeline->toBson(&pipelineBuilder);
BSONObj pipelineBson(pipelineBuilder.done());
(log() << "\n---- pipelineBson\n" <<
pipelineBson.jsonString(Strict, 1) << "\n----\n").flush();
}
/* on the shard servers, create the local pipeline */
intrusive_ptr<ExpressionContext> pShardCtx(
ExpressionContext::create(&InterruptStatusMongod::status));
intrusive_ptr<Pipeline> pShardPipeline(
Pipeline::parseCommand(errmsg, shardBson, pShardCtx));
if (!pShardPipeline.get()) {
return false;
}
/* run the shard pipeline */
BSONObjBuilder shardResultBuilder;
string shardErrmsg;
pShardPipeline->run(shardResultBuilder, shardErrmsg, pSource);
BSONObj shardResult(shardResultBuilder.done());
/* pick out the shard result, and prepare to read it */
intrusive_ptr<DocumentSourceBsonArray> pShardSource;
BSONObjIterator shardIter(shardResult);
while(shardIter.more()) {
BSONElement shardElement(shardIter.next());
const char *pFieldName = shardElement.fieldName();
if ((strcmp(pFieldName, "result") == 0) ||
(strcmp(pFieldName, "serverPipeline") == 0)) {
pShardSource = DocumentSourceBsonArray::create(
&shardElement, pCtx);
/*
Connect the output of the shard pipeline with the mongos
pipeline that will merge the results.
*/
return pPipeline->run(result, errmsg, pShardSource);
}
}
/* NOTREACHED */
verify(false);
return false;
}
/*
Execute the pipeline for the explain. This is common to both the
locked and unlocked code path. However, the results are different.
For an explain, with no lock, it really outputs the pipeline
chain rather than fetching the data.
*/
bool executeSplitPipeline(BSONObjBuilder& result, string& errmsg,
const string& ns, const string& db,
intrusive_ptr<Pipeline>& pPipeline,
intrusive_ptr<ExpressionContext>& pCtx) {
/* setup as if we're in the router */
pCtx->inRouter = true;
/*
Here, we'll split the pipeline in the same way we would for sharding,
for testing purposes.
Run the shard pipeline first, then feed the results into the remains
of the existing pipeline.
Start by splitting the pipeline.
*/
intrusive_ptr<Pipeline> pShardSplit = pPipeline->splitForSharded();
// Write the split pipeline as we would in order to transmit it to the shard servers.
Document shardCmd = pShardSplit->serialize();
DEV log() << "\n---- shardDescription\n" << shardCmd.toString() << "\n----\n";
// for debugging purposes, show what the pipeline now looks like
DEV log() << "\n---- pipelineDescription\n" << pPipeline->serialize() << "\n----\n";
/* on the shard servers, create the local pipeline */
intrusive_ptr<ExpressionContext> pShardCtx =
new ExpressionContext(InterruptStatusMongod::status, NamespaceString(ns));
BSONObj shardObj = shardCmd.toBson();
intrusive_ptr<Pipeline> pShardPipeline(
Pipeline::parseCommand(errmsg, shardObj, pShardCtx));
if (!pShardPipeline.get()) {
return false;
}
PipelineD::prepareCursorSource(pShardPipeline, nsToDatabase(ns), pCtx);
/* run the shard pipeline */
BSONObjBuilder shardResultBuilder;
string shardErrmsg;
pShardPipeline->stitch();
pShardPipeline->run(shardResultBuilder);
BSONObj shardResult(shardResultBuilder.done());
/* pick out the shard result, and prepare to read it */
intrusive_ptr<DocumentSourceBsonArray> pShardSource;
BSONObjIterator shardIter(shardResult);
while(shardIter.more()) {
BSONElement shardElement(shardIter.next());
const char *pFieldName = shardElement.fieldName();
if ((strcmp(pFieldName, "result") == 0) ||
(strcmp(pFieldName, "serverPipeline") == 0)) {
pPipeline->addInitialSource(DocumentSourceBsonArray::create(&shardElement, pCtx));
pPipeline->stitch();
/*
Connect the output of the shard pipeline with the mongos
pipeline that will merge the results.
*/
pPipeline->run(result);
return true;
}
}
/* NOTREACHED */
verify(false);
return false;
}
