/**
* Currently the allowable shard keys are either
* i) a hashed single field, e.g. { a : "hashed" }, or
* ii) a compound list of ascending, potentially-nested field paths, e.g. { a : 1 , b.c : 1 }
*/
static vector<FieldRef*> parseShardKeyPattern(const BSONObj& keyPattern) {
OwnedPointerVector<FieldRef> parsedPaths;
static const vector<FieldRef*> empty;
BSONObjIterator patternIt(keyPattern);
while (patternIt.more()) {
BSONElement patternEl = patternIt.next();
parsedPaths.push_back(new FieldRef(patternEl.fieldNameStringData()));
const FieldRef& patternPath = *parsedPaths.back();
// Empty path
if (patternPath.numParts() == 0)
return empty;
// Extra "." in path?
if (patternPath.dottedField() != patternEl.fieldNameStringData())
return empty;
// Empty parts of the path, ".."?
for (size_t i = 0; i < patternPath.numParts(); ++i) {
if (patternPath.getPart(i).size() == 0)
return empty;
}
// Numeric and ascending (1.0), or "hashed" and single field
if (!patternEl.isNumber()) {
if (keyPattern.nFields() != 1 || !isHashedPatternEl(patternEl))
return empty;
} else if (patternEl.numberInt() != 1) {
return empty;
}
}
return parsedPaths.release();
}
vector<RecordIterator*> SimpleRecordStoreV1::getManyIterators( OperationContext* txn ) const {
OwnedPointerVector<RecordIterator> iterators;
const Extent* ext;
for (DiskLoc extLoc = details()->firstExtent(txn); !extLoc.isNull(); extLoc = ext->xnext) {
ext = _getExtent(txn, extLoc);
if (ext->firstRecord.isNull())
continue;
iterators.push_back(
new RecordStoreV1Base::IntraExtentIterator(txn, ext->firstRecord, this));
}
return iterators.release();
}
vector<PlanStageStats*> MultiPlanStage::generateCandidateStats() {
OwnedPointerVector<PlanStageStats> candidateStats;
for (size_t ix = 0; ix < _candidates.size(); ix++) {
if (ix == (size_t)_bestPlanIdx) { continue; }
if (ix == (size_t)_backupPlanIdx) { continue; }
PlanStageStats* stats = _candidates[ix].root->getStats();
candidateStats.push_back(stats);
}
return candidateStats.release();
}
virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int options,
string& errmsg, BSONObjBuilder& result,
bool fromRepl = false ) {
NamespaceString ns( dbname, cmdObj[name].String() );
AutoGetCollectionForRead ctx(txn, ns.ns());
Collection* collection = ctx.getCollection();
if ( !collection )
return appendCommandStatus( result,
Status( ErrorCodes::NamespaceNotFound,
str::stream() <<
"ns does not exist: " << ns.ns() ) );
size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() );
if ( numCursors == 0 || numCursors > 10000 )
return appendCommandStatus( result,
Status( ErrorCodes::BadValue,
str::stream() <<
"numCursors has to be between 1 and 10000" <<
" was: " << numCursors ) );
OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators(txn));
if (iterators.size() < numCursors) {
numCursors = iterators.size();
}
OwnedPointerVector<PlanExecutor> execs;
for ( size_t i = 0; i < numCursors; i++ ) {
WorkingSet* ws = new WorkingSet();
MultiIteratorStage* mis = new MultiIteratorStage(txn, ws, collection);
PlanExecutor* rawExec;
// Takes ownership of 'ws' and 'mis'.
Status execStatus = PlanExecutor::make(txn, ws, mis, collection,
PlanExecutor::YIELD_AUTO, &rawExec);
invariant(execStatus.isOK());
auto_ptr<PlanExecutor> curExec(rawExec);
// The PlanExecutor was registered on construction due to the YIELD_AUTO policy.
// We have to deregister it, as it will be registered with ClientCursor.
curExec->deregisterExec();
// Need to save state while yielding locks between now and getMore().
curExec->saveState();
execs.push_back(curExec.release());
}
// transfer iterators to executors using a round-robin distribution.
// TODO consider using a common work queue once invalidation issues go away.
for (size_t i = 0; i < iterators.size(); i++) {
PlanExecutor* theExec = execs[i % execs.size()];
MultiIteratorStage* mis = static_cast<MultiIteratorStage*>(theExec->getRootStage());
// This wasn't called above as they weren't assigned yet
iterators[i]->saveState();
mis->addIterator(iterators.releaseAt(i));
}
{
BSONArrayBuilder bucketsBuilder;
for (size_t i = 0; i < execs.size(); i++) {
// transfer ownership of an executor to the ClientCursor (which manages its own
// lifetime).
ClientCursor* cc = new ClientCursor( collection->getCursorManager(),
execs.releaseAt(i),
ns.ns() );
BSONObjBuilder threadResult;
appendCursorResponseObject( cc->cursorid(),
ns.ns(),
BSONArray(),
&threadResult );
threadResult.appendBool( "ok", 1 );
bucketsBuilder.append( threadResult.obj() );
}
result.appendArray( "cursors", bucketsBuilder.obj() );
}
return true;
}
virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int options,
string& errmsg, BSONObjBuilder& result,
bool fromRepl = false ) {
NamespaceString ns( dbname, cmdObj[name].String() );
AutoGetCollectionForRead ctx(txn, ns.ns());
Collection* collection = ctx.getCollection();
if ( !collection )
return appendCommandStatus( result,
Status( ErrorCodes::NamespaceNotFound,
str::stream() <<
"ns does not exist: " << ns.ns() ) );
size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() );
if ( numCursors == 0 || numCursors > 10000 )
return appendCommandStatus( result,
Status( ErrorCodes::BadValue,
str::stream() <<
"numCursors has to be between 1 and 10000" <<
" was: " << numCursors ) );
OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators(txn));
if (iterators.size() < numCursors) {
numCursors = iterators.size();
}
OwnedPointerVector<PlanExecutor> execs;
for ( size_t i = 0; i < numCursors; i++ ) {
WorkingSet* ws = new WorkingSet();
MultiIteratorStage* mis = new MultiIteratorStage(txn, ws, collection);
// Takes ownership of 'ws' and 'mis'.
auto_ptr<PlanExecutor> curExec(new PlanExecutor(txn, ws, mis, collection));
// Each of the plan executors should yield automatically. We pass "false" to
// indicate that 'curExec' should not register itself, as it will get registered
// by ClientCursor instead.
curExec->setYieldPolicy(PlanExecutor::YIELD_AUTO, false);
// Need to save state while yielding locks between now and newGetMore.
curExec->saveState();
execs.push_back(curExec.release());
}
// transfer iterators to executors using a round-robin distribution.
// TODO consider using a common work queue once invalidation issues go away.
for (size_t i = 0; i < iterators.size(); i++) {
PlanExecutor* theExec = execs[i % execs.size()];
MultiIteratorStage* mis = static_cast<MultiIteratorStage*>(theExec->getRootStage());
mis->addIterator(iterators.releaseAt(i));
}
{
BSONArrayBuilder bucketsBuilder;
for (size_t i = 0; i < execs.size(); i++) {
// transfer ownership of an executor to the ClientCursor (which manages its own
// lifetime).
ClientCursor* cc = new ClientCursor( collection, execs.releaseAt(i) );
// we are mimicking the aggregation cursor output here
// that is why there are ns, ok and empty firstBatch
BSONObjBuilder threadResult;
{
BSONObjBuilder cursor;
cursor.appendArray( "firstBatch", BSONObj() );
cursor.append( "ns", ns );
cursor.append( "id", cc->cursorid() );
threadResult.append( "cursor", cursor.obj() );
}
threadResult.appendBool( "ok", 1 );
bucketsBuilder.append( threadResult.obj() );
}
result.appendArray( "cursors", bucketsBuilder.obj() );
}
return true;
}
// static
void Explain::explainStages(PlanExecutor* exec,
ExplainCommon::Verbosity verbosity,
BSONObjBuilder* out) {
//
// Step 1: run the stages as required by the verbosity level.
//
// Inspect the tree to see if there is a MultiPlanStage.
MultiPlanStage* mps = getMultiPlanStage(exec->getRootStage());
// Get stats of the winning plan from the trial period, if the verbosity level
// is high enough and there was a runoff between multiple plans.
auto_ptr<PlanStageStats> winningStatsTrial;
if (verbosity >= ExplainCommon::EXEC_ALL_PLANS && NULL != mps) {
winningStatsTrial.reset(exec->getStats());
invariant(winningStatsTrial.get());
}
// If we need execution stats, then run the plan in order to gather the stats.
Status executePlanStatus = Status::OK();
if (verbosity >= ExplainCommon::EXEC_STATS) {
executePlanStatus = exec->executePlan();
}
//
// Step 2: collect plan stats (which also give the structure of the plan tree).
//
// Get stats for the winning plan.
scoped_ptr<PlanStageStats> winningStats(exec->getStats());
// Get stats for the rejected plans, if more than one plan was considered.
OwnedPointerVector<PlanStageStats> allPlansStats;
if (NULL != mps) {
allPlansStats = mps->generateCandidateStats();
}
//
// Step 3: use the stats trees to produce explain BSON.
//
CanonicalQuery* query = exec->getCanonicalQuery();
if (verbosity >= ExplainCommon::QUERY_PLANNER) {
generatePlannerInfo(query, winningStats.get(), allPlansStats.vector(), out);
}
if (verbosity >= ExplainCommon::EXEC_STATS) {
BSONObjBuilder execBob(out->subobjStart("executionStats"));
// If there is an execution error while running the query, the error is reported under
// the "executionStats" section and the explain as a whole succeeds.
execBob.append("executionSuccess", executePlanStatus.isOK());
if (!executePlanStatus.isOK()) {
execBob.append("errorMessage", executePlanStatus.reason());
execBob.append("errorCode", executePlanStatus.code());
}
// Generate exec stats BSON for the winning plan.
OperationContext* opCtx = exec->getOpCtx();
long long totalTimeMillis = opCtx->getCurOp()->elapsedMillis();
generateExecStats(winningStats.get(), verbosity, &execBob, totalTimeMillis);
// Also generate exec stats for all plans, if the verbosity level is high enough.
// These stats reflect what happened during the trial period that ranked the plans.
if (verbosity >= ExplainCommon::EXEC_ALL_PLANS) {
// If we ranked multiple plans against each other, then add stats collected
// from the trial period of the winning plan. The "allPlansExecution" section
// will contain an apples-to-apples comparison of the winning plan's stats against
// all rejected plans' stats collected during the trial period.
if (NULL != mps) {
invariant(winningStatsTrial.get());
allPlansStats.push_back(winningStatsTrial.release());
}
BSONArrayBuilder allPlansBob(execBob.subarrayStart("allPlansExecution"));
for (size_t i = 0; i < allPlansStats.size(); ++i) {
BSONObjBuilder planBob(allPlansBob.subobjStart());
generateExecStats(allPlansStats[i], verbosity, &planBob);
planBob.doneFast();
}
allPlansBob.doneFast();
}
execBob.doneFast();
}
generateServerInfo(out);
}
static Status parseGeoJSONPolygonCoordinates(const BSONElement& elem, S2Polygon *out) {
if (Array != elem.type()) { return BAD_VALUE("Polygon coordinates must be an array"); }
OwnedPointerVector<S2Loop> loops;
Status status = Status::OK();
string err;
BSONObjIterator it(elem.Obj());
// Iterate all loops of the polygon.
while (it.more()) {
// Parse the array of vertices of a loop.
BSONElement coordinateElt = it.next();
vector<S2Point> points;
status = parseArrayOfCoodinates(coordinateElt, &points);
if (!status.isOK()) return status;
// Check if the loop is closed.
status = isLoopClosed(points, coordinateElt);
if (!status.isOK()) return status;
eraseDuplicatePoints(&points);
// Drop the duplicated last point.
points.resize(points.size() - 1);
S2Loop* loop = new S2Loop(points);
loops.push_back(loop);
// Check whether this loop is valid.
// 1. At least 3 vertices.
// 2. All vertices must be unit length. Guaranteed by parsePoints().
// 3. Loops are not allowed to have any duplicate vertices.
// 4. Non-adjacent edges are not allowed to intersect.
if (!loop->IsValid(&err)) {
return BAD_VALUE("Loop is not valid: " << coordinateElt.toString(false) << " "
<< err);
}
// If the loop is more than one hemisphere, invert it.
loop->Normalize();
// Check the first loop must be the exterior ring and any others must be
// interior rings or holes.
if (loops.size() > 1 && !loops[0]->Contains(loop)) {
return BAD_VALUE("Secondary loops not contained by first exterior loop - "
"secondary loops must be holes: " << coordinateElt.toString(false)
<< " first loop: " << elem.Obj().firstElement().toString(false));
}
}
// Check if the given loops form a valid polygon.
// 1. If a loop contains an edge AB, then no other loop may contain AB or BA.
// 2. No loop covers more than half of the sphere.
// 3. No two loops cross.
if (!S2Polygon::IsValid(loops.vector(), &err))
return BAD_VALUE("Polygon isn't valid: " << err << " " << elem.toString(false));
// Given all loops are valid / normalized and S2Polygon::IsValid() above returns true.
// The polygon must be valid. See S2Polygon member function IsValid().
// Transfer ownership of the loops and clears loop vector.
out->Init(&loops.mutableVector());
// Check if every loop of this polygon shares at most one vertex with
// its parent loop.
if (!out->IsNormalized(&err))
// "err" looks like "Loop 1 shares more than one vertex with its parent loop 0"
return BAD_VALUE(err << ": " << elem.toString(false));
// S2Polygon contains more than one ring, which is allowed by S2, but not by GeoJSON.
//
// Loops are indexed according to a preorder traversal of the nesting hierarchy.
// GetLastDescendant() returns the index of the last loop that is contained within
// a given loop. We guarantee that the first loop is the exterior ring.
if (out->GetLastDescendant(0) < out->num_loops() - 1) {
return BAD_VALUE("Only one exterior polygon loop is allowed: " << elem.toString(false));
}
// In GeoJSON, only one nesting is allowed.
// The depth of a loop is set by polygon according to the nesting hierarchy of polygon,
// so the exterior ring's depth is 0, a hole in it is 1, etc.
for (int i = 0; i < out->num_loops(); i++) {
if (out->loop(i)->depth() > 1) {
return BAD_VALUE("Polygon interior loops cannot be nested: "<< elem.toString(false));
}
}
return Status::OK();
}
virtual bool run( const string& dbname, BSONObj& cmdObj, int options,
string& errmsg, BSONObjBuilder& result,
bool fromRepl = false ) {
NamespaceString ns( dbname, cmdObj[name].String() );
Client::ReadContext ctx(ns.ns());
Database* db = ctx.ctx().db();
Collection* collection = db->getCollection( ns );
if ( !collection )
return appendCommandStatus( result,
Status( ErrorCodes::NamespaceNotFound,
str::stream() <<
"ns does not exist: " << ns.ns() ) );
size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() );
if ( numCursors == 0 || numCursors > 10000 )
return appendCommandStatus( result,
Status( ErrorCodes::BadValue,
str::stream() <<
"numCursors has to be between 1 and 10000" <<
" was: " << numCursors ) );
OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators());
if (iterators.size() < numCursors) {
numCursors = iterators.size();
}
OwnedPointerVector<MultiIteratorRunner> runners;
for ( size_t i = 0; i < numCursors; i++ ) {
runners.push_back(new MultiIteratorRunner(ns.ns(), collection));
}
// transfer iterators to runners using a round-robin distribution.
// TODO consider using a common work queue once invalidation issues go away.
for (size_t i = 0; i < iterators.size(); i++) {
runners[i % runners.size()]->addIterator(iterators.releaseAt(i));
}
{
BSONArrayBuilder bucketsBuilder;
for (size_t i = 0; i < runners.size(); i++) {
// transfer ownership of a runner to the ClientCursor (which manages its own
// lifetime).
ClientCursor* cc = new ClientCursor( collection, runners.releaseAt(i) );
// we are mimicking the aggregation cursor output here
// that is why there are ns, ok and empty firstBatch
BSONObjBuilder threadResult;
{
BSONObjBuilder cursor;
cursor.appendArray( "firstBatch", BSONObj() );
cursor.append( "ns", ns );
cursor.append( "id", cc->cursorid() );
threadResult.append( "cursor", cursor.obj() );
}
threadResult.appendBool( "ok", 1 );
bucketsBuilder.append( threadResult.obj() );
}
result.appendArray( "cursors", bucketsBuilder.obj() );
}
return true;
}
TEST(KeyStringTest, NumberOrderLots) {
std::vector<BSONObj> numbers;
{
numbers.push_back(BSON("" << 0));
numbers.push_back(BSON("" << 0.0));
numbers.push_back(BSON("" << -0.0));
numbers.push_back(BSON("" << std::numeric_limits<long long>::min()));
numbers.push_back(BSON("" << std::numeric_limits<long long>::max()));
numbers.push_back(BSON("" << static_cast<double>(std::numeric_limits<long long>::min())));
numbers.push_back(BSON("" << static_cast<double>(std::numeric_limits<long long>::max())));
numbers.push_back(BSON("" << std::numeric_limits<double>::min()));
numbers.push_back(BSON("" << std::numeric_limits<double>::max()));
numbers.push_back(BSON("" << std::numeric_limits<int>::min()));
numbers.push_back(BSON("" << std::numeric_limits<int>::max()));
numbers.push_back(BSON("" << std::numeric_limits<short>::min()));
numbers.push_back(BSON("" << std::numeric_limits<short>::max()));
for (int i = 0; i < 64; i++) {
int64_t x = 1LL << i;
numbers.push_back(BSON("" << static_cast<long long>(x)));
numbers.push_back(BSON("" << static_cast<int>(x)));
numbers.push_back(BSON("" << static_cast<double>(x)));
numbers.push_back(BSON("" << (static_cast<double>(x) + .1)));
numbers.push_back(BSON("" << (static_cast<long long>(x) + 1)));
numbers.push_back(BSON("" << (static_cast<int>(x) + 1)));
numbers.push_back(BSON("" << (static_cast<double>(x) + 1)));
numbers.push_back(BSON("" << (static_cast<double>(x) + 1.1)));
numbers.push_back(BSON("" << -static_cast<long long>(x)));
numbers.push_back(BSON("" << -static_cast<int>(x)));
numbers.push_back(BSON("" << -static_cast<double>(x)));
numbers.push_back(BSON("" << -(static_cast<double>(x) + .1)));
numbers.push_back(BSON("" << -(static_cast<long long>(x) + 1)));
numbers.push_back(BSON("" << -(static_cast<int>(x) + 1)));
numbers.push_back(BSON("" << -(static_cast<double>(x) + 1)));
numbers.push_back(BSON("" << -(static_cast<double>(x) + 1.1)));
}
for (double i = 0; i < 1000; i++) {
double x = pow(2.1, i);
numbers.push_back(BSON("" << x));
}
}
Ordering ordering = Ordering::make(BSON("a" << 1));
OwnedPointerVector<KeyString> keyStrings;
for (size_t i = 0; i < numbers.size(); i++) {
keyStrings.push_back(new KeyString(numbers[i], ordering));
}
for (size_t i = 0; i < numbers.size(); i++) {
for (size_t j = 0; j < numbers.size(); j++) {
const KeyString& a = *keyStrings[i];
const KeyString& b = *keyStrings[j];
ASSERT_EQUALS(a.compare(b), -b.compare(a));
if (a.compare(b) !=
compareNumbers(numbers[i].firstElement(), numbers[j].firstElement())) {
log() << numbers[i] << " " << numbers[j];
}
ASSERT_EQUALS(a.compare(b),
compareNumbers(numbers[i].firstElement(), numbers[j].firstElement()));
}
}
}
