virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int options, string& errmsg, BSONObjBuilder& result, bool fromRepl = false ) { NamespaceString ns( dbname, cmdObj[name].String() ); AutoGetCollectionForRead ctx(txn, ns.ns()); Collection* collection = ctx.getCollection(); if ( !collection ) return appendCommandStatus( result, Status( ErrorCodes::NamespaceNotFound, str::stream() << "ns does not exist: " << ns.ns() ) ); size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() ); if ( numCursors == 0 || numCursors > 10000 ) return appendCommandStatus( result, Status( ErrorCodes::BadValue, str::stream() << "numCursors has to be between 1 and 10000" << " was: " << numCursors ) ); OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators(txn)); if (iterators.size() < numCursors) { numCursors = iterators.size(); } OwnedPointerVector<PlanExecutor> execs; for ( size_t i = 0; i < numCursors; i++ ) { WorkingSet* ws = new WorkingSet(); MultiIteratorStage* mis = new MultiIteratorStage(txn, ws, collection); PlanExecutor* rawExec; // Takes ownership of 'ws' and 'mis'. Status execStatus = PlanExecutor::make(txn, ws, mis, collection, PlanExecutor::YIELD_AUTO, &rawExec); invariant(execStatus.isOK()); auto_ptr<PlanExecutor> curExec(rawExec); // The PlanExecutor was registered on construction due to the YIELD_AUTO policy. // We have to deregister it, as it will be registered with ClientCursor. curExec->deregisterExec(); // Need to save state while yielding locks between now and getMore(). curExec->saveState(); execs.push_back(curExec.release()); } // transfer iterators to executors using a round-robin distribution. // TODO consider using a common work queue once invalidation issues go away. for (size_t i = 0; i < iterators.size(); i++) { PlanExecutor* theExec = execs[i % execs.size()]; MultiIteratorStage* mis = static_cast<MultiIteratorStage*>(theExec->getRootStage()); // This wasn't called above as they weren't assigned yet iterators[i]->saveState(); mis->addIterator(iterators.releaseAt(i)); } { BSONArrayBuilder bucketsBuilder; for (size_t i = 0; i < execs.size(); i++) { // transfer ownership of an executor to the ClientCursor (which manages its own // lifetime). ClientCursor* cc = new ClientCursor( collection->getCursorManager(), execs.releaseAt(i), ns.ns() ); BSONObjBuilder threadResult; appendCursorResponseObject( cc->cursorid(), ns.ns(), BSONArray(), &threadResult ); threadResult.appendBool( "ok", 1 ); bucketsBuilder.append( threadResult.obj() ); } result.appendArray( "cursors", bucketsBuilder.obj() ); } return true; }
// static void Explain::explainStages(PlanExecutor* exec, ExplainCommon::Verbosity verbosity, BSONObjBuilder* out) { // // Step 1: run the stages as required by the verbosity level. // // Inspect the tree to see if there is a MultiPlanStage. MultiPlanStage* mps = getMultiPlanStage(exec->getRootStage()); // Get stats of the winning plan from the trial period, if the verbosity level // is high enough and there was a runoff between multiple plans. auto_ptr<PlanStageStats> winningStatsTrial; if (verbosity >= ExplainCommon::EXEC_ALL_PLANS && NULL != mps) { winningStatsTrial.reset(exec->getStats()); invariant(winningStatsTrial.get()); } // If we need execution stats, then run the plan in order to gather the stats. Status executePlanStatus = Status::OK(); if (verbosity >= ExplainCommon::EXEC_STATS) { executePlanStatus = exec->executePlan(); } // // Step 2: collect plan stats (which also give the structure of the plan tree). // // Get stats for the winning plan. scoped_ptr<PlanStageStats> winningStats(exec->getStats()); // Get stats for the rejected plans, if more than one plan was considered. OwnedPointerVector<PlanStageStats> allPlansStats; if (NULL != mps) { allPlansStats = mps->generateCandidateStats(); } // // Step 3: use the stats trees to produce explain BSON. // CanonicalQuery* query = exec->getCanonicalQuery(); if (verbosity >= ExplainCommon::QUERY_PLANNER) { generatePlannerInfo(query, winningStats.get(), allPlansStats.vector(), out); } if (verbosity >= ExplainCommon::EXEC_STATS) { BSONObjBuilder execBob(out->subobjStart("executionStats")); // If there is an execution error while running the query, the error is reported under // the "executionStats" section and the explain as a whole succeeds. execBob.append("executionSuccess", executePlanStatus.isOK()); if (!executePlanStatus.isOK()) { execBob.append("errorMessage", executePlanStatus.reason()); execBob.append("errorCode", executePlanStatus.code()); } // Generate exec stats BSON for the winning plan. OperationContext* opCtx = exec->getOpCtx(); long long totalTimeMillis = opCtx->getCurOp()->elapsedMillis(); generateExecStats(winningStats.get(), verbosity, &execBob, totalTimeMillis); // Also generate exec stats for all plans, if the verbosity level is high enough. // These stats reflect what happened during the trial period that ranked the plans. if (verbosity >= ExplainCommon::EXEC_ALL_PLANS) { // If we ranked multiple plans against each other, then add stats collected // from the trial period of the winning plan. The "allPlansExecution" section // will contain an apples-to-apples comparison of the winning plan's stats against // all rejected plans' stats collected during the trial period. if (NULL != mps) { invariant(winningStatsTrial.get()); allPlansStats.push_back(winningStatsTrial.release()); } BSONArrayBuilder allPlansBob(execBob.subarrayStart("allPlansExecution")); for (size_t i = 0; i < allPlansStats.size(); ++i) { BSONObjBuilder planBob(allPlansBob.subobjStart()); generateExecStats(allPlansStats[i], verbosity, &planBob); planBob.doneFast(); } allPlansBob.doneFast(); } execBob.doneFast(); } generateServerInfo(out); }
virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int options, string& errmsg, BSONObjBuilder& result, bool fromRepl = false ) { NamespaceString ns( dbname, cmdObj[name].String() ); AutoGetCollectionForRead ctx(txn, ns.ns()); Collection* collection = ctx.getCollection(); if ( !collection ) return appendCommandStatus( result, Status( ErrorCodes::NamespaceNotFound, str::stream() << "ns does not exist: " << ns.ns() ) ); size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() ); if ( numCursors == 0 || numCursors > 10000 ) return appendCommandStatus( result, Status( ErrorCodes::BadValue, str::stream() << "numCursors has to be between 1 and 10000" << " was: " << numCursors ) ); OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators(txn)); if (iterators.size() < numCursors) { numCursors = iterators.size(); } OwnedPointerVector<PlanExecutor> execs; for ( size_t i = 0; i < numCursors; i++ ) { WorkingSet* ws = new WorkingSet(); MultiIteratorStage* mis = new MultiIteratorStage(txn, ws, collection); // Takes ownership of 'ws' and 'mis'. auto_ptr<PlanExecutor> curExec(new PlanExecutor(txn, ws, mis, collection)); // Each of the plan executors should yield automatically. We pass "false" to // indicate that 'curExec' should not register itself, as it will get registered // by ClientCursor instead. curExec->setYieldPolicy(PlanExecutor::YIELD_AUTO, false); // Need to save state while yielding locks between now and newGetMore. curExec->saveState(); execs.push_back(curExec.release()); } // transfer iterators to executors using a round-robin distribution. // TODO consider using a common work queue once invalidation issues go away. for (size_t i = 0; i < iterators.size(); i++) { PlanExecutor* theExec = execs[i % execs.size()]; MultiIteratorStage* mis = static_cast<MultiIteratorStage*>(theExec->getRootStage()); mis->addIterator(iterators.releaseAt(i)); } { BSONArrayBuilder bucketsBuilder; for (size_t i = 0; i < execs.size(); i++) { // transfer ownership of an executor to the ClientCursor (which manages its own // lifetime). ClientCursor* cc = new ClientCursor( collection, execs.releaseAt(i) ); // we are mimicking the aggregation cursor output here // that is why there are ns, ok and empty firstBatch BSONObjBuilder threadResult; { BSONObjBuilder cursor; cursor.appendArray( "firstBatch", BSONObj() ); cursor.append( "ns", ns ); cursor.append( "id", cc->cursorid() ); threadResult.append( "cursor", cursor.obj() ); } threadResult.appendBool( "ok", 1 ); bucketsBuilder.append( threadResult.obj() ); } result.appendArray( "cursors", bucketsBuilder.obj() ); } return true; }
bool mergeChunks( OperationContext* txn, const NamespaceString& nss, const BSONObj& minKey, const BSONObj& maxKey, const OID& epoch, string* errMsg ) { // // Get sharding state up-to-date // ConnectionString configLoc = ConnectionString::parse( shardingState.getConfigServer(), *errMsg ); if ( !configLoc.isValid() ){ warning() << *errMsg << endl; return false; } // // Get the distributed lock // ScopedDistributedLock collLock( configLoc, nss.ns() ); collLock.setLockMessage( stream() << "merging chunks in " << nss.ns() << " from " << minKey << " to " << maxKey ); Status acquisitionStatus = collLock.tryAcquire(); if (!acquisitionStatus.isOK()) { *errMsg = stream() << "could not acquire collection lock for " << nss.ns() << " to merge chunks in [" << minKey << "," << maxKey << ")" << causedBy(acquisitionStatus); warning() << *errMsg << endl; return false; } // // We now have the collection lock, refresh metadata to latest version and sanity check // ChunkVersion shardVersion; Status status = shardingState.refreshMetadataNow(txn, nss.ns(), &shardVersion); if ( !status.isOK() ) { *errMsg = str::stream() << "could not merge chunks, failed to refresh metadata for " << nss.ns() << causedBy( status.reason() ); warning() << *errMsg << endl; return false; } if ( epoch.isSet() && shardVersion.epoch() != epoch ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " has changed" << " since merge was sent" << "(sent epoch : " << epoch.toString() << ", current epoch : " << shardVersion.epoch().toString() << ")"; warning() << *errMsg << endl; return false; } CollectionMetadataPtr metadata = shardingState.getCollectionMetadata( nss.ns() ); if ( !metadata || metadata->getKeyPattern().isEmpty() ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " is not sharded"; warning() << *errMsg << endl; return false; } dassert( metadata->getShardVersion().equals( shardVersion ) ); if ( !metadata->isValidKey( minKey ) || !metadata->isValidKey( maxKey ) ) { *errMsg = stream() << "could not merge chunks, the range " << rangeToString( minKey, maxKey ) << " is not valid" << " for collection " << nss.ns() << " with key pattern " << metadata->getKeyPattern(); warning() << *errMsg << endl; return false; } // // Get merged chunk information // ChunkVersion mergeVersion = metadata->getCollVersion(); mergeVersion.incMinor(); OwnedPointerVector<ChunkType> chunksToMerge; ChunkType itChunk; itChunk.setMin( minKey ); itChunk.setMax( minKey ); itChunk.setNS( nss.ns() ); itChunk.setShard( shardingState.getShardName() ); while ( itChunk.getMax().woCompare( maxKey ) < 0 && metadata->getNextChunk( itChunk.getMax(), &itChunk ) ) { auto_ptr<ChunkType> saved( new ChunkType ); itChunk.cloneTo( saved.get() ); chunksToMerge.mutableVector().push_back( saved.release() ); } if ( chunksToMerge.empty() ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " range starting at " << minKey << " and ending at " << maxKey << " does not belong to shard " << shardingState.getShardName(); warning() << *errMsg << endl; return false; } // // Validate the range starts and ends at chunks and has no holes, error if not valid // BSONObj firstDocMin = ( *chunksToMerge.begin() )->getMin(); BSONObj firstDocMax = ( *chunksToMerge.begin() )->getMax(); // minKey is inclusive bool minKeyInRange = rangeContains( firstDocMin, firstDocMax, minKey ); if ( !minKeyInRange ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " range starting at " << minKey << " does not belong to shard " << shardingState.getShardName(); warning() << *errMsg << endl; return false; } BSONObj lastDocMin = ( *chunksToMerge.rbegin() )->getMin(); BSONObj lastDocMax = ( *chunksToMerge.rbegin() )->getMax(); // maxKey is exclusive bool maxKeyInRange = lastDocMin.woCompare( maxKey ) < 0 && lastDocMax.woCompare( maxKey ) >= 0; if ( !maxKeyInRange ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " range ending at " << maxKey << " does not belong to shard " << shardingState.getShardName(); warning() << *errMsg << endl; return false; } bool validRangeStartKey = firstDocMin.woCompare( minKey ) == 0; bool validRangeEndKey = lastDocMax.woCompare( maxKey ) == 0; if ( !validRangeStartKey || !validRangeEndKey ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " does not contain a chunk " << ( !validRangeStartKey ? "starting at " + minKey.toString() : "" ) << ( !validRangeStartKey && !validRangeEndKey ? " or " : "" ) << ( !validRangeEndKey ? "ending at " + maxKey.toString() : "" ); warning() << *errMsg << endl; return false; } if ( chunksToMerge.size() == 1 ) { *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " already contains chunk for " << rangeToString( minKey, maxKey ); warning() << *errMsg << endl; return false; } bool holeInRange = false; // Look for hole in range ChunkType* prevChunk = *chunksToMerge.begin(); ChunkType* nextChunk = NULL; for ( OwnedPointerVector<ChunkType>::const_iterator it = chunksToMerge.begin(); it != chunksToMerge.end(); ++it ) { if ( it == chunksToMerge.begin() ) continue; nextChunk = *it; if ( prevChunk->getMax().woCompare( nextChunk->getMin() ) != 0 ) { holeInRange = true; break; } prevChunk = nextChunk; } if ( holeInRange ) { dassert( NULL != nextChunk ); *errMsg = stream() << "could not merge chunks, collection " << nss.ns() << " has a hole in the range " << rangeToString( minKey, maxKey ) << " at " << rangeToString( prevChunk->getMax(), nextChunk->getMin() ); warning() << *errMsg << endl; return false; } // // Run apply ops command // BSONObj applyOpsCmd = buildApplyOpsCmd( chunksToMerge, shardVersion, mergeVersion ); bool ok; BSONObj result; try { ScopedDbConnection conn( configLoc, 30.0 ); ok = conn->runCommand( "config", applyOpsCmd, result ); if ( !ok ) *errMsg = result.toString(); conn.done(); } catch( const DBException& ex ) { ok = false; *errMsg = ex.toString(); } if ( !ok ) { *errMsg = stream() << "could not merge chunks for " << nss.ns() << ", writing to config failed" << causedBy( errMsg ); warning() << *errMsg << endl; return false; } // // Install merged chunk metadata // { Lock::DBLock writeLk(txn->lockState(), nss.db(), newlm::MODE_X); shardingState.mergeChunks(txn, nss.ns(), minKey, maxKey, mergeVersion); } // // Log change // BSONObj mergeLogEntry = buildMergeLogEntry( chunksToMerge, shardVersion, mergeVersion ); configServer.logChange( "merge", nss.ns(), mergeLogEntry ); return true; }
static Status parseGeoJSONPolygonCoordinates(const BSONElement& elem, S2Polygon *out) { if (Array != elem.type()) { return BAD_VALUE("Polygon coordinates must be an array"); } OwnedPointerVector<S2Loop> loops; Status status = Status::OK(); string err; BSONObjIterator it(elem.Obj()); // Iterate all loops of the polygon. while (it.more()) { // Parse the array of vertices of a loop. BSONElement coordinateElt = it.next(); vector<S2Point> points; status = parseArrayOfCoodinates(coordinateElt, &points); if (!status.isOK()) return status; // Check if the loop is closed. status = isLoopClosed(points, coordinateElt); if (!status.isOK()) return status; eraseDuplicatePoints(&points); // Drop the duplicated last point. points.resize(points.size() - 1); S2Loop* loop = new S2Loop(points); loops.push_back(loop); // Check whether this loop is valid. // 1. At least 3 vertices. // 2. All vertices must be unit length. Guaranteed by parsePoints(). // 3. Loops are not allowed to have any duplicate vertices. // 4. Non-adjacent edges are not allowed to intersect. if (!loop->IsValid(&err)) { return BAD_VALUE("Loop is not valid: " << coordinateElt.toString(false) << " " << err); } // If the loop is more than one hemisphere, invert it. loop->Normalize(); // Check the first loop must be the exterior ring and any others must be // interior rings or holes. if (loops.size() > 1 && !loops[0]->Contains(loop)) { return BAD_VALUE("Secondary loops not contained by first exterior loop - " "secondary loops must be holes: " << coordinateElt.toString(false) << " first loop: " << elem.Obj().firstElement().toString(false)); } } // Check if the given loops form a valid polygon. // 1. If a loop contains an edge AB, then no other loop may contain AB or BA. // 2. No loop covers more than half of the sphere. // 3. No two loops cross. if (!S2Polygon::IsValid(loops.vector(), &err)) return BAD_VALUE("Polygon isn't valid: " << err << " " << elem.toString(false)); // Given all loops are valid / normalized and S2Polygon::IsValid() above returns true. // The polygon must be valid. See S2Polygon member function IsValid(). // Transfer ownership of the loops and clears loop vector. out->Init(&loops.mutableVector()); // Check if every loop of this polygon shares at most one vertex with // its parent loop. if (!out->IsNormalized(&err)) // "err" looks like "Loop 1 shares more than one vertex with its parent loop 0" return BAD_VALUE(err << ": " << elem.toString(false)); // S2Polygon contains more than one ring, which is allowed by S2, but not by GeoJSON. // // Loops are indexed according to a preorder traversal of the nesting hierarchy. // GetLastDescendant() returns the index of the last loop that is contained within // a given loop. We guarantee that the first loop is the exterior ring. if (out->GetLastDescendant(0) < out->num_loops() - 1) { return BAD_VALUE("Only one exterior polygon loop is allowed: " << elem.toString(false)); } // In GeoJSON, only one nesting is allowed. // The depth of a loop is set by polygon according to the nesting hierarchy of polygon, // so the exterior ring's depth is 0, a hole in it is 1, etc. for (int i = 0; i < out->num_loops(); i++) { if (out->loop(i)->depth() > 1) { return BAD_VALUE("Polygon interior loops cannot be nested: "<< elem.toString(false)); } } return Status::OK(); }
virtual bool run( const string& dbname, BSONObj& cmdObj, int options, string& errmsg, BSONObjBuilder& result, bool fromRepl = false ) { NamespaceString ns( dbname, cmdObj[name].String() ); Client::ReadContext ctx(ns.ns()); Database* db = ctx.ctx().db(); Collection* collection = db->getCollection( ns ); if ( !collection ) return appendCommandStatus( result, Status( ErrorCodes::NamespaceNotFound, str::stream() << "ns does not exist: " << ns.ns() ) ); size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() ); if ( numCursors == 0 || numCursors > 10000 ) return appendCommandStatus( result, Status( ErrorCodes::BadValue, str::stream() << "numCursors has to be between 1 and 10000" << " was: " << numCursors ) ); OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators()); if (iterators.size() < numCursors) { numCursors = iterators.size(); } OwnedPointerVector<MultiIteratorRunner> runners; for ( size_t i = 0; i < numCursors; i++ ) { runners.push_back(new MultiIteratorRunner(ns.ns(), collection)); } // transfer iterators to runners using a round-robin distribution. // TODO consider using a common work queue once invalidation issues go away. for (size_t i = 0; i < iterators.size(); i++) { runners[i % runners.size()]->addIterator(iterators.releaseAt(i)); } { BSONArrayBuilder bucketsBuilder; for (size_t i = 0; i < runners.size(); i++) { // transfer ownership of a runner to the ClientCursor (which manages its own // lifetime). ClientCursor* cc = new ClientCursor( collection, runners.releaseAt(i) ); // we are mimicking the aggregation cursor output here // that is why there are ns, ok and empty firstBatch BSONObjBuilder threadResult; { BSONObjBuilder cursor; cursor.appendArray( "firstBatch", BSONObj() ); cursor.append( "ns", ns ); cursor.append( "id", cc->cursorid() ); threadResult.append( "cursor", cursor.obj() ); } threadResult.appendBool( "ok", 1 ); bucketsBuilder.append( threadResult.obj() ); } result.appendArray( "cursors", bucketsBuilder.obj() ); } return true; }
StatusWith<CompactStats> Collection::compact( const CompactOptions* compactOptions ) { if ( isCapped() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact capped collection" ); if ( _indexCatalog.numIndexesInProgress() ) return StatusWith<CompactStats>( ErrorCodes::BadValue, "cannot compact when indexes in progress" ); NamespaceDetails* d = details(); // this is a big job, so might as well make things tidy before we start just to be nice. getDur().commitIfNeeded(); list<DiskLoc> extents; for( DiskLoc L = d->firstExtent(); !L.isNull(); L = L.ext()->xnext ) extents.push_back(L); log() << "compact " << extents.size() << " extents" << endl; // same data, but might perform a little different after compact? _infoCache.reset(); vector<BSONObj> indexSpecs; { IndexCatalog::IndexIterator ii( _indexCatalog.getIndexIterator( false ) ); while ( ii.more() ) { IndexDescriptor* descriptor = ii.next(); indexSpecs.push_back( _compactAdjustIndexSpec( descriptor->infoObj() ) ); } } log() << "compact orphan deleted lists" << endl; d->orphanDeletedList(); // Start over from scratch with our extent sizing and growth d->setLastExtentSize( 0 ); // before dropping indexes, at least make sure we can allocate one extent! if ( allocateSpaceForANewRecord( _ns.ns().c_str(), d, Record::HeaderSize+1, false).isNull() ) { return StatusWith<CompactStats>( ErrorCodes::InternalError, "compact error no space available to allocate" ); } // note that the drop indexes call also invalidates all clientcursors for the namespace, // which is important and wanted here log() << "compact dropping indexes" << endl; Status status = _indexCatalog.dropAllIndexes( true ); if ( !status.isOK() ) { return StatusWith<CompactStats>( status ); } getDur().commitIfNeeded(); CompactStats stats; OwnedPointerVector<IndexCatalog::IndexBuildBlock> indexBuildBlocks; vector<IndexAccessMethod*> indexesToInsertTo; vector< std::pair<IndexAccessMethod*,IndexAccessMethod*> > bulkToCommit; for ( size_t i = 0; i < indexSpecs.size(); i++ ) { killCurrentOp.checkForInterrupt(false); BSONObj info = indexSpecs[i]; info = _compactAdjustIndexSpec( info ); info = _indexCatalog.fixIndexSpec( info ); auto_ptr<IndexCatalog::IndexBuildBlock> block( new IndexCatalog::IndexBuildBlock( this,info ) ); Status status = block->init(); if ( !status.isOK() ) return StatusWith<CompactStats>(status); IndexAccessMethod* accessMethod = block->getEntry()->accessMethod(); status = accessMethod->initializeAsEmpty(); if ( !status.isOK() ) return StatusWith<CompactStats>(status); IndexAccessMethod* bulk = accessMethod->initiateBulk(); if ( bulk ) { indexesToInsertTo.push_back( bulk ); bulkToCommit.push_back( std::pair<IndexAccessMethod*,IndexAccessMethod*>( accessMethod, bulk ) ); } else { indexesToInsertTo.push_back( accessMethod ); } indexBuildBlocks.mutableVector().push_back( block.release() ); } // reset data size and record counts to 0 for this namespace // as we're about to tally them up again for each new extent d->setStats( 0, 0 ); ProgressMeterHolder pm(cc().curop()->setMessage("compact extent", "Extent Compacting Progress", extents.size())); int extentNumber = 0; for( list<DiskLoc>::iterator i = extents.begin(); i != extents.end(); i++ ) { _compactExtent(*i, extentNumber++, indexesToInsertTo, compactOptions, &stats ); pm.hit(); } verify( d->firstExtent().ext()->xprev.isNull() ); // indexes will do their own progress meter? pm.finished(); log() << "starting index commits"; for ( size_t i = 0; i < bulkToCommit.size(); i++ ) { bulkToCommit[i].first->commitBulk( bulkToCommit[i].second, false, NULL ); } for ( size_t i = 0; i < indexBuildBlocks.size(); i++ ) { IndexCatalog::IndexBuildBlock* block = indexBuildBlocks.mutableVector()[i]; block->success(); } return StatusWith<CompactStats>( stats ); }
PlanStage::StageState TextStage::fillOutResults() { Database* db = cc().database(); Collection* collection = db->getCollection( _params.ns ); if (NULL == collection) { warning() << "TextStage params namespace error"; return PlanStage::FAILURE; } vector<IndexDescriptor*> idxMatches; collection->getIndexCatalog()->findIndexByType("text", idxMatches); if (1 != idxMatches.size()) { warning() << "Expected exactly one text index"; return PlanStage::FAILURE; } // Get all the index scans for each term in our query. OwnedPointerVector<PlanStage> scanners; for (size_t i = 0; i < _params.query.getTerms().size(); i++) { const string& term = _params.query.getTerms()[i]; IndexScanParams params; params.bounds.startKey = FTSIndexFormat::getIndexKey(MAX_WEIGHT, term, _params.indexPrefix); params.bounds.endKey = FTSIndexFormat::getIndexKey(0, term, _params.indexPrefix); params.bounds.endKeyInclusive = true; params.bounds.isSimpleRange = true; params.descriptor = idxMatches[0]; params.direction = -1; IndexScan* ixscan = new IndexScan(params, _ws, NULL); scanners.mutableVector().push_back(ixscan); } // Map: diskloc -> aggregate score for doc. typedef unordered_map<DiskLoc, double, DiskLoc::Hasher> ScoreMap; ScoreMap scores; // For each index scan, read all results and store scores. size_t currentIndexScanner = 0; while (currentIndexScanner < scanners.size()) { BSONObj keyObj; DiskLoc loc; WorkingSetID id; PlanStage::StageState state = scanners.vector()[currentIndexScanner]->work(&id); if (PlanStage::ADVANCED == state) { WorkingSetMember* wsm = _ws->get(id); IndexKeyDatum& keyDatum = wsm->keyData.back(); filterAndScore(keyDatum.keyData, wsm->loc, &scores[wsm->loc]); _ws->free(id); } else if (PlanStage::IS_EOF == state) { // Done with this scan. ++currentIndexScanner; } else if (PlanStage::NEED_FETCH == state) { // We're calling work() on ixscans and they have no way to return a fetch. verify(false); } else if (PlanStage::NEED_TIME == state) { // We are a blocking stage, so ignore scanner's request for more time. } else { verify(PlanStage::FAILURE == state); warning() << "error from index scan during text stage: invalid FAILURE state"; return PlanStage::FAILURE; } } // Filter for phrases and negative terms, score and truncate. for (ScoreMap::iterator i = scores.begin(); i != scores.end(); ++i) { DiskLoc loc = i->first; double score = i->second; // Ignore non-matched documents. if (score < 0) { continue; } // Filter for phrases and negated terms if (_params.query.hasNonTermPieces()) { if (!_ftsMatcher.matchesNonTerm(loc.obj())) { continue; } } // Add results to working set as LOC_AND_UNOWNED_OBJ initially. // On invalidation, we copy the object and change the state to // OWNED_OBJ. // Fill out a WSM. WorkingSetID id = _ws->allocate(); WorkingSetMember* member = _ws->get(id); member->loc = loc; member->obj = member->loc.obj(); member->state = WorkingSetMember::LOC_AND_UNOWNED_OBJ; member->addComputed(new TextScoreComputedData(score)); _results.push_back(id); _wsidByDiskLoc[member->loc] = id; } _filledOutResults = true; if (_results.size() == 0) { return PlanStage::IS_EOF; } return PlanStage::NEED_TIME; }