void run() { IndexDetails& id = addIndexWithInfo(); // Create a SortPhaseOne. SortPhaseOne phaseOne; phaseOne.sorter.reset( new BSONObjExternalSorter( id.idxInterface(), BSON( "a" << 1 ) ) ); // Add index keys to the phaseOne. int32_t nKeys = 130; for( int32_t i = 0; i < nKeys; ++i ) { phaseOne.sorter->add( BSON( "a" << i ), /* dummy disk loc */ DiskLoc(), false ); } phaseOne.nkeys = phaseOne.n = nKeys; phaseOne.sorter->sort( false ); // Set up remaining arguments. set<DiskLoc> dups; CurOp* op = cc().curop(); ProgressMeterHolder pm (op->setMessage("BuildBottomUp", "BuildBottomUp Progress", nKeys, nKeys)); pm.finished(); Timer timer; // The index's root has not yet been set. ASSERT( id.head.isNull() ); // Finish building the index. buildBottomUpPhases2And3<V1>( true, id, *phaseOne.sorter, false, dups, op, &phaseOne, pm, timer, true ); // The index's root is set after the build is complete. ASSERT( !id.head.isNull() ); // Create a cursor over the index. scoped_ptr<BtreeCursor> cursor( BtreeCursor::make( nsdetails( _ns ), id, BSON( "" << -1 ), // startKey below minimum key. BSON( "" << nKeys ), // endKey above maximum key. true, // endKeyInclusive true. 1 // direction forward. ) ); // Check that the keys in the index are the expected ones. int32_t expectedKey = 0; for( ; cursor->ok(); cursor->advance(), ++expectedKey ) { ASSERT_EQUALS( expectedKey, cursor->currKey().firstElement().number() ); } ASSERT_EQUALS( nKeys, expectedKey ); }
void run() { IndexDescriptor* id = addIndexWithInfo(); // Create a SortPhaseOne. SortPhaseOne phaseOne; phaseOne.sorter.reset(new BSONObjExternalSorter(_aFirstSort)); // It's necessary to index sufficient keys that a RARELY condition will be triggered, // but few enough keys that the btree builder will not create an internal node and check // for an interrupt internally (which would cause this test to pass spuriously). int32_t nKeys = 130; // Add index keys to the phaseOne. for( int32_t i = 0; i < nKeys; ++i ) { phaseOne.sorter->add( BSON( "a" << i ), /* dummy disk loc */ DiskLoc(), false ); } phaseOne.nkeys = phaseOne.n = nKeys; phaseOne.sorter->sort( false ); // Set up remaining arguments. set<DiskLoc> dups; CurOp* op = cc().curop(); ProgressMeterHolder pm (op->setMessage("InterruptBuildBottomUp", "InterruptBuildBottomUp Progress", nKeys, nKeys)); pm.finished(); Timer timer; // The index's root has not yet been set. ASSERT( id->getHead().isNull() ); // Register a request to kill the current operation. cc().curop()->kill(); if ( _mayInterrupt ) { // The build is aborted due to the kill request. ASSERT_THROWS ( buildBottomUpPhases2And3<V1>( true, id, *phaseOne.sorter, false, dups, op, &phaseOne, pm, timer, _mayInterrupt ), UserException ); // The root of the index is not set because the build did not complete. ASSERT( id->getHead().isNull() ); } else { // The build is aborted despite the kill request because mayInterrupt == false. buildBottomUpPhases2And3<V1>( true, id, *phaseOne.sorter, false, dups, op, &phaseOne, pm, timer, _mayInterrupt ); // The index's root is set after the build is complete. ASSERT( !id->getHead().isNull() ); } }
uint64_t BtreeBasedBuilder::fastBuildIndex(const char* ns, NamespaceDetails* d, IndexDetails& idx, bool mayInterrupt, int idxNo) { CurOp * op = cc().curop(); Timer t; tlog(1) << "fastBuildIndex " << ns << ' ' << idx.info.obj().toString() << endl; bool dupsAllowed = !idx.unique() || ignoreUniqueIndex(idx); bool dropDups = idx.dropDups() || inDBRepair; BSONObj order = idx.keyPattern(); getDur().writingDiskLoc(idx.head).Null(); if ( logLevel > 1 ) printMemInfo( "before index start" ); /* get and sort all the keys ----- */ ProgressMeterHolder pm(op->setMessage("index: (1/3) external sort", "Index: (1/3) External Sort Progress", d->stats.nrecords, 10)); SortPhaseOne phase1; addKeysToPhaseOne(d, ns, idx, order, &phase1, d->stats.nrecords, pm.get(), mayInterrupt, idxNo ); pm.finished(); BSONObjExternalSorter& sorter = *(phase1.sorter); if( phase1.multi ) { d->setIndexIsMultikey(ns, idxNo); } if ( logLevel > 1 ) printMemInfo( "before final sort" ); phase1.sorter->sort( mayInterrupt ); if ( logLevel > 1 ) printMemInfo( "after final sort" ); LOG(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles() << " files " << " in " << t.seconds() << " secs" << endl; set<DiskLoc> dupsToDrop; /* build index --- */ if( idx.version() == 0 ) buildBottomUpPhases2And3<V0>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, &phase1, pm, t, mayInterrupt); else if( idx.version() == 1 ) buildBottomUpPhases2And3<V1>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, &phase1, pm, t, mayInterrupt); else verify(false); if( dropDups ) log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl; BtreeBasedBuilder::doDropDups(ns, d, dupsToDrop, mayInterrupt); return phase1.n; }
uint64_t BtreeBasedBuilder::fastBuildIndex( Collection* collection, IndexDescriptor* idx, bool mayInterrupt ) { CurOp * op = cc().curop(); Timer t; MONGO_TLOG(1) << "fastBuildIndex " << collection->ns() << ' ' << idx->toString() << endl; bool dupsAllowed = !idx->unique() || ignoreUniqueIndex(idx->getOnDisk()); bool dropDups = idx->dropDups() || inDBRepair; BSONObj order = idx->keyPattern(); getDur().writingDiskLoc(idx->getOnDisk().head).Null(); if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) ) printMemInfo( "before index start" ); /* get and sort all the keys ----- */ ProgressMeterHolder pm(op->setMessage("index: (1/3) external sort", "Index: (1/3) External Sort Progress", collection->numRecords(), 10)); SortPhaseOne phase1; addKeysToPhaseOne(collection, idx, order, &phase1, pm.get(), mayInterrupt ); pm.finished(); BSONObjExternalSorter& sorter = *(phase1.sorter); if( phase1.multi ) { collection->getIndexCatalog()->markMultikey( idx ); } if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) ) printMemInfo( "before final sort" ); phase1.sorter->sort( mayInterrupt ); if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) ) printMemInfo( "after final sort" ); LOG(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles() << " files " << " in " << t.seconds() << " secs" << endl; set<DiskLoc> dupsToDrop; /* build index --- */ if( idx->version() == 0 ) buildBottomUpPhases2And3<V0>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, &phase1, pm, t, mayInterrupt); else if( idx->version() == 1 ) buildBottomUpPhases2And3<V1>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, &phase1, pm, t, mayInterrupt); else verify(false); if( dropDups ) log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl; doDropDups(collection, dupsToDrop, mayInterrupt); return phase1.n; }
// throws DBException unsigned long long fastBuildIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) { CurOp * op = cc().curop(); Timer t; tlog(1) << "fastBuildIndex " << ns << " idxNo:" << idxNo << ' ' << idx.info.obj().toString() << endl; bool dupsAllowed = !idx.unique(); bool dropDups = idx.dropDups() || inDBRepair; BSONObj order = idx.keyPattern(); getDur().writingDiskLoc(idx.head).Null(); if ( logLevel > 1 ) printMemInfo( "before index start" ); /* get and sort all the keys ----- */ ProgressMeterHolder pm( op->setMessage( "index: (1/3) external sort" , d->stats.nrecords , 10 ) ); SortPhaseOne _ours; SortPhaseOne *phase1 = precalced; if( phase1 == 0 ) { phase1 = &_ours; SortPhaseOne& p1 = *phase1; shared_ptr<Cursor> c = theDataFileMgr.findAll(ns); p1.sorter.reset( new BSONObjExternalSorter(idx.idxInterface(), order) ); p1.sorter->hintNumObjects( d->stats.nrecords ); const IndexSpec& spec = idx.getSpec(); while ( c->ok() ) { BSONObj o = c->current(); DiskLoc loc = c->currLoc(); p1.addKeys(spec, o, loc); c->advance(); pm.hit(); if ( logLevel > 1 && p1.n % 10000 == 0 ) { printMemInfo( "\t iterating objects" ); } }; } pm.finished(); BSONObjExternalSorter& sorter = *(phase1->sorter); if( phase1->multi ) d->setIndexIsMultikey(ns, idxNo); if ( logLevel > 1 ) printMemInfo( "before final sort" ); phase1->sorter->sort(); if ( logLevel > 1 ) printMemInfo( "after final sort" ); log(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles() << " files " << " in " << t.seconds() << " secs" << endl; set<DiskLoc> dupsToDrop; /* build index --- */ if( idx.version() == 0 ) buildBottomUpPhases2And3<V0>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, phase1, pm, t); else if( idx.version() == 1 ) buildBottomUpPhases2And3<V1>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, phase1, pm, t); else verify(false); if( dropDups ) log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl; for( set<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ ){ theDataFileMgr.deleteRecord( ns, i->rec(), *i, false /* cappedOk */ , true /* noWarn */ , isMaster( ns ) /* logOp */ ); getDur().commitIfNeeded(); } return phase1->n; }
bool run(const char *dbname, BSONObj& cmd, string& errmsg, BSONObjBuilder& result, bool fromRepl ){ Timer t; Client::GodScope cg; Client& client = cc(); CurOp * op = client.curop(); MRSetup mr( nsToDatabase( dbname ) , cmd ); log(1) << "mr ns: " << mr.ns << endl; if ( ! db.exists( mr.ns ) ){ errmsg = "ns doesn't exist"; return false; } bool shouldHaveData = false; long long num = 0; long long inReduce = 0; BSONObjBuilder countsBuilder; BSONObjBuilder timingBuilder; try { MRState state( mr ); state.scope->injectNative( "emit" , fast_emit ); MRTL * mrtl = new MRTL( state ); _tlmr.reset( mrtl ); ProgressMeter & pm = op->setMessage( "m/r: (1/3) emit phase" , db.count( mr.ns , mr.filter ) ); long long mapTime = 0; { readlock lock( mr.ns ); Client::Context ctx( mr.ns ); auto_ptr<Cursor> temp = QueryPlanSet(mr.ns.c_str() , mr.filter , BSONObj() ).getBestGuess()->newCursor(); auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.ns.c_str() ) ); if ( ! mr.filter.isEmpty() ) cursor->matcher.reset( new CoveredIndexMatcher( mr.filter , cursor->indexKeyPattern() ) ); Timer mt; while ( cursor->ok() ){ if ( ! cursor->currentMatches() ){ cursor->advance(); continue; } BSONObj o = cursor->current(); cursor->advance(); if ( mr.verbose ) mt.reset(); state.scope->setThis( &o ); if ( state.scope->invoke( state.map , state.setup.mapparams , 0 , true ) ) throw UserException( 9014, (string)"map invoke failed: " + state.scope->getError() ); if ( mr.verbose ) mapTime += mt.micros(); num++; if ( num % 100 == 0 ){ ClientCursor::YieldLock yield = cursor->yieldHold(); Timer t; mrtl->checkSize(); inReduce += t.micros(); if ( ! yield.stillOk() ){ cursor.release(); break; } killCurrentOp.checkForInterrupt(); } pm.hit(); if ( mr.limit && num >= mr.limit ) break; } } pm.finished(); killCurrentOp.checkForInterrupt(); countsBuilder.appendNumber( "input" , num ); countsBuilder.appendNumber( "emit" , mrtl->numEmits ); if ( mrtl->numEmits ) shouldHaveData = true; timingBuilder.append( "mapTime" , mapTime / 1000 ); timingBuilder.append( "emitLoop" , t.millis() ); // final reduce op->setMessage( "m/r: (2/3) final reduce in memory" ); mrtl->reduceInMemory(); mrtl->dump(); BSONObj sortKey = BSON( "0" << 1 ); db.ensureIndex( mr.incLong , sortKey ); { writelock lock( mr.tempLong.c_str() ); Client::Context ctx( mr.tempLong.c_str() ); assert( userCreateNS( mr.tempLong.c_str() , BSONObj() , errmsg , mr.replicate ) ); } { readlock rl(mr.incLong.c_str()); Client::Context ctx( mr.incLong ); BSONObj prev; BSONList all; pm = op->setMessage( "m/r: (3/3) final reduce to collection" , db.count( mr.incLong ) ); auto_ptr<Cursor> temp = QueryPlanSet(mr.incLong.c_str() , BSONObj() , sortKey ).getBestGuess()->newCursor(); auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.incLong.c_str() ) ); while ( cursor->ok() ){ BSONObj o = cursor->current().getOwned(); cursor->advance(); pm.hit(); if ( o.woSortOrder( prev , sortKey ) == 0 ){ all.push_back( o ); if ( pm.hits() % 1000 == 0 ){ if ( ! cursor->yield() ){ cursor.release(); break; } killCurrentOp.checkForInterrupt(); } continue; } ClientCursor::YieldLock yield = cursor->yieldHold(); state.finalReduce( all ); all.clear(); prev = o; all.push_back( o ); if ( ! yield.stillOk() ){ cursor.release(); break; } killCurrentOp.checkForInterrupt(); } { dbtemprelease tl; state.finalReduce( all ); } pm.finished(); } _tlmr.reset( 0 ); } catch ( ... ){ log() << "mr failed, removing collection" << endl; db.dropCollection( mr.tempLong ); db.dropCollection( mr.incLong ); throw; } long long finalCount = 0; { dblock lock; db.dropCollection( mr.incLong ); finalCount = mr.renameIfNeeded( db ); } timingBuilder.append( "total" , t.millis() ); result.append( "result" , mr.finalShort ); result.append( "timeMillis" , t.millis() ); countsBuilder.appendNumber( "output" , finalCount ); if ( mr.verbose ) result.append( "timing" , timingBuilder.obj() ); result.append( "counts" , countsBuilder.obj() ); if ( finalCount == 0 && shouldHaveData ){ result.append( "cmd" , cmd ); errmsg = "there were emits but no data!"; return false; } return true; }