void run() {
IndexDetails& id = addIndexWithInfo();
// Create a SortPhaseOne.
SortPhaseOne phaseOne;
phaseOne.sorter.reset( new BSONObjExternalSorter( id.idxInterface(),
BSON( "a" << 1 ) ) );
// Add index keys to the phaseOne.
int32_t nKeys = 130;
for( int32_t i = 0; i < nKeys; ++i ) {
phaseOne.sorter->add( BSON( "a" << i ), /* dummy disk loc */ DiskLoc(), false );
}
phaseOne.nkeys = phaseOne.n = nKeys;
phaseOne.sorter->sort( false );
// Set up remaining arguments.
set<DiskLoc> dups;
CurOp* op = cc().curop();
ProgressMeterHolder pm (op->setMessage("BuildBottomUp",
"BuildBottomUp Progress",
nKeys,
nKeys));
pm.finished();
Timer timer;
// The index's root has not yet been set.
ASSERT( id.head.isNull() );
// Finish building the index.
buildBottomUpPhases2And3<V1>( true,
id,
*phaseOne.sorter,
false,
dups,
op,
&phaseOne,
pm,
timer,
true );
// The index's root is set after the build is complete.
ASSERT( !id.head.isNull() );
// Create a cursor over the index.
scoped_ptr<BtreeCursor> cursor(
BtreeCursor::make( nsdetails( _ns ),
id,
BSON( "" << -1 ), // startKey below minimum key.
BSON( "" << nKeys ), // endKey above maximum key.
true, // endKeyInclusive true.
1 // direction forward.
) );
// Check that the keys in the index are the expected ones.
int32_t expectedKey = 0;
for( ; cursor->ok(); cursor->advance(), ++expectedKey ) {
ASSERT_EQUALS( expectedKey, cursor->currKey().firstElement().number() );
}
ASSERT_EQUALS( nKeys, expectedKey );
}
void run() {
IndexDescriptor* id = addIndexWithInfo();
// Create a SortPhaseOne.
SortPhaseOne phaseOne;
phaseOne.sorter.reset(new BSONObjExternalSorter(_aFirstSort));
// It's necessary to index sufficient keys that a RARELY condition will be triggered,
// but few enough keys that the btree builder will not create an internal node and check
// for an interrupt internally (which would cause this test to pass spuriously).
int32_t nKeys = 130;
// Add index keys to the phaseOne.
for( int32_t i = 0; i < nKeys; ++i ) {
phaseOne.sorter->add( BSON( "a" << i ), /* dummy disk loc */ DiskLoc(), false );
}
phaseOne.nkeys = phaseOne.n = nKeys;
phaseOne.sorter->sort( false );
// Set up remaining arguments.
set<DiskLoc> dups;
CurOp* op = cc().curop();
ProgressMeterHolder pm (op->setMessage("InterruptBuildBottomUp",
"InterruptBuildBottomUp Progress",
nKeys,
nKeys));
pm.finished();
Timer timer;
// The index's root has not yet been set.
ASSERT( id->getHead().isNull() );
// Register a request to kill the current operation.
cc().curop()->kill();
if ( _mayInterrupt ) {
// The build is aborted due to the kill request.
ASSERT_THROWS
( buildBottomUpPhases2And3<V1>( true,
id,
*phaseOne.sorter,
false,
dups,
op,
&phaseOne,
pm,
timer,
_mayInterrupt ),
UserException );
// The root of the index is not set because the build did not complete.
ASSERT( id->getHead().isNull() );
}
else {
// The build is aborted despite the kill request because mayInterrupt == false.
buildBottomUpPhases2And3<V1>( true,
id,
*phaseOne.sorter,
false,
dups,
op,
&phaseOne,
pm,
timer,
_mayInterrupt );
// The index's root is set after the build is complete.
ASSERT( !id->getHead().isNull() );
}
}
uint64_t BtreeBasedBuilder::fastBuildIndex(const char* ns, NamespaceDetails* d,
IndexDetails& idx, bool mayInterrupt,
int idxNo) {
CurOp * op = cc().curop();
Timer t;
tlog(1) << "fastBuildIndex " << ns << ' ' << idx.info.obj().toString() << endl;
bool dupsAllowed = !idx.unique() || ignoreUniqueIndex(idx);
bool dropDups = idx.dropDups() || inDBRepair;
BSONObj order = idx.keyPattern();
getDur().writingDiskLoc(idx.head).Null();
if ( logLevel > 1 ) printMemInfo( "before index start" );
/* get and sort all the keys ----- */
ProgressMeterHolder pm(op->setMessage("index: (1/3) external sort",
"Index: (1/3) External Sort Progress",
d->stats.nrecords,
10));
SortPhaseOne phase1;
addKeysToPhaseOne(d, ns, idx, order, &phase1, d->stats.nrecords, pm.get(),
mayInterrupt, idxNo );
pm.finished();
BSONObjExternalSorter& sorter = *(phase1.sorter);
if( phase1.multi ) {
d->setIndexIsMultikey(ns, idxNo);
}
if ( logLevel > 1 ) printMemInfo( "before final sort" );
phase1.sorter->sort( mayInterrupt );
if ( logLevel > 1 ) printMemInfo( "after final sort" );
LOG(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles()
<< " files " << " in " << t.seconds() << " secs" << endl;
set<DiskLoc> dupsToDrop;
/* build index --- */
if( idx.version() == 0 )
buildBottomUpPhases2And3<V0>(dupsAllowed,
idx,
sorter,
dropDups,
dupsToDrop,
op,
&phase1,
pm,
t,
mayInterrupt);
else if( idx.version() == 1 )
buildBottomUpPhases2And3<V1>(dupsAllowed,
idx,
sorter,
dropDups,
dupsToDrop,
op,
&phase1,
pm,
t,
mayInterrupt);
else
verify(false);
if( dropDups )
log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl;
BtreeBasedBuilder::doDropDups(ns, d, dupsToDrop, mayInterrupt);
return phase1.n;
}
uint64_t BtreeBasedBuilder::fastBuildIndex( Collection* collection,
IndexDescriptor* idx,
bool mayInterrupt ) {
CurOp * op = cc().curop();
Timer t;
MONGO_TLOG(1) << "fastBuildIndex " << collection->ns() << ' ' << idx->toString() << endl;
bool dupsAllowed = !idx->unique() || ignoreUniqueIndex(idx->getOnDisk());
bool dropDups = idx->dropDups() || inDBRepair;
BSONObj order = idx->keyPattern();
getDur().writingDiskLoc(idx->getOnDisk().head).Null();
if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) )
printMemInfo( "before index start" );
/* get and sort all the keys ----- */
ProgressMeterHolder pm(op->setMessage("index: (1/3) external sort",
"Index: (1/3) External Sort Progress",
collection->numRecords(),
10));
SortPhaseOne phase1;
addKeysToPhaseOne(collection, idx, order, &phase1, pm.get(), mayInterrupt );
pm.finished();
BSONObjExternalSorter& sorter = *(phase1.sorter);
if( phase1.multi ) {
collection->getIndexCatalog()->markMultikey( idx );
}
if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) )
printMemInfo( "before final sort" );
phase1.sorter->sort( mayInterrupt );
if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2) ) )
printMemInfo( "after final sort" );
LOG(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles()
<< " files " << " in " << t.seconds() << " secs" << endl;
set<DiskLoc> dupsToDrop;
/* build index --- */
if( idx->version() == 0 )
buildBottomUpPhases2And3<V0>(dupsAllowed,
idx,
sorter,
dropDups,
dupsToDrop,
op,
&phase1,
pm,
t,
mayInterrupt);
else if( idx->version() == 1 )
buildBottomUpPhases2And3<V1>(dupsAllowed,
idx,
sorter,
dropDups,
dupsToDrop,
op,
&phase1,
pm,
t,
mayInterrupt);
else
verify(false);
if( dropDups )
log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl;
doDropDups(collection, dupsToDrop, mayInterrupt);
return phase1.n;
}
// throws DBException
unsigned long long fastBuildIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) {
CurOp * op = cc().curop();
Timer t;
tlog(1) << "fastBuildIndex " << ns << " idxNo:" << idxNo << ' ' << idx.info.obj().toString() << endl;
bool dupsAllowed = !idx.unique();
bool dropDups = idx.dropDups() || inDBRepair;
BSONObj order = idx.keyPattern();
getDur().writingDiskLoc(idx.head).Null();
if ( logLevel > 1 ) printMemInfo( "before index start" );
/* get and sort all the keys ----- */
ProgressMeterHolder pm( op->setMessage( "index: (1/3) external sort" , d->stats.nrecords , 10 ) );
SortPhaseOne _ours;
SortPhaseOne *phase1 = precalced;
if( phase1 == 0 ) {
phase1 = &_ours;
SortPhaseOne& p1 = *phase1;
shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
p1.sorter.reset( new BSONObjExternalSorter(idx.idxInterface(), order) );
p1.sorter->hintNumObjects( d->stats.nrecords );
const IndexSpec& spec = idx.getSpec();
while ( c->ok() ) {
BSONObj o = c->current();
DiskLoc loc = c->currLoc();
p1.addKeys(spec, o, loc);
c->advance();
pm.hit();
if ( logLevel > 1 && p1.n % 10000 == 0 ) {
printMemInfo( "\t iterating objects" );
}
};
}
pm.finished();
BSONObjExternalSorter& sorter = *(phase1->sorter);
if( phase1->multi )
d->setIndexIsMultikey(ns, idxNo);
if ( logLevel > 1 ) printMemInfo( "before final sort" );
phase1->sorter->sort();
if ( logLevel > 1 ) printMemInfo( "after final sort" );
log(t.seconds() > 5 ? 0 : 1) << "\t external sort used : " << sorter.numFiles() << " files " << " in " << t.seconds() << " secs" << endl;
set<DiskLoc> dupsToDrop;
/* build index --- */
if( idx.version() == 0 )
buildBottomUpPhases2And3<V0>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, phase1, pm, t);
else if( idx.version() == 1 )
buildBottomUpPhases2And3<V1>(dupsAllowed, idx, sorter, dropDups, dupsToDrop, op, phase1, pm, t);
else
verify(false);
if( dropDups )
log() << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl;
for( set<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ ){
theDataFileMgr.deleteRecord( ns, i->rec(), *i, false /* cappedOk */ , true /* noWarn */ , isMaster( ns ) /* logOp */ );
getDur().commitIfNeeded();
}
return phase1->n;
}
bool run(const char *dbname, BSONObj& cmd, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
Timer t;
Client::GodScope cg;
Client& client = cc();
CurOp * op = client.curop();
MRSetup mr( nsToDatabase( dbname ) , cmd );
log(1) << "mr ns: " << mr.ns << endl;
if ( ! db.exists( mr.ns ) ){
errmsg = "ns doesn't exist";
return false;
}
bool shouldHaveData = false;
long long num = 0;
long long inReduce = 0;
BSONObjBuilder countsBuilder;
BSONObjBuilder timingBuilder;
try {
MRState state( mr );
state.scope->injectNative( "emit" , fast_emit );
MRTL * mrtl = new MRTL( state );
_tlmr.reset( mrtl );
ProgressMeter & pm = op->setMessage( "m/r: (1/3) emit phase" , db.count( mr.ns , mr.filter ) );
long long mapTime = 0;
{
readlock lock( mr.ns );
Client::Context ctx( mr.ns );
auto_ptr<Cursor> temp = QueryPlanSet(mr.ns.c_str() , mr.filter , BSONObj() ).getBestGuess()->newCursor();
auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.ns.c_str() ) );
if ( ! mr.filter.isEmpty() )
cursor->matcher.reset( new CoveredIndexMatcher( mr.filter , cursor->indexKeyPattern() ) );
Timer mt;
while ( cursor->ok() ){
if ( ! cursor->currentMatches() ){
cursor->advance();
continue;
}
BSONObj o = cursor->current();
cursor->advance();
if ( mr.verbose ) mt.reset();
state.scope->setThis( &o );
if ( state.scope->invoke( state.map , state.setup.mapparams , 0 , true ) )
throw UserException( 9014, (string)"map invoke failed: " + state.scope->getError() );
if ( mr.verbose ) mapTime += mt.micros();
num++;
if ( num % 100 == 0 ){
ClientCursor::YieldLock yield = cursor->yieldHold();
Timer t;
mrtl->checkSize();
inReduce += t.micros();
if ( ! yield.stillOk() ){
cursor.release();
break;
}
killCurrentOp.checkForInterrupt();
}
pm.hit();
if ( mr.limit && num >= mr.limit )
break;
}
}
pm.finished();
killCurrentOp.checkForInterrupt();
countsBuilder.appendNumber( "input" , num );
countsBuilder.appendNumber( "emit" , mrtl->numEmits );
if ( mrtl->numEmits )
shouldHaveData = true;
timingBuilder.append( "mapTime" , mapTime / 1000 );
timingBuilder.append( "emitLoop" , t.millis() );
// final reduce
op->setMessage( "m/r: (2/3) final reduce in memory" );
mrtl->reduceInMemory();
mrtl->dump();
BSONObj sortKey = BSON( "0" << 1 );
db.ensureIndex( mr.incLong , sortKey );
{
writelock lock( mr.tempLong.c_str() );
Client::Context ctx( mr.tempLong.c_str() );
assert( userCreateNS( mr.tempLong.c_str() , BSONObj() , errmsg , mr.replicate ) );
}
{
readlock rl(mr.incLong.c_str());
Client::Context ctx( mr.incLong );
BSONObj prev;
BSONList all;
pm = op->setMessage( "m/r: (3/3) final reduce to collection" , db.count( mr.incLong ) );
auto_ptr<Cursor> temp = QueryPlanSet(mr.incLong.c_str() , BSONObj() , sortKey ).getBestGuess()->newCursor();
auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.incLong.c_str() ) );
while ( cursor->ok() ){
BSONObj o = cursor->current().getOwned();
cursor->advance();
pm.hit();
if ( o.woSortOrder( prev , sortKey ) == 0 ){
all.push_back( o );
if ( pm.hits() % 1000 == 0 ){
if ( ! cursor->yield() ){
cursor.release();
break;
}
killCurrentOp.checkForInterrupt();
}
continue;
}
ClientCursor::YieldLock yield = cursor->yieldHold();
state.finalReduce( all );
all.clear();
prev = o;
all.push_back( o );
if ( ! yield.stillOk() ){
cursor.release();
break;
}
killCurrentOp.checkForInterrupt();
}
{
dbtemprelease tl;
state.finalReduce( all );
}
pm.finished();
}
_tlmr.reset( 0 );
}
catch ( ... ){
log() << "mr failed, removing collection" << endl;
db.dropCollection( mr.tempLong );
db.dropCollection( mr.incLong );
throw;
}
long long finalCount = 0;
{
dblock lock;
db.dropCollection( mr.incLong );
finalCount = mr.renameIfNeeded( db );
}
timingBuilder.append( "total" , t.millis() );
result.append( "result" , mr.finalShort );
result.append( "timeMillis" , t.millis() );
countsBuilder.appendNumber( "output" , finalCount );
if ( mr.verbose ) result.append( "timing" , timingBuilder.obj() );
result.append( "counts" , countsBuilder.obj() );
if ( finalCount == 0 && shouldHaveData ){
result.append( "cmd" , cmd );
errmsg = "there were emits but no data!";
return false;
}
return true;
}
