/** * Add the provided (obj, loc) pair to all indices. */ void indexRecord(const char *ns, NamespaceDetails *d, const BSONObj &obj, const DiskLoc &loc) { int numIndices = d->getTotalIndexCount(); for (int i = 0; i < numIndices; ++i) { IndexDetails &id = d->idx(i); try { addKeysToIndex(ns, d, i, obj, loc, !id.unique() || ignoreUniqueIndex(id)); } catch (AssertionException&) { // TODO: the new index layer indexes either all or no keys, so j <= i can be j < i. for (int j = 0; j <= i; j++) { try { _unindexRecord(d, j, obj, loc, false); } catch(...) { LOG(3) << "unindex fails on rollback after unique " "key constraint prevented insert" << std::endl; } } throw; } } }
unsigned long long addExistingToIndex( OperationContext* txn, Collection* collection, const IndexDescriptor* descriptor, IndexAccessMethod* accessMethod, bool canBeKilled ) { string ns = collection->ns().ns(); // our copy for sanity bool dupsAllowed = !descriptor->unique(); bool dropDups = descriptor->dropDups(); string curopMessage; { stringstream ss; ss << "Index Build"; if ( canBeKilled ) ss << "(background)"; curopMessage = ss.str(); } ProgressMeter& progress = cc().curop()->setMessage( curopMessage.c_str(), curopMessage, collection->numRecords() ); unsigned long long n = 0; unsigned long long numDropped = 0; auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns,collection)); std::string idxName = descriptor->indexName(); // After this yields in the loop, idx may point at a different index (if indexes get // flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should // depend on idx. idxNo should be recalculated after each yield. BSONObj js; DiskLoc loc; while (Runner::RUNNER_ADVANCED == runner->getNext(&js, &loc)) { try { if ( !dupsAllowed && dropDups ) { LastError::Disabled led( lastError.get() ); addKeysToIndex(txn, collection, descriptor, accessMethod, js, loc); } else { addKeysToIndex(txn, collection, descriptor, accessMethod, js, loc); } } catch( AssertionException& e ) { if (ErrorCodes::isInterruption(DBException::convertExceptionCode(e.getCode()))) { txn->checkForInterrupt(); } // TODO: Does exception really imply dropDups exception? if (dropDups) { bool runnerEOF = runner->isEOF(); runner->saveState(); BSONObj toDelete; collection->deleteDocument( txn, loc, false, true, &toDelete ); logOp( txn, "d", ns.c_str(), toDelete ); if (!runner->restoreState(txn)) { // Runner got killed somehow. This probably shouldn't happen. if (runnerEOF) { // Quote: "We were already at the end. Normal. // TODO: Why is this normal? } else { uasserted(ErrorCodes::CursorNotFound, "cursor gone during bg index; dropDups"); } break; } // We deleted a record, but we didn't actually yield the dblock. // TODO: Why did the old code assume we yielded the lock? numDropped++; } else { log() << "background addExistingToIndex exception " << e.what() << endl; throw; } } n++; progress.hit(); txn->recoveryUnit()->commitIfNeeded(); if (canBeKilled) { // Checking for interrupt here is necessary because the bg index // interruptors can only interrupt this index build while they hold // a write lock, and yieldAndCheckIfOK only checks for // interrupt prior to yielding our write lock. We need to check the kill flag // here before another iteration of the loop. txn->checkForInterrupt(); } progress.setTotalWhileRunning( collection->numRecords() ); } progress.finished(); if ( dropDups && numDropped ) log() << "\t index build dropped: " << numDropped << " dups"; return n; }
unsigned long long addExistingToIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) { bool dupsAllowed = !idx.unique(); bool dropDups = idx.dropDups(); ProgressMeter& progress = cc().curop()->setMessage( "bg index build" , d->stats.nrecords ); unsigned long long n = 0; unsigned long long numDropped = 0; auto_ptr<ClientCursor> cc; { shared_ptr<Cursor> c = theDataFileMgr.findAll(ns); cc.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, ns) ); } while ( cc->ok() ) { BSONObj js = cc->current(); try { { if ( !dupsAllowed && dropDups ) { LastError::Disabled led( lastError.get() ); addKeysToIndex(ns, d, idxNo, js, cc->currLoc(), dupsAllowed); } else { addKeysToIndex(ns, d, idxNo, js, cc->currLoc(), dupsAllowed); } } cc->advance(); } catch( AssertionException& e ) { if( e.interrupted() ) { killCurrentOp.checkForInterrupt(); } if ( dropDups ) { DiskLoc toDelete = cc->currLoc(); bool ok = cc->advance(); ClientCursor::YieldData yieldData; massert( 16093, "after yield cursor deleted" , cc->prepareToYield( yieldData ) ); theDataFileMgr.deleteRecord( ns, toDelete.rec(), toDelete, false, true , true ); if( !cc->recoverFromYield( yieldData ) ) { cc.release(); if( !ok ) { /* we were already at the end. normal. */ } else { uasserted(12585, "cursor gone during bg index; dropDups"); } break; } numDropped++; } else { log() << "background addExistingToIndex exception " << e.what() << endl; throw; } } n++; progress.hit(); getDur().commitIfNeeded(); if ( cc->yieldSometimes( ClientCursor::WillNeed ) ) { progress.setTotalWhileRunning( d->stats.nrecords ); } else { cc.release(); uasserted(12584, "cursor gone during bg index"); break; } } progress.finished(); if ( dropDups ) log() << "\t backgroundIndexBuild dupsToDrop: " << numDropped << endl; return n; }
unsigned long long addExistingToIndex( Collection* collection, const IndexDescriptor* descriptor, IndexAccessMethod* accessMethod, bool shouldYield ) { string ns = collection->ns().ns(); // our copy for sanity bool dupsAllowed = !descriptor->unique(); bool dropDups = descriptor->dropDups(); string curopMessage; { stringstream ss; ss << "Index Build"; if ( shouldYield ) ss << "(background)"; curopMessage = ss.str(); } ProgressMeter& progress = cc().curop()->setMessage( curopMessage.c_str(), curopMessage, collection->numRecords() ); unsigned long long n = 0; unsigned long long numDropped = 0; auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns)); // We're not delegating yielding to the runner because we need to know when a yield // happens. RunnerYieldPolicy yieldPolicy; std::string idxName = descriptor->indexName(); // After this yields in the loop, idx may point at a different index (if indexes get // flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should // depend on idx. idxNo should be recalculated after each yield. BSONObj js; DiskLoc loc; while (Runner::RUNNER_ADVANCED == runner->getNext(&js, &loc)) { try { if ( !dupsAllowed && dropDups ) { LastError::Disabled led( lastError.get() ); addKeysToIndex(collection, descriptor, accessMethod, js, loc); } else { addKeysToIndex(collection, descriptor, accessMethod, js, loc); } } catch( AssertionException& e ) { if( e.interrupted() ) { killCurrentOp.checkForInterrupt(); } // TODO: Does exception really imply dropDups exception? if (dropDups) { bool runnerEOF = runner->isEOF(); runner->saveState(); BSONObj toDelete; collection->deleteDocument( loc, false, true, &toDelete ); logOp( "d", ns.c_str(), toDelete ); if (!runner->restoreState()) { // Runner got killed somehow. This probably shouldn't happen. if (runnerEOF) { // Quote: "We were already at the end. Normal. // TODO: Why is this normal? } else { uasserted(12585, "cursor gone during bg index; dropDups"); } break; } // We deleted a record, but we didn't actually yield the dblock. // TODO: Why did the old code assume we yielded the lock? numDropped++; } else { log() << "background addExistingToIndex exception " << e.what() << endl; throw; } } n++; progress.hit(); getDur().commitIfNeeded(); if (shouldYield && yieldPolicy.shouldYield()) { if (!yieldPolicy.yieldAndCheckIfOK(runner.get())) { uasserted(12584, "cursor gone during bg index"); break; } progress.setTotalWhileRunning( collection->numRecords() ); // Recalculate idxNo if we yielded IndexDescriptor* idx = collection->getIndexCatalog()->findIndexByName( idxName, true ); verify( idx && idx == descriptor ); } } progress.finished(); if ( dropDups && numDropped ) log() << "\t index build dropped: " << numDropped << " dups"; return n; }
unsigned long long addExistingToIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx) { bool dupsAllowed = !idx.unique(); bool dropDups = idx.dropDups(); ProgressMeter& progress = cc().curop()->setMessage("bg index build", "Background Index Build Progress", d->numRecords()); unsigned long long n = 0; unsigned long long numDropped = 0; auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns)); // We're not delegating yielding to the runner because we need to know when a yield // happens. RunnerYieldPolicy yieldPolicy; std::string idxName = idx.indexName(); int idxNo = IndexBuildsInProgress::get(ns, idxName); // After this yields in the loop, idx may point at a different index (if indexes get // flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should // depend on idx. idxNo should be recalculated after each yield. BSONObj js; DiskLoc loc; while (Runner::RUNNER_ADVANCED == runner->getNext(&js, &loc)) { try { if ( !dupsAllowed && dropDups ) { LastError::Disabled led( lastError.get() ); addKeysToIndex(ns, d, idxNo, js, loc, dupsAllowed); } else { addKeysToIndex(ns, d, idxNo, js, loc, dupsAllowed); } } catch( AssertionException& e ) { if( e.interrupted() ) { killCurrentOp.checkForInterrupt(); } // TODO: Does exception really imply dropDups exception? if (dropDups) { bool runnerEOF = runner->isEOF(); runner->saveState(); theDataFileMgr.deleteRecord(d, ns, loc.rec(), loc, false, true, true); if (!runner->restoreState()) { // Runner got killed somehow. This probably shouldn't happen. if (runnerEOF) { // Quote: "We were already at the end. Normal. // TODO: Why is this normal? } else { uasserted(12585, "cursor gone during bg index; dropDups"); } break; } // We deleted a record, but we didn't actually yield the dblock. // TODO: Why did the old code assume we yielded the lock? numDropped++; } else { log() << "background addExistingToIndex exception " << e.what() << endl; throw; } } n++; progress.hit(); getDur().commitIfNeeded(); if (yieldPolicy.shouldYield()) { if (!yieldPolicy.yieldAndCheckIfOK(runner.get())) { uasserted(12584, "cursor gone during bg index"); break; } progress.setTotalWhileRunning( d->numRecords() ); // Recalculate idxNo if we yielded idxNo = IndexBuildsInProgress::get(ns, idxName); } } progress.finished(); if ( dropDups ) log() << "\t backgroundIndexBuild dupsToDrop: " << numDropped << endl; return n; }