/**
* Add the provided (obj, loc) pair to all indices.
*/
void indexRecord(const char *ns, NamespaceDetails *d, const BSONObj &obj, const DiskLoc &loc) {
int numIndices = d->getTotalIndexCount();
for (int i = 0; i < numIndices; ++i) {
IndexDetails &id = d->idx(i);
try {
addKeysToIndex(ns, d, i, obj, loc, !id.unique() || ignoreUniqueIndex(id));
}
catch (AssertionException&) {
// TODO: the new index layer indexes either all or no keys, so j <= i can be j < i.
for (int j = 0; j <= i; j++) {
try {
_unindexRecord(d, j, obj, loc, false);
}
catch(...) {
LOG(3) << "unindex fails on rollback after unique "
"key constraint prevented insert" << std::endl;
}
}
throw;
}
}
}
unsigned long long addExistingToIndex( OperationContext* txn,
Collection* collection,
const IndexDescriptor* descriptor,
IndexAccessMethod* accessMethod,
bool canBeKilled ) {
string ns = collection->ns().ns(); // our copy for sanity
bool dupsAllowed = !descriptor->unique();
bool dropDups = descriptor->dropDups();
string curopMessage;
{
stringstream ss;
ss << "Index Build";
if ( canBeKilled )
ss << "(background)";
curopMessage = ss.str();
}
ProgressMeter& progress =
cc().curop()->setMessage( curopMessage.c_str(),
curopMessage,
collection->numRecords() );
unsigned long long n = 0;
unsigned long long numDropped = 0;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns,collection));
std::string idxName = descriptor->indexName();
// After this yields in the loop, idx may point at a different index (if indexes get
// flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should
// depend on idx. idxNo should be recalculated after each yield.
BSONObj js;
DiskLoc loc;
while (Runner::RUNNER_ADVANCED == runner->getNext(&js, &loc)) {
try {
if ( !dupsAllowed && dropDups ) {
LastError::Disabled led( lastError.get() );
addKeysToIndex(txn, collection, descriptor, accessMethod, js, loc);
}
else {
addKeysToIndex(txn, collection, descriptor, accessMethod, js, loc);
}
}
catch( AssertionException& e ) {
if (ErrorCodes::isInterruption(DBException::convertExceptionCode(e.getCode()))) {
txn->checkForInterrupt();
}
// TODO: Does exception really imply dropDups exception?
if (dropDups) {
bool runnerEOF = runner->isEOF();
runner->saveState();
BSONObj toDelete;
collection->deleteDocument( txn, loc, false, true, &toDelete );
logOp( txn, "d", ns.c_str(), toDelete );
if (!runner->restoreState(txn)) {
// Runner got killed somehow. This probably shouldn't happen.
if (runnerEOF) {
// Quote: "We were already at the end. Normal.
// TODO: Why is this normal?
}
else {
uasserted(ErrorCodes::CursorNotFound,
"cursor gone during bg index; dropDups");
}
break;
}
// We deleted a record, but we didn't actually yield the dblock.
// TODO: Why did the old code assume we yielded the lock?
numDropped++;
}
else {
log() << "background addExistingToIndex exception " << e.what() << endl;
throw;
}
}
n++;
progress.hit();
txn->recoveryUnit()->commitIfNeeded();
if (canBeKilled) {
// Checking for interrupt here is necessary because the bg index
// interruptors can only interrupt this index build while they hold
// a write lock, and yieldAndCheckIfOK only checks for
// interrupt prior to yielding our write lock. We need to check the kill flag
// here before another iteration of the loop.
txn->checkForInterrupt();
}
progress.setTotalWhileRunning( collection->numRecords() );
}
progress.finished();
if ( dropDups && numDropped )
log() << "\t index build dropped: " << numDropped << " dups";
return n;
}
unsigned long long addExistingToIndex(const char *ns, NamespaceDetails *d, IndexDetails& idx, int idxNo) {
bool dupsAllowed = !idx.unique();
bool dropDups = idx.dropDups();
ProgressMeter& progress = cc().curop()->setMessage( "bg index build" , d->stats.nrecords );
unsigned long long n = 0;
unsigned long long numDropped = 0;
auto_ptr<ClientCursor> cc;
{
shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
cc.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, ns) );
}
while ( cc->ok() ) {
BSONObj js = cc->current();
try {
{
if ( !dupsAllowed && dropDups ) {
LastError::Disabled led( lastError.get() );
addKeysToIndex(ns, d, idxNo, js, cc->currLoc(), dupsAllowed);
}
else {
addKeysToIndex(ns, d, idxNo, js, cc->currLoc(), dupsAllowed);
}
}
cc->advance();
}
catch( AssertionException& e ) {
if( e.interrupted() ) {
killCurrentOp.checkForInterrupt();
}
if ( dropDups ) {
DiskLoc toDelete = cc->currLoc();
bool ok = cc->advance();
ClientCursor::YieldData yieldData;
massert( 16093, "after yield cursor deleted" , cc->prepareToYield( yieldData ) );
theDataFileMgr.deleteRecord( ns, toDelete.rec(), toDelete, false, true , true );
if( !cc->recoverFromYield( yieldData ) ) {
cc.release();
if( !ok ) {
/* we were already at the end. normal. */
}
else {
uasserted(12585, "cursor gone during bg index; dropDups");
}
break;
}
numDropped++;
}
else {
log() << "background addExistingToIndex exception " << e.what() << endl;
throw;
}
}
n++;
progress.hit();
getDur().commitIfNeeded();
if ( cc->yieldSometimes( ClientCursor::WillNeed ) ) {
progress.setTotalWhileRunning( d->stats.nrecords );
}
else {
cc.release();
uasserted(12584, "cursor gone during bg index");
break;
}
}
progress.finished();
if ( dropDups )
log() << "\t backgroundIndexBuild dupsToDrop: " << numDropped << endl;
return n;
}
unsigned long long addExistingToIndex( Collection* collection,
const IndexDescriptor* descriptor,
IndexAccessMethod* accessMethod,
bool shouldYield ) {
string ns = collection->ns().ns(); // our copy for sanity
bool dupsAllowed = !descriptor->unique();
bool dropDups = descriptor->dropDups();
string curopMessage;
{
stringstream ss;
ss << "Index Build";
if ( shouldYield )
ss << "(background)";
curopMessage = ss.str();
}
ProgressMeter& progress =
cc().curop()->setMessage( curopMessage.c_str(),
curopMessage,
collection->numRecords() );
unsigned long long n = 0;
unsigned long long numDropped = 0;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
// We're not delegating yielding to the runner because we need to know when a yield
// happens.
RunnerYieldPolicy yieldPolicy;
std::string idxName = descriptor->indexName();
// After this yields in the loop, idx may point at a different index (if indexes get
// flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should
// depend on idx. idxNo should be recalculated after each yield.
BSONObj js;
DiskLoc loc;
while (Runner::RUNNER_ADVANCED == runner->getNext(&js, &loc)) {
try {
if ( !dupsAllowed && dropDups ) {
LastError::Disabled led( lastError.get() );
addKeysToIndex(collection, descriptor, accessMethod, js, loc);
}
else {
addKeysToIndex(collection, descriptor, accessMethod, js, loc);
}
}
catch( AssertionException& e ) {
if( e.interrupted() ) {
killCurrentOp.checkForInterrupt();
}
// TODO: Does exception really imply dropDups exception?
if (dropDups) {
bool runnerEOF = runner->isEOF();
runner->saveState();
BSONObj toDelete;
collection->deleteDocument( loc, false, true, &toDelete );
logOp( "d", ns.c_str(), toDelete );
if (!runner->restoreState()) {
// Runner got killed somehow. This probably shouldn't happen.
if (runnerEOF) {
// Quote: "We were already at the end. Normal.
// TODO: Why is this normal?
}
else {
uasserted(12585, "cursor gone during bg index; dropDups");
}
break;
}
// We deleted a record, but we didn't actually yield the dblock.
// TODO: Why did the old code assume we yielded the lock?
numDropped++;
}
else {
log() << "background addExistingToIndex exception " << e.what() << endl;
throw;
}
}
n++;
progress.hit();
getDur().commitIfNeeded();
if (shouldYield && yieldPolicy.shouldYield()) {
if (!yieldPolicy.yieldAndCheckIfOK(runner.get())) {
uasserted(12584, "cursor gone during bg index");
break;
}
progress.setTotalWhileRunning( collection->numRecords() );
// Recalculate idxNo if we yielded
IndexDescriptor* idx = collection->getIndexCatalog()->findIndexByName( idxName,
true );
verify( idx && idx == descriptor );
}
}
progress.finished();
if ( dropDups && numDropped )
log() << "\t index build dropped: " << numDropped << " dups";
return n;
}
unsigned long long addExistingToIndex(const char *ns, NamespaceDetails *d,
IndexDetails& idx) {
bool dupsAllowed = !idx.unique();
bool dropDups = idx.dropDups();
ProgressMeter& progress = cc().curop()->setMessage("bg index build",
"Background Index Build Progress",
d->numRecords());
unsigned long long n = 0;
unsigned long long numDropped = 0;
auto_ptr<Runner> runner(InternalPlanner::collectionScan(ns));
// We're not delegating yielding to the runner because we need to know when a yield
// happens.
RunnerYieldPolicy yieldPolicy;
std::string idxName = idx.indexName();
int idxNo = IndexBuildsInProgress::get(ns, idxName);
// After this yields in the loop, idx may point at a different index (if indexes get
// flipped, see insert_makeIndex) or even an empty IndexDetails, so nothing below should
// depend on idx. idxNo should be recalculated after each yield.
BSONObj js;
DiskLoc loc;
while (Runner::RUNNER_ADVANCED == runner->getNext(&js, &loc)) {
try {
if ( !dupsAllowed && dropDups ) {
LastError::Disabled led( lastError.get() );
addKeysToIndex(ns, d, idxNo, js, loc, dupsAllowed);
}
else {
addKeysToIndex(ns, d, idxNo, js, loc, dupsAllowed);
}
}
catch( AssertionException& e ) {
if( e.interrupted() ) {
killCurrentOp.checkForInterrupt();
}
// TODO: Does exception really imply dropDups exception?
if (dropDups) {
bool runnerEOF = runner->isEOF();
runner->saveState();
theDataFileMgr.deleteRecord(d, ns, loc.rec(), loc, false, true, true);
if (!runner->restoreState()) {
// Runner got killed somehow. This probably shouldn't happen.
if (runnerEOF) {
// Quote: "We were already at the end. Normal.
// TODO: Why is this normal?
}
else {
uasserted(12585, "cursor gone during bg index; dropDups");
}
break;
}
// We deleted a record, but we didn't actually yield the dblock.
// TODO: Why did the old code assume we yielded the lock?
numDropped++;
}
else {
log() << "background addExistingToIndex exception " << e.what() << endl;
throw;
}
}
n++;
progress.hit();
getDur().commitIfNeeded();
if (yieldPolicy.shouldYield()) {
if (!yieldPolicy.yieldAndCheckIfOK(runner.get())) {
uasserted(12584, "cursor gone during bg index");
break;
}
progress.setTotalWhileRunning( d->numRecords() );
// Recalculate idxNo if we yielded
idxNo = IndexBuildsInProgress::get(ns, idxName);
}
}
progress.finished();
if ( dropDups )
log() << "\t backgroundIndexBuild dupsToDrop: " << numDropped << endl;
return n;
}
