static void emptyOplog() { writelock lk(rsoplog); Client::Context ctx(rsoplog); NamespaceDetails *d = nsdetails(rsoplog); // temp if( d && d->stats.nrecords == 0 ) return; // already empty, ok. log(1) << "replSet empty oplog" << rsLog; d->emptyCappedCollection(rsoplog); /* string errmsg; bob res; dropCollection(rsoplog, errmsg, res); log() << "replSet recreated oplog so it is empty. todo optimize this..." << rsLog; createOplog();*/ // TEMP: restart to recreate empty oplog //log() << "replSet FATAL error during initial sync. mongod restart required." << rsLog; //dbexit( EXIT_CLEAN ); /* writelock lk(rsoplog); Client::Context c(rsoplog, dbpath, 0, doauth/false); NamespaceDetails *oplogDetails = nsdetails(rsoplog); uassert(13412, str::stream() << "replSet error " << rsoplog << " is missing", oplogDetails != 0); oplogDetails->cappedTruncateAfter(rsoplog, h.commonPointOurDiskloc, false); */ }
static void emptyOplog() { Client::WriteContext ctx(rsoplog); NamespaceDetails *d = nsdetails(rsoplog); // temp if( d && d->stats.nrecords == 0 ) return; // already empty, ok. LOG(1) << "replSet empty oplog" << rsLog; d->emptyCappedCollection(rsoplog); }
void ReplSetImpl::syncFixUp(HowToFixUp& h, OplogReader& r) { DBClientConnection *them = r.conn(); // fetch all first so we needn't handle interruption in a fancy way unsigned long long totSize = 0; list< pair<DocID,bo> > goodVersions; bo newMinValid; /* fetch all the goodVersions of each document from current primary */ DocID d; unsigned long long n = 0; try { for( set<DocID>::iterator i = h.toRefetch.begin(); i != h.toRefetch.end(); i++ ) { d = *i; assert( !d._id.eoo() ); { /* TODO : slow. lots of round trips. */ n++; bo good= them->findOne(d.ns, d._id.wrap()).getOwned(); totSize += good.objsize(); uassert( 13410, "replSet too much data to roll back", totSize < 300 * 1024 * 1024 ); // note good might be eoo, indicating we should delete it goodVersions.push_back(pair<DocID,bo>(d,good)); } } newMinValid = r.getLastOp(rsoplog); if( newMinValid.isEmpty() ) { sethbmsg("rollback error newMinValid empty?"); return; } } catch(DBException& e) { sethbmsg(str::stream() << "rollback re-get objects: " << e.toString(),0); log() << "rollback couldn't re-get ns:" << d.ns << " _id:" << d._id << ' ' << n << '/' << h.toRefetch.size() << rsLog; throw e; } MemoryMappedFile::flushAll(true); sethbmsg("rollback 3.5"); if( h.rbid != getRBID(r.conn()) ) { // our source rolled back itself. so the data we received isn't necessarily consistent. sethbmsg("rollback rbid on source changed during rollback, cancelling this attempt"); return; } // update them sethbmsg(str::stream() << "rollback 4 n:" << goodVersions.size()); bool warn = false; assert( !h.commonPointOurDiskloc.isNull() ); dbMutex.assertWriteLocked(); /* we have items we are writing that aren't from a point-in-time. thus best not to come online until we get to that point in freshness. */ setMinValid(newMinValid); /** any full collection resyncs required? */ if( !h.collectionsToResync.empty() ) { for( set<string>::iterator i = h.collectionsToResync.begin(); i != h.collectionsToResync.end(); i++ ) { string ns = *i; sethbmsg(str::stream() << "rollback 4.1 coll resync " << ns); Client::Context c(*i, dbpath, 0, /*doauth*/false); try { bob res; string errmsg; dropCollection(ns, errmsg, res); { dbtemprelease r; bool ok = copyCollectionFromRemote(them->getServerAddress(), ns, bo(), errmsg, false); if( !ok ) { log() << "replSet rollback error resyncing collection " << ns << ' ' << errmsg << rsLog; throw "rollback error resyncing rollection [1]"; } } } catch(...) { log() << "replset rollback error resyncing collection " << ns << rsLog; throw "rollback error resyncing rollection [2]"; } } /* we did more reading from primary, so check it again for a rollback (which would mess us up), and make minValid newer. */ sethbmsg("rollback 4.2"); { string err; try { newMinValid = r.getLastOp(rsoplog); if( newMinValid.isEmpty() ) { err = "can't get minvalid from primary"; } else { setMinValid(newMinValid); } } catch(...) { err = "can't get/set minvalid"; } if( h.rbid != getRBID(r.conn()) ) { // our source rolled back itself. so the data we received isn't necessarily consistent. // however, we've now done writes. thus we have a problem. err += "rbid at primary changed during resync/rollback"; } if( !err.empty() ) { log() << "replSet error rolling back : " << err << ". A full resync will be necessary." << rsLog; /* todo: reset minvalid so that we are permanently in fatal state */ /* todo: don't be fatal, but rather, get all the data first. */ sethbmsg("rollback error"); throw rsfatal(); } } sethbmsg("rollback 4.3"); } sethbmsg("rollback 4.6"); /** drop collections to drop before doing individual fixups - that might make things faster below actually if there were subsequent inserts to rollback */ for( set<string>::iterator i = h.toDrop.begin(); i != h.toDrop.end(); i++ ) { Client::Context c(*i, dbpath, 0, /*doauth*/false); try { bob res; string errmsg; log(1) << "replSet rollback drop: " << *i << rsLog; dropCollection(*i, errmsg, res); } catch(...) { log() << "replset rollback error dropping collection " << *i << rsLog; } } sethbmsg("rollback 4.7"); Client::Context c(rsoplog, dbpath, 0, /*doauth*/false); NamespaceDetails *oplogDetails = nsdetails(rsoplog); uassert(13423, str::stream() << "replSet error in rollback can't find " << rsoplog, oplogDetails); map<string,shared_ptr<RemoveSaver> > removeSavers; unsigned deletes = 0, updates = 0; for( list<pair<DocID,bo> >::iterator i = goodVersions.begin(); i != goodVersions.end(); i++ ) { const DocID& d = i->first; bo pattern = d._id.wrap(); // { _id : ... } try { assert( d.ns && *d.ns ); if( h.collectionsToResync.count(d.ns) ) { /* we just synced this entire collection */ continue; } /* keep an archive of items rolled back */ shared_ptr<RemoveSaver>& rs = removeSavers[d.ns]; if ( ! rs ) rs.reset( new RemoveSaver( "rollback" , "" , d.ns ) ); // todo: lots of overhead in context, this can be faster Client::Context c(d.ns, dbpath, 0, /*doauth*/false); if( i->second.isEmpty() ) { // wasn't on the primary; delete. /* TODO1.6 : can't delete from a capped collection. need to handle that here. */ deletes++; NamespaceDetails *nsd = nsdetails(d.ns); if( nsd ) { if( nsd->capped ) { /* can't delete from a capped collection - so we truncate instead. if this item must go, so must all successors!!! */ try { /** todo: IIRC cappedTrunateAfter does not handle completely empty. todo. */ // this will crazy slow if no _id index. long long start = Listener::getElapsedTimeMillis(); DiskLoc loc = Helpers::findOne(d.ns, pattern, false); if( Listener::getElapsedTimeMillis() - start > 200 ) log() << "replSet warning roll back slow no _id index for " << d.ns << " perhaps?" << rsLog; //would be faster but requires index: DiskLoc loc = Helpers::findById(nsd, pattern); if( !loc.isNull() ) { try { nsd->cappedTruncateAfter(d.ns, loc, true); } catch(DBException& e) { if( e.getCode() == 13415 ) { // hack: need to just make cappedTruncate do this... nsd->emptyCappedCollection(d.ns); } else { throw; } } } } catch(DBException& e) { log() << "replSet error rolling back capped collection rec " << d.ns << ' ' << e.toString() << rsLog; } } else { try { deletes++; deleteObjects(d.ns, pattern, /*justone*/true, /*logop*/false, /*god*/true, rs.get() ); } catch(...) { log() << "replSet error rollback delete failed ns:" << d.ns << rsLog; } } // did we just empty the collection? if so let's check if it even exists on the source. if( nsd->stats.nrecords == 0 ) { try { string sys = cc().database()->name + ".system.namespaces"; bo o = them->findOne(sys, QUERY("name"<<d.ns)); if( o.isEmpty() ) { // we should drop try { bob res; string errmsg; dropCollection(d.ns, errmsg, res); } catch(...) { log() << "replset error rolling back collection " << d.ns << rsLog; } } } catch(DBException& ) { /* this isn't *that* big a deal, but is bad. */ log() << "replSet warning rollback error querying for existence of " << d.ns << " at the primary, ignoring" << rsLog; } } } } else { // todo faster... OpDebug debug; updates++; _updateObjects(/*god*/true, d.ns, i->second, pattern, /*upsert=*/true, /*multi=*/false , /*logtheop=*/false , debug, rs.get() ); } } catch(DBException& e) { log() << "replSet exception in rollback ns:" << d.ns << ' ' << pattern.toString() << ' ' << e.toString() << " ndeletes:" << deletes << rsLog; warn = true; } } removeSavers.clear(); // this effectively closes all of them sethbmsg(str::stream() << "rollback 5 d:" << deletes << " u:" << updates); MemoryMappedFile::flushAll(true); sethbmsg("rollback 6"); // clean up oplog log(2) << "replSet rollback truncate oplog after " << h.commonPoint.toStringPretty() << rsLog; // todo: fatal error if this throws? oplogDetails->cappedTruncateAfter(rsoplog, h.commonPointOurDiskloc, false); /* reset cached lastoptimewritten and h value */ loadLastOpTimeWritten(); sethbmsg("rollback 7"); MemoryMappedFile::flushAll(true); // done if( warn ) sethbmsg("issues during syncRollback, see log"); else sethbmsg("rollback done"); }