static void emptyOplog() {
writelock lk(rsoplog);
Client::Context ctx(rsoplog);
NamespaceDetails *d = nsdetails(rsoplog);
// temp
if( d && d->stats.nrecords == 0 )
return; // already empty, ok.
log(1) << "replSet empty oplog" << rsLog;
d->emptyCappedCollection(rsoplog);
/*
string errmsg;
bob res;
dropCollection(rsoplog, errmsg, res);
log() << "replSet recreated oplog so it is empty. todo optimize this..." << rsLog;
createOplog();*/
// TEMP: restart to recreate empty oplog
//log() << "replSet FATAL error during initial sync. mongod restart required." << rsLog;
//dbexit( EXIT_CLEAN );
/*
writelock lk(rsoplog);
Client::Context c(rsoplog, dbpath, 0, doauth/false);
NamespaceDetails *oplogDetails = nsdetails(rsoplog);
uassert(13412, str::stream() << "replSet error " << rsoplog << " is missing", oplogDetails != 0);
oplogDetails->cappedTruncateAfter(rsoplog, h.commonPointOurDiskloc, false);
*/
}
static void emptyOplog() {
Client::WriteContext ctx(rsoplog);
NamespaceDetails *d = nsdetails(rsoplog);
// temp
if( d && d->stats.nrecords == 0 )
return; // already empty, ok.
LOG(1) << "replSet empty oplog" << rsLog;
d->emptyCappedCollection(rsoplog);
}
void ReplSetImpl::syncFixUp(HowToFixUp& h, OplogReader& r) {
DBClientConnection *them = r.conn();
// fetch all first so we needn't handle interruption in a fancy way
unsigned long long totSize = 0;
list< pair<DocID,bo> > goodVersions;
bo newMinValid;
/* fetch all the goodVersions of each document from current primary */
DocID d;
unsigned long long n = 0;
try {
for( set<DocID>::iterator i = h.toRefetch.begin(); i != h.toRefetch.end(); i++ ) {
d = *i;
assert( !d._id.eoo() );
{
/* TODO : slow. lots of round trips. */
n++;
bo good= them->findOne(d.ns, d._id.wrap()).getOwned();
totSize += good.objsize();
uassert( 13410, "replSet too much data to roll back", totSize < 300 * 1024 * 1024 );
// note good might be eoo, indicating we should delete it
goodVersions.push_back(pair<DocID,bo>(d,good));
}
}
newMinValid = r.getLastOp(rsoplog);
if( newMinValid.isEmpty() ) {
sethbmsg("rollback error newMinValid empty?");
return;
}
}
catch(DBException& e) {
sethbmsg(str::stream() << "rollback re-get objects: " << e.toString(),0);
log() << "rollback couldn't re-get ns:" << d.ns << " _id:" << d._id << ' ' << n << '/' << h.toRefetch.size() << rsLog;
throw e;
}
MemoryMappedFile::flushAll(true);
sethbmsg("rollback 3.5");
if( h.rbid != getRBID(r.conn()) ) {
// our source rolled back itself. so the data we received isn't necessarily consistent.
sethbmsg("rollback rbid on source changed during rollback, cancelling this attempt");
return;
}
// update them
sethbmsg(str::stream() << "rollback 4 n:" << goodVersions.size());
bool warn = false;
assert( !h.commonPointOurDiskloc.isNull() );
dbMutex.assertWriteLocked();
/* we have items we are writing that aren't from a point-in-time. thus best not to come online
until we get to that point in freshness. */
setMinValid(newMinValid);
/** any full collection resyncs required? */
if( !h.collectionsToResync.empty() ) {
for( set<string>::iterator i = h.collectionsToResync.begin(); i != h.collectionsToResync.end(); i++ ) {
string ns = *i;
sethbmsg(str::stream() << "rollback 4.1 coll resync " << ns);
Client::Context c(*i, dbpath, 0, /*doauth*/false);
try {
bob res;
string errmsg;
dropCollection(ns, errmsg, res);
{
dbtemprelease r;
bool ok = copyCollectionFromRemote(them->getServerAddress(), ns, bo(), errmsg, false);
if( !ok ) {
log() << "replSet rollback error resyncing collection " << ns << ' ' << errmsg << rsLog;
throw "rollback error resyncing rollection [1]";
}
}
}
catch(...) {
log() << "replset rollback error resyncing collection " << ns << rsLog;
throw "rollback error resyncing rollection [2]";
}
}
/* we did more reading from primary, so check it again for a rollback (which would mess us up), and
make minValid newer.
*/
sethbmsg("rollback 4.2");
{
string err;
try {
newMinValid = r.getLastOp(rsoplog);
if( newMinValid.isEmpty() ) {
err = "can't get minvalid from primary";
}
else {
setMinValid(newMinValid);
}
}
catch(...) {
err = "can't get/set minvalid";
}
if( h.rbid != getRBID(r.conn()) ) {
// our source rolled back itself. so the data we received isn't necessarily consistent.
// however, we've now done writes. thus we have a problem.
err += "rbid at primary changed during resync/rollback";
}
if( !err.empty() ) {
log() << "replSet error rolling back : " << err << ". A full resync will be necessary." << rsLog;
/* todo: reset minvalid so that we are permanently in fatal state */
/* todo: don't be fatal, but rather, get all the data first. */
sethbmsg("rollback error");
throw rsfatal();
}
}
sethbmsg("rollback 4.3");
}
sethbmsg("rollback 4.6");
/** drop collections to drop before doing individual fixups - that might make things faster below actually if there were subsequent inserts to rollback */
for( set<string>::iterator i = h.toDrop.begin(); i != h.toDrop.end(); i++ ) {
Client::Context c(*i, dbpath, 0, /*doauth*/false);
try {
bob res;
string errmsg;
log(1) << "replSet rollback drop: " << *i << rsLog;
dropCollection(*i, errmsg, res);
}
catch(...) {
log() << "replset rollback error dropping collection " << *i << rsLog;
}
}
sethbmsg("rollback 4.7");
Client::Context c(rsoplog, dbpath, 0, /*doauth*/false);
NamespaceDetails *oplogDetails = nsdetails(rsoplog);
uassert(13423, str::stream() << "replSet error in rollback can't find " << rsoplog, oplogDetails);
map<string,shared_ptr<RemoveSaver> > removeSavers;
unsigned deletes = 0, updates = 0;
for( list<pair<DocID,bo> >::iterator i = goodVersions.begin(); i != goodVersions.end(); i++ ) {
const DocID& d = i->first;
bo pattern = d._id.wrap(); // { _id : ... }
try {
assert( d.ns && *d.ns );
if( h.collectionsToResync.count(d.ns) ) {
/* we just synced this entire collection */
continue;
}
/* keep an archive of items rolled back */
shared_ptr<RemoveSaver>& rs = removeSavers[d.ns];
if ( ! rs )
rs.reset( new RemoveSaver( "rollback" , "" , d.ns ) );
// todo: lots of overhead in context, this can be faster
Client::Context c(d.ns, dbpath, 0, /*doauth*/false);
if( i->second.isEmpty() ) {
// wasn't on the primary; delete.
/* TODO1.6 : can't delete from a capped collection. need to handle that here. */
deletes++;
NamespaceDetails *nsd = nsdetails(d.ns);
if( nsd ) {
if( nsd->capped ) {
/* can't delete from a capped collection - so we truncate instead. if this item must go,
so must all successors!!! */
try {
/** todo: IIRC cappedTrunateAfter does not handle completely empty. todo. */
// this will crazy slow if no _id index.
long long start = Listener::getElapsedTimeMillis();
DiskLoc loc = Helpers::findOne(d.ns, pattern, false);
if( Listener::getElapsedTimeMillis() - start > 200 )
log() << "replSet warning roll back slow no _id index for " << d.ns << " perhaps?" << rsLog;
//would be faster but requires index: DiskLoc loc = Helpers::findById(nsd, pattern);
if( !loc.isNull() ) {
try {
nsd->cappedTruncateAfter(d.ns, loc, true);
}
catch(DBException& e) {
if( e.getCode() == 13415 ) {
// hack: need to just make cappedTruncate do this...
nsd->emptyCappedCollection(d.ns);
}
else {
throw;
}
}
}
}
catch(DBException& e) {
log() << "replSet error rolling back capped collection rec " << d.ns << ' ' << e.toString() << rsLog;
}
}
else {
try {
deletes++;
deleteObjects(d.ns, pattern, /*justone*/true, /*logop*/false, /*god*/true, rs.get() );
}
catch(...) {
log() << "replSet error rollback delete failed ns:" << d.ns << rsLog;
}
}
// did we just empty the collection? if so let's check if it even exists on the source.
if( nsd->stats.nrecords == 0 ) {
try {
string sys = cc().database()->name + ".system.namespaces";
bo o = them->findOne(sys, QUERY("name"<<d.ns));
if( o.isEmpty() ) {
// we should drop
try {
bob res;
string errmsg;
dropCollection(d.ns, errmsg, res);
}
catch(...) {
log() << "replset error rolling back collection " << d.ns << rsLog;
}
}
}
catch(DBException& ) {
/* this isn't *that* big a deal, but is bad. */
log() << "replSet warning rollback error querying for existence of " << d.ns << " at the primary, ignoring" << rsLog;
}
}
}
}
else {
// todo faster...
OpDebug debug;
updates++;
_updateObjects(/*god*/true, d.ns, i->second, pattern, /*upsert=*/true, /*multi=*/false , /*logtheop=*/false , debug, rs.get() );
}
}
catch(DBException& e) {
log() << "replSet exception in rollback ns:" << d.ns << ' ' << pattern.toString() << ' ' << e.toString() << " ndeletes:" << deletes << rsLog;
warn = true;
}
}
removeSavers.clear(); // this effectively closes all of them
sethbmsg(str::stream() << "rollback 5 d:" << deletes << " u:" << updates);
MemoryMappedFile::flushAll(true);
sethbmsg("rollback 6");
// clean up oplog
log(2) << "replSet rollback truncate oplog after " << h.commonPoint.toStringPretty() << rsLog;
// todo: fatal error if this throws?
oplogDetails->cappedTruncateAfter(rsoplog, h.commonPointOurDiskloc, false);
/* reset cached lastoptimewritten and h value */
loadLastOpTimeWritten();
sethbmsg("rollback 7");
MemoryMappedFile::flushAll(true);
// done
if( warn )
sethbmsg("issues during syncRollback, see log");
else
sethbmsg("rollback done");
}