void OplogReader::tailingQueryGTE(const char *ns, GTID gtid, const BSONObj* fields ) { BSONObjBuilder q; addGTIDToBSON("$gte", gtid, q); BSONObjBuilder query; query.append("_id", q.done()); tailingQuery(ns, Query(query.done()).hint(BSON("_id" << 1)), fields); }
// assumes it is locked on entry void logToReplInfo(GTID minLiveGTID, GTID minUnappliedGTID) { BufBuilder bufbuilder(256); BSONObjBuilder b(bufbuilder); b.append("_id", "minLive"); addGTIDToBSON("GTID", minLiveGTID, b); BSONObj bb = b.done(); const uint64_t flags = Collection::NO_UNIQUE_CHECKS | Collection::NO_LOCKTREE; replInfoDetails->insertObject(bb, flags); bufbuilder.reset(); BSONObjBuilder b2(bufbuilder); b2.append("_id", "minUnapplied"); addGTIDToBSON("GTID", minUnappliedGTID, b2); BSONObj bb2 = b2.done(); replInfoDetails->insertObject(bb2, flags); }
void hotOptimizeOplogTo(GTID gtid, uint64_t* loops_run) { Client::ReadContext ctx(rsoplog); // do a hot optimize up until gtid; BSONObjBuilder q; addGTIDToBSON("", gtid, q); rsOplogDetails->optimizePK(minKey, q.done(), loops_run); }
bool OplogReader::propogateSlaveLocation(GTID lastGTID){ BSONObjBuilder cmd; cmd.append("updateSlave", 1); addGTIDToBSON("gtid", lastGTID, cmd); BSONObj ret; return _conn->runCommand( "local", cmd.done(), ret ); }
shared_ptr<DBClientCursor> OplogReader::getRollbackCursor(GTID lastGTID) { shared_ptr<DBClientCursor> retCursor; BSONObjBuilder q; addGTIDToBSON("$lte", lastGTID, q); BSONObjBuilder query; query.append("_id", q.done()); retCursor.reset( _conn->query(rsoplog, Query(query.done()).sort(reverseIDObj), 0, 0, NULL, QueryOption_SlaveOk).release() ); return retCursor; }
void logTransactionOpsRef(GTID gtid, uint64_t timestamp, uint64_t hash, OID& oid) { Lock::DBRead lk1("local"); BSONObjBuilder b; addGTIDToBSON("_id", gtid, b); b.appendDate("ts", timestamp); b.append("h", (long long)hash); b.append("a", true); b.append("ref", oid); BSONObj bb = b.done(); writeEntryToOplog(bb, true); }
void hotOptimizeOplogTo(GTID gtid, const int timeout, uint64_t *loops_run) { LOCK_REASON(lockReason, "repl: optimizing oplog"); Client::ReadContext ctx(rsoplog, lockReason); // do a hot optimize up until gtid; BSONObjBuilder q; addGTIDToBSON("", gtid, q); // TODO: rsOplogDetails should be stored as OplogCollection, not Collection OplogCollection *cl = rsOplogDetails->as<OplogCollection>(); cl->optimizePK(minKey, q.done(), timeout, loops_run); }
bool gtidExistsInOplog(GTID gtid) { Client::ReadContext ctx(rsoplog); // TODO: Should this be using rsOplogDetails, verifying non-null? NamespaceDetails *d = nsdetails(rsoplog); BSONObjBuilder q; BSONObj result; addGTIDToBSON("_id", gtid, q); const bool found = d != NULL && d->findOne( q.done(), result ); return found; }
bool gtidExistsInOplog(GTID gtid) { LOCK_REASON(lockReason, "repl: querying for GTID in oplog"); Client::ReadContext ctx(rsoplog, lockReason); // TODO: Should this be using rsOplogDetails, verifying non-null? Collection *cl = getCollection(rsoplog); BSONObjBuilder q; BSONObj result; addGTIDToBSON("_id", gtid, q); const bool found = cl != NULL && cl->findOne( q.done(), result ); return found; }
static void _logTransactionOps(GTID gtid, uint64_t timestamp, uint64_t hash, BSONArray& opInfo) { Lock::DBRead lk1("local"); BSONObjBuilder b; addGTIDToBSON("_id", gtid, b); b.appendDate("ts", timestamp); b.append("h", (long long)hash); b.append("a", true); b.append("ops", opInfo); BSONObj bb = b.done(); // write it to oplog LOG(3) << "writing " << bb.toString(false, true) << " to master " << endl; writeEntryToOplog(bb, true); }
void BackgroundSync::runRollback(OplogReader& r, uint64_t oplogTS) { // starting from ourLast, we need to read the remote oplog // backwards until we find an entry in the remote oplog // that has the same GTID, timestamp, and hash as // what we have in our oplog. If we don't find one that is within // some reasonable timeframe, then we go fatal GTID ourLast = theReplSet->gtidManager->getLiveState(); GTID idToRollbackTo; uint64_t rollbackPointTS = 0; uint64_t rollbackPointHash = 0; incRBID(); try { shared_ptr<DBClientCursor> rollbackCursor = r.getRollbackCursor(ourLast); while (rollbackCursor->more()) { BSONObj remoteObj = rollbackCursor->next(); GTID remoteGTID = getGTIDFromBSON("_id", remoteObj); uint64_t remoteTS = remoteObj["ts"]._numberLong(); uint64_t remoteLastHash = remoteObj["h"].numberLong(); if (remoteTS + 1800*1000 < oplogTS) { log() << "Rollback takes us too far back, throwing exception. remoteTS: " << remoteTS << " oplogTS: " << oplogTS << rsLog; throw RollbackOplogException("replSet rollback too long a time period for a rollback (at least 30 minutes)."); break; } //now try to find an entry in our oplog with that GTID BSONObjBuilder localQuery; BSONObj localObj; addGTIDToBSON("_id", remoteGTID, localQuery); bool foundLocally = false; { LOCK_REASON(lockReason, "repl: looking up oplog entry for rollback"); Client::ReadContext ctx(rsoplog, lockReason); Client::Transaction transaction(DB_SERIALIZABLE); foundLocally = Collection::findOne(rsoplog, localQuery.done(), localObj); transaction.commit(); } if (foundLocally) { GTID localGTID = getGTIDFromBSON("_id", localObj); uint64_t localTS = localObj["ts"]._numberLong(); uint64_t localLastHash = localObj["h"].numberLong(); if (localLastHash == remoteLastHash && localTS == remoteTS && GTID::cmp(localGTID, remoteGTID) == 0 ) { idToRollbackTo = localGTID; rollbackPointTS = localTS; rollbackPointHash = localLastHash; log() << "found id to rollback to " << idToRollbackTo << rsLog; break; } } } // At this point, either we have found the point to try to rollback to, // or we have determined that we cannot rollback if (idToRollbackTo.isInitial()) { // we cannot rollback throw RollbackOplogException("could not find ID to rollback to"); } } catch (DBException& e) { log() << "Caught DBException during rollback " << e.toString() << rsLog; throw RollbackOplogException("DBException while trying to find ID to rollback to: " + e.toString()); } catch (std::exception& e2) { log() << "Caught std::exception during rollback " << e2.what() << rsLog; throw RollbackOplogException(str::stream() << "Exception while trying to find ID to rollback to: " << e2.what()); } // proceed with the rollback to point idToRollbackTo // probably ought to grab a global write lock while doing this // I don't think we want oplog cursors reading from this machine // while we are rolling back. Or at least do something to protect against this // first, let's get all the operations that are being applied out of the way, // we don't want to rollback an item in the oplog while simultaneously, // the applier thread is applying it to the oplog { boost::unique_lock<boost::mutex> lock(_mutex); while (_deque.size() > 0) { log() << "waiting for applier to finish work before doing rollback " << rsLog; _queueDone.wait(lock); } verifySettled(); } // now let's tell the system we are going to rollback, to do so, // abort live multi statement transactions, invalidate cursors, and // change the state to RS_ROLLBACK { // so we know nothing is simultaneously occurring RWLockRecursive::Exclusive e(operationLock); LOCK_REASON(lockReason, "repl: killing all operations for rollback"); Lock::GlobalWrite lk(lockReason); ClientCursor::invalidateAllCursors(); Client::abortLiveTransactions(); theReplSet->goToRollbackState(); } try { // now that we are settled, we have to take care of the GTIDManager // and the repl info thread. // We need to reset the state of the GTIDManager to the point // we intend to rollback to, and we need to make sure that the repl info thread // has captured this information. theReplSet->gtidManager->resetAfterInitialSync( idToRollbackTo, rollbackPointTS, rollbackPointHash ); // now force an update of the repl info thread theReplSet->forceUpdateReplInfo(); // at this point, everything should be settled, the applier should // have nothing left (and remain that way, because this is the only // thread that can put work on the applier). Now we can rollback // the data. while (true) { BSONObj o; { LOCK_REASON(lockReason, "repl: checking for oplog data"); Lock::DBRead lk(rsoplog, lockReason); Client::Transaction txn(DB_SERIALIZABLE); // if there is nothing in the oplog, break o = getLastEntryInOplog(); if( o.isEmpty() ) { break; } } GTID lastGTID = getGTIDFromBSON("_id", o); // if we have rolled back enough, break from while loop if (GTID::cmp(lastGTID, idToRollbackTo) <= 0) { dassert(GTID::cmp(lastGTID, idToRollbackTo) == 0); break; } rollbackTransactionFromOplog(o, true); } theReplSet->leaveRollbackState(); } catch (DBException& e) { log() << "Caught DBException during rollback " << e.toString() << rsLog; throw RollbackOplogException("DBException while trying to run rollback: " + e.toString()); } catch (std::exception& e2) { log() << "Caught std::exception during rollback " << e2.what() << rsLog; throw RollbackOplogException(str::stream() << "Exception while trying to run rollback: " << e2.what()); } }
void findRollbackPoint( OplogReader& r, uint64_t oplogTS, GTID* idToRollbackTo, uint64_t* rollbackPointTS, uint64_t* rollbackPointHash ) { bool gtidFound = false; try { GTID ourLast = theReplSet->gtidManager->getLiveState(); shared_ptr<DBClientCursor> rollbackCursor = r.getRollbackCursor(ourLast); uassert(17350, "rollback failed to get a cursor to start reading backwards from.", rollbackCursor.get()); while (rollbackCursor->more()) { BSONObj remoteObj = rollbackCursor->next(); GTID remoteGTID = getGTIDFromBSON("_id", remoteObj); uint64_t remoteTS = remoteObj["ts"]._numberLong(); uint64_t remoteLastHash = remoteObj["h"].numberLong(); if (remoteTS + 1800*1000 < oplogTS) { log() << "Rollback takes us too far back, throwing exception. remoteTS: " << remoteTS << " oplogTS: " << oplogTS << rsLog; throw RollbackOplogException("replSet rollback too long a time period for a rollback (at least 30 minutes)."); break; } //now try to find an entry in our oplog with that GTID BSONObjBuilder localQuery; BSONObj localObj; addGTIDToBSON("_id", remoteGTID, localQuery); bool foundLocally = false; { LOCK_REASON(lockReason, "repl: looking up oplog entry for rollback"); Client::ReadContext ctx(rsoplog, lockReason); Client::Transaction transaction(DB_SERIALIZABLE); foundLocally = Collection::findOne(rsoplog, localQuery.done(), localObj); transaction.commit(); } if (foundLocally) { GTID localGTID = getGTIDFromBSON("_id", localObj); uint64_t localTS = localObj["ts"]._numberLong(); uint64_t localLastHash = localObj["h"].numberLong(); if (localLastHash == remoteLastHash && localTS == remoteTS && GTID::cmp(localGTID, remoteGTID) == 0 ) { *idToRollbackTo = localGTID; *rollbackPointTS = localTS; *rollbackPointHash = localLastHash; gtidFound = true; log() << "found id to rollback to " << idToRollbackTo->toString() << rsLog; break; } } } // At this point, either we have found the point to try to rollback to, // or we have determined that we cannot rollback if (!gtidFound) { // we cannot rollback throw RollbackOplogException("could not find ID to rollback to"); } } catch (DBException& e) { log() << "Caught DBException during rollback " << e.toString() << rsLog; throw RollbackOplogException("DBException while trying to find ID to rollback to: " + e.toString()); } catch (std::exception& e2) { log() << "Caught std::exception during rollback " << e2.what() << rsLog; throw RollbackOplogException(str::stream() << "Exception while trying to find ID to rollback to: " << e2.what()); } }