void BackgroundSync::_producerThread() {
MemberState state = theReplSet->state();
// we want to pause when the state changes to primary
if (isAssumingPrimary() || state.primary()) {
if (!_pause) {
stop();
}
sleepsecs(1);
return;
}
if (state.fatal() || state.startup()) {
sleepsecs(5);
return;
}
// if this member has an empty oplog, we cannot start syncing
if (theReplSet->lastOpTimeWritten.isNull()) {
sleepsecs(1);
return;
}
// we want to unpause when we're no longer primary
// start() also loads _lastOpTimeFetched, which we know is set from the "if"
else if (_pause) {
start();
}
produce();
}
void SyncSourceFeedback::run() {
Client::initThread("SyncSourceFeedbackThread");
OperationContextImpl txn;
bool positionChanged = false;
bool handshakeNeeded = false;
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
while (!inShutdown()) { // TODO(spencer): Remove once legacy repl coordinator is gone.
{
boost::unique_lock<boost::mutex> lock(_mtx);
while (!_positionChanged && !_handshakeNeeded && !_shutdownSignaled) {
_cond.wait(lock);
}
if (_shutdownSignaled) {
break;
}
positionChanged = _positionChanged;
handshakeNeeded = _handshakeNeeded;
_positionChanged = false;
_handshakeNeeded = false;
}
MemberState state = replCoord->getCurrentMemberState();
if (state.primary() || state.fatal() || state.startup()) {
continue;
}
const Member* target = BackgroundSync::get()->getSyncTarget();
if (_syncTarget != target) {
_resetConnection();
_syncTarget = target;
}
if (!hasConnection()) {
// fix connection if need be
if (!target) {
sleepmillis(500);
continue;
}
if (!_connect(&txn, target->fullName())) {
sleepmillis(500);
continue;
}
}
if (handshakeNeeded) {
if (!replHandshake(&txn)) {
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
continue;
}
}
if (positionChanged) {
if (!updateUpstream(&txn)) {
boost::unique_lock<boost::mutex> lock(_mtx);
_positionChanged = true;
}
}
}
cc().shutdown();
}
void BackgroundSync::notifierThread() {
Client::initThread("rsSyncNotifier");
replLocalAuth();
// This makes the initial connection to our sync source for oplog position notification.
// It also sets the supportsUpdater flag so we know which method to use.
// If this function fails, we ignore that situation because it will be taken care of
// the first time markOplog() is called in the loop below.
{
boost::unique_lock<boost::mutex> oplogLockSSF(theReplSet->syncSourceFeedback.oplock);
connectOplogNotifier();
}
theReplSet->syncSourceFeedback.go();
while (!inShutdown()) {
bool clearTarget = false;
if (!theReplSet) {
sleepsecs(5);
continue;
}
MemberState state = theReplSet->state();
if (state.primary() || state.fatal() || state.startup()) {
sleepsecs(5);
continue;
}
try {
{
boost::unique_lock<boost::mutex> lock(_lastOpMutex);
while (_consumedOpTime == theReplSet->lastOpTimeWritten) {
_lastOpCond.wait(lock);
}
}
markOplog();
}
catch (DBException &e) {
clearTarget = true;
log() << "replset tracking exception: " << e.getInfo() << rsLog;
sleepsecs(1);
}
catch (std::exception &e2) {
clearTarget = true;
log() << "replset tracking error" << e2.what() << rsLog;
sleepsecs(1);
}
if (clearTarget) {
boost::unique_lock<boost::mutex> lock(_mutex);
_oplogMarkerTarget = NULL;
}
}
cc().shutdown();
}
void BackgroundSync::notifierThread() {
Client::initThread("rsSyncNotifier");
replLocalAuth();
theReplSet->syncSourceFeedback.go();
while (!inShutdown()) {
bool clearTarget = false;
if (!theReplSet) {
sleepsecs(5);
continue;
}
MemberState state = theReplSet->state();
if (state.primary() || state.fatal() || state.startup()) {
sleepsecs(5);
continue;
}
try {
{
boost::unique_lock<boost::mutex> lock(_lastOpMutex);
while (_consumedOpTime == theReplSet->lastOpTimeWritten) {
_lastOpCond.wait(lock);
}
}
markOplog();
}
catch (DBException &e) {
clearTarget = true;
log() << "replset tracking exception: " << e.getInfo() << rsLog;
sleepsecs(1);
}
catch (std::exception &e2) {
clearTarget = true;
log() << "replset tracking error" << e2.what() << rsLog;
sleepsecs(1);
}
if (clearTarget) {
boost::unique_lock<boost::mutex> lock(_mutex);
_oplogMarkerTarget = NULL;
}
}
cc().shutdown();
}
void BackgroundSync::producerThread() {
{
boost::unique_lock<boost::mutex> lock(_mutex);
_opSyncInProgress = true;
}
Client::initThread("rsBackgroundSync");
replLocalAuth();
uint32_t timeToSleep = 0;
while (!_opSyncShouldExit) {
try {
if (timeToSleep) {
{
boost::unique_lock<boost::mutex> lck(_mutex);
_opSyncRunning = false;
// notify other threads that we are not running
_opSyncRunningCondVar.notify_all();
}
for (uint32_t i = 0; i < timeToSleep; i++) {
sleepsecs(1);
// get out if we need to
if (_opSyncShouldExit) { break; }
}
timeToSleep = 0;
}
// get out if we need to
if (_opSyncShouldExit) { break; }
{
boost::unique_lock<boost::mutex> lck(_mutex);
_opSyncRunning = false;
while (!_opSyncShouldRun && !_opSyncShouldExit) {
// notify other threads that we are not running
_opSyncRunningCondVar.notify_all();
// wait for permission that we can run
_opSyncCanRunCondVar.wait(lck);
}
// notify other threads that we are running
_opSyncRunningCondVar.notify_all();
_opSyncRunning = true;
}
// get out if we need to
if (_opSyncShouldExit) { break; }
MemberState state = theReplSet->state();
if (state.fatal() || state.startup()) {
timeToSleep = 5;
continue;
}
// this does the work of reading a remote oplog
// and writing it to our oplog
timeToSleep = produce();
}
catch (DBException& e) {
sethbmsg(str::stream() << "db exception in producer: " << e.toString());
timeToSleep = 10;
}
catch (std::exception& e2) {
sethbmsg(str::stream() << "exception in producer: " << e2.what());
timeToSleep = 10;
}
}
cc().shutdown();
{
boost::unique_lock<boost::mutex> lock(_mutex);
_opSyncRunning = false;
_opSyncInProgress = false;
}
}
