void BackgroundSync::_producerThread() {
MemberState state = theReplSet->state();
// we want to pause when the state changes to primary
if (isAssumingPrimary() || state.primary()) {
if (!_pause) {
stop();
}
sleepsecs(1);
return;
}
if (state.startup()) {
sleepsecs(1);
return;
}
OperationContextImpl txn;
// We need to wait until initial sync has started.
if (_replCoord->getMyLastOptime().isNull()) {
sleepsecs(1);
return;
}
// we want to unpause when we're no longer primary
// start() also loads _lastOpTimeFetched, which we know is set from the "if"
else if (_pause) {
start(&txn);
}
produce(&txn);
}
void BackgroundSync::_producerThread() {
MemberState state = theReplSet->state();
// we want to pause when the state changes to primary
if (isAssumingPrimary() || state.primary()) {
if (!_pause) {
stop();
}
sleepsecs(1);
return;
}
if (state.fatal() || state.startup()) {
sleepsecs(5);
return;
}
// if this member has an empty oplog, we cannot start syncing
if (theReplSet->lastOpTimeWritten.isNull()) {
sleepsecs(1);
return;
}
// we want to unpause when we're no longer primary
// start() also loads _lastOpTimeFetched, which we know is set from the "if"
else if (_pause) {
start();
}
produce();
}
void BackgroundSync::_producerThread(executor::TaskExecutor* taskExecutor) {
const MemberState state = _replCoord->getMemberState();
// we want to pause when the state changes to primary
if (_replCoord->isWaitingForApplierToDrain() || state.primary()) {
if (!isPaused()) {
stop();
}
sleepsecs(1);
return;
}
// TODO(spencer): Use a condition variable to await loading a config.
if (state.startup()) {
// Wait for a config to be loaded
sleepsecs(1);
return;
}
// We need to wait until initial sync has started.
if (_replCoord->getMyLastOptime().isNull()) {
sleepsecs(1);
return;
}
// we want to unpause when we're no longer primary
// start() also loads _lastOpTimeFetched, which we know is set from the "if"
OperationContextImpl txn;
if (isPaused()) {
start(&txn);
}
_produce(&txn, taskExecutor);
}
void SyncSourceFeedback::run() {
Client::initThread("SyncSourceFeedbackThread");
OperationContextImpl txn;
bool positionChanged = false;
bool handshakeNeeded = false;
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
while (!inShutdown()) { // TODO(spencer): Remove once legacy repl coordinator is gone.
{
boost::unique_lock<boost::mutex> lock(_mtx);
while (!_positionChanged && !_handshakeNeeded && !_shutdownSignaled) {
_cond.wait(lock);
}
if (_shutdownSignaled) {
break;
}
positionChanged = _positionChanged;
handshakeNeeded = _handshakeNeeded;
_positionChanged = false;
_handshakeNeeded = false;
}
MemberState state = replCoord->getCurrentMemberState();
if (state.primary() || state.fatal() || state.startup()) {
continue;
}
const Member* target = BackgroundSync::get()->getSyncTarget();
if (_syncTarget != target) {
_resetConnection();
_syncTarget = target;
}
if (!hasConnection()) {
// fix connection if need be
if (!target) {
sleepmillis(500);
continue;
}
if (!_connect(&txn, target->fullName())) {
sleepmillis(500);
continue;
}
}
if (handshakeNeeded) {
if (!replHandshake(&txn)) {
boost::unique_lock<boost::mutex> lock(_mtx);
_handshakeNeeded = true;
continue;
}
}
if (positionChanged) {
if (!updateUpstream(&txn)) {
boost::unique_lock<boost::mutex> lock(_mtx);
_positionChanged = true;
}
}
}
cc().shutdown();
}
void RSDataSync::_run() {
Client::initThread("rsSync");
AuthorizationSession::get(cc())->grantInternalAuthorization();
// Overwrite prefetch index mode in BackgroundSync if ReplSettings has a mode set.
auto&& replSettings = _replCoord->getSettings();
if (replSettings.isPrefetchIndexModeSet())
_replCoord->setIndexPrefetchConfig(replSettings.getPrefetchIndexMode());
while (!_bgsync->inShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (_replCoord->getMemberState().startup()) {
warning() << "did not receive a valid config yet";
sleepsecs(1);
continue;
}
const MemberState memberState = _replCoord->getMemberState();
// TODO(siyuan) Control the behavior using applier state.
// An arbiter can never transition to any other state, and doesn't replicate, ever
if (memberState.arbiter()) {
break;
}
// If we are removed then we don't belong to the set anymore
if (memberState.removed()) {
sleepsecs(5);
continue;
}
try {
if (_replCoord->getApplierState() == ReplicationCoordinator::ApplierState::Stopped) {
sleepsecs(1);
continue;
}
auto status = _replCoord->setFollowerMode(MemberState::RS_RECOVERING);
if (!status.isOK()) {
LOG(2) << "Failed to transition to RECOVERING to start data replication"
<< causedBy(status);
continue;
}
// Once we call into SyncTail::oplogApplication we never return, so this code only runs
// at startup. It is not valid to transition from PRIMARY to RECOVERING ever, or from
// SECONDARY to RECOVERING without holding a global X lock, so we invariant to make
// sure this never happens.
invariant(!memberState.primary() && !memberState.secondary());
SyncTail(_bgsync, multiSyncApply).oplogApplication(_replCoord);
} catch (...) {
auto status = exceptionToStatus();
severe() << "Exception thrown in RSDataSync: " << redact(status);
std::terminate();
}
}
}
void BackgroundSync::notifierThread() {
Client::initThread("rsSyncNotifier");
replLocalAuth();
// This makes the initial connection to our sync source for oplog position notification.
// It also sets the supportsUpdater flag so we know which method to use.
// If this function fails, we ignore that situation because it will be taken care of
// the first time markOplog() is called in the loop below.
{
boost::unique_lock<boost::mutex> oplogLockSSF(theReplSet->syncSourceFeedback.oplock);
connectOplogNotifier();
}
theReplSet->syncSourceFeedback.go();
while (!inShutdown()) {
bool clearTarget = false;
if (!theReplSet) {
sleepsecs(5);
continue;
}
MemberState state = theReplSet->state();
if (state.primary() || state.fatal() || state.startup()) {
sleepsecs(5);
continue;
}
try {
{
boost::unique_lock<boost::mutex> lock(_lastOpMutex);
while (_consumedOpTime == theReplSet->lastOpTimeWritten) {
_lastOpCond.wait(lock);
}
}
markOplog();
}
catch (DBException &e) {
clearTarget = true;
log() << "replset tracking exception: " << e.getInfo() << rsLog;
sleepsecs(1);
}
catch (std::exception &e2) {
clearTarget = true;
log() << "replset tracking error" << e2.what() << rsLog;
sleepsecs(1);
}
if (clearTarget) {
boost::unique_lock<boost::mutex> lock(_mutex);
_oplogMarkerTarget = NULL;
}
}
cc().shutdown();
}
void RSDataSync::_run() {
Client::initThread("rsSync");
AuthorizationSession::get(cc())->grantInternalAuthorization();
// Overwrite prefetch index mode in BackgroundSync if ReplSettings has a mode set.
auto&& replSettings = _replCoord->getSettings();
if (replSettings.isPrefetchIndexModeSet())
_replCoord->setIndexPrefetchConfig(replSettings.getPrefetchIndexMode());
while (!_isInShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (_replCoord->getMemberState().startup()) {
warning() << "did not receive a valid config yet";
sleepsecs(1);
continue;
}
const MemberState memberState = _replCoord->getMemberState();
// An arbiter can never transition to any other state, and doesn't replicate, ever
if (memberState.arbiter()) {
break;
}
// If we are removed then we don't belong to the set anymore
if (memberState.removed()) {
sleepsecs(5);
continue;
}
try {
if (memberState.primary() && !_replCoord->isWaitingForApplierToDrain()) {
sleepsecs(1);
continue;
}
if (!_replCoord->setFollowerMode(MemberState::RS_RECOVERING)) {
continue;
}
SyncTail tail(_bgsync, multiSyncApply);
tail.oplogApplication(_replCoord, [this]() { return _isInShutdown(); });
} catch (...) {
std::terminate();
}
}
LockGuard lk(_mutex);
_inShutdown = false;
_stopped = true;
}
void SyncSourceFeedback::run() {
Client::initThread("SyncSourceFeedback");
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
while (true) { // breaks once _shutdownSignaled is true
{
stdx::unique_lock<stdx::mutex> lock(_mtx);
while (!_positionChanged && !_shutdownSignaled) {
if (_cond.wait_for(lock, _keepAliveInterval) == stdx::cv_status::timeout) {
break;
}
}
if (_shutdownSignaled) {
break;
}
_positionChanged = false;
}
auto txn = cc().makeOperationContext();
MemberState state = replCoord->getMemberState();
if (state.primary() || state.startup()) {
_resetConnection();
continue;
}
const HostAndPort target = BackgroundSync::get()->getSyncTarget();
if (_syncTarget != target) {
_resetConnection();
_syncTarget = target;
}
if (!hasConnection()) {
// fix connection if need be
if (target.empty()) {
sleepmillis(500);
stdx::unique_lock<stdx::mutex> lock(_mtx);
_positionChanged = true;
continue;
}
if (!_connect(txn.get(), target)) {
sleepmillis(500);
stdx::unique_lock<stdx::mutex> lock(_mtx);
_positionChanged = true;
continue;
}
}
Status status = updateUpstream(txn.get());
if (!status.isOK()) {
sleepmillis(500);
stdx::unique_lock<stdx::mutex> lock(_mtx);
_positionChanged = true;
}
}
}
void BackgroundSync::notifierThread() {
Client::initThread("rsSyncNotifier");
replLocalAuth();
theReplSet->syncSourceFeedback.go();
while (!inShutdown()) {
bool clearTarget = false;
if (!theReplSet) {
sleepsecs(5);
continue;
}
MemberState state = theReplSet->state();
if (state.primary() || state.fatal() || state.startup()) {
sleepsecs(5);
continue;
}
try {
{
boost::unique_lock<boost::mutex> lock(_lastOpMutex);
while (_consumedOpTime == theReplSet->lastOpTimeWritten) {
_lastOpCond.wait(lock);
}
}
markOplog();
}
catch (DBException &e) {
clearTarget = true;
log() << "replset tracking exception: " << e.getInfo() << rsLog;
sleepsecs(1);
}
catch (std::exception &e2) {
clearTarget = true;
log() << "replset tracking error" << e2.what() << rsLog;
sleepsecs(1);
}
if (clearTarget) {
boost::unique_lock<boost::mutex> lock(_mutex);
_oplogMarkerTarget = NULL;
}
}
cc().shutdown();
}
virtual bool run(OperationContext* txn,
const string& dbname,
BSONObj& cmdObj,
int,
string& errmsg,
BSONObjBuilder& result,
bool fromRepl) {
ScopedTransaction transaction(txn, MODE_X);
Lock::GlobalWrite globalWriteLock(txn->lockState());
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (getGlobalReplicationCoordinator()->getSettings().usingReplSets()) {
const MemberState memberState = replCoord->getMemberState();
if (memberState.startup()) {
return appendCommandStatus(result, Status(ErrorCodes::NotYetInitialized,
"no replication yet active"));
}
if (memberState.primary() ||
!replCoord->setFollowerMode(MemberState::RS_STARTUP2)) {
return appendCommandStatus(result, Status(ErrorCodes::NotSecondary,
"primaries cannot resync"));
}
BackgroundSync::get()->setInitialSyncRequestedFlag(true);
return true;
}
// below this comment pertains only to master/slave replication
if ( cmdObj.getBoolField( "force" ) ) {
if ( !waitForSyncToFinish(txn, errmsg ) )
return false;
replAllDead = "resync forced";
}
// TODO(dannenberg) replAllDead is bad and should be removed when masterslave is removed
if (!replAllDead) {
errmsg = "not dead, no need to resync";
return false;
}
if ( !waitForSyncToFinish(txn, errmsg ) )
return false;
ReplSource::forceResyncDead( txn, "client" );
result.append( "info", "triggered resync for all sources" );
return true;
}
void BackgroundSync::_producerThread() {
const MemberState state = _replCoord->getMemberState();
// we want to pause when the state changes to primary
if (_replCoord->isWaitingForApplierToDrain() || state.primary()) {
if (!isPaused()) {
stop();
}
if (_replCoord->isWaitingForApplierToDrain()) {
// Signal to consumers that we have entered the paused state if the signal isn't already
// in the queue.
const boost::optional<BSONObj> lastObjectPushed = _buffer.lastObjectPushed();
if (!lastObjectPushed || !lastObjectPushed->isEmpty()) {
const BSONObj sentinelDoc;
_buffer.pushEvenIfFull(sentinelDoc);
bufferCountGauge.increment();
bufferSizeGauge.increment(sentinelDoc.objsize());
}
}
sleepsecs(1);
return;
}
// TODO(spencer): Use a condition variable to await loading a config.
if (state.startup()) {
// Wait for a config to be loaded
sleepsecs(1);
return;
}
// We need to wait until initial sync has started.
if (_replCoord->getMyLastOptime().isNull()) {
sleepsecs(1);
return;
}
// we want to unpause when we're no longer primary
// start() also loads _lastOpTimeFetched, which we know is set from the "if"
OperationContextImpl txn;
if (isPaused()) {
start(&txn);
}
_produce(&txn);
}
void BackgroundSync::_runProducer() {
const MemberState state = _replCoord->getMemberState();
// Stop when the state changes to primary.
//
// TODO(siyuan) Drain mode should imply we're the primary. Fix this condition and the one below
// after fixing step-down during drain mode.
if (!_replCoord->isCatchingUp() &&
(_replCoord->isWaitingForApplierToDrain() || state.primary())) {
if (!isStopped()) {
stop();
}
if (_replCoord->isWaitingForApplierToDrain()) {
auto txn = cc().makeOperationContext();
_signalNoNewDataForApplier(txn.get());
}
sleepsecs(1);
return;
}
// TODO(spencer): Use a condition variable to await loading a config.
if (state.startup()) {
// Wait for a config to be loaded
sleepsecs(1);
return;
}
// We need to wait until initial sync has started.
if (_replCoord->getMyLastAppliedOpTime().isNull()) {
sleepsecs(1);
return;
}
// we want to start when we're no longer primary
// start() also loads _lastOpTimeFetched, which we know is set from the "if"
auto txn = cc().makeOperationContext();
if (isStopped()) {
start(txn.get());
}
_produce(txn.get());
}
void BackgroundSync::_producerThread(
ReplicationCoordinatorExternalState* replicationCoordinatorExternalState) {
const MemberState state = _replCoord->getMemberState();
// Stop when the state changes to primary.
if (_replCoord->isWaitingForApplierToDrain() || state.primary()) {
if (!isStopped()) {
stop();
}
if (_replCoord->isWaitingForApplierToDrain()) {
_signalNoNewDataForApplier();
}
sleepsecs(1);
return;
}
// TODO(spencer): Use a condition variable to await loading a config.
if (state.startup()) {
// Wait for a config to be loaded
sleepsecs(1);
return;
}
// We need to wait until initial sync has started.
if (_replCoord->getMyLastAppliedOpTime().isNull()) {
sleepsecs(1);
return;
}
// we want to start when we're no longer primary
// start() also loads _lastOpTimeFetched, which we know is set from the "if"
const ServiceContext::UniqueOperationContext txnPtr = cc().makeOperationContext();
OperationContext& txn = *txnPtr;
if (isStopped()) {
start(&txn);
}
_produce(&txn, replicationCoordinatorExternalState);
}
void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* bgsync) {
Client::initThread("SyncSourceFeedback");
HostAndPort syncTarget;
// keepAliveInterval indicates how frequently to forward progress in the absence of updates.
Milliseconds keepAliveInterval(0);
while (true) { // breaks once _shutdownSignaled is true
auto txn = cc().makeOperationContext();
if (keepAliveInterval == Milliseconds(0)) {
keepAliveInterval = calculateKeepAliveInterval(txn.get(), _mtx);
}
{
// Take SyncSourceFeedback lock before calling into ReplicationCoordinator
// to avoid deadlock because ReplicationCoordinator could conceivably calling back into
// this class.
stdx::unique_lock<stdx::mutex> lock(_mtx);
while (!_positionChanged && !_shutdownSignaled) {
if (_cond.wait_for(lock, keepAliveInterval.toSystemDuration()) ==
stdx::cv_status::timeout) {
MemberState state = ReplicationCoordinator::get(txn.get())->getMemberState();
if (!(state.primary() || state.startup())) {
break;
}
}
}
if (_shutdownSignaled) {
break;
}
_positionChanged = false;
}
{
stdx::lock_guard<stdx::mutex> lock(_mtx);
MemberState state = ReplicationCoordinator::get(txn.get())->getMemberState();
if (state.primary() || state.startup()) {
continue;
}
}
const HostAndPort target = bgsync->getSyncTarget();
// Log sync source changes.
if (target.empty()) {
if (syncTarget != target) {
syncTarget = target;
}
// Loop back around again; the keepalive functionality will cause us to retry
continue;
}
if (syncTarget != target) {
LOG(1) << "setting syncSourceFeedback to " << target;
syncTarget = target;
// Update keepalive value from config.
auto oldKeepAliveInterval = keepAliveInterval;
keepAliveInterval = calculateKeepAliveInterval(txn.get(), _mtx);
if (oldKeepAliveInterval != keepAliveInterval) {
LOG(1) << "new syncSourceFeedback keep alive duration = " << keepAliveInterval
<< " (previously " << oldKeepAliveInterval << ")";
}
}
Reporter reporter(
executor,
makePrepareReplSetUpdatePositionCommandFn(txn.get(), _mtx, syncTarget, bgsync),
syncTarget,
keepAliveInterval);
{
stdx::lock_guard<stdx::mutex> lock(_mtx);
_reporter = &reporter;
}
ON_BLOCK_EXIT([this]() {
stdx::lock_guard<stdx::mutex> lock(_mtx);
_reporter = nullptr;
});
auto status = _updateUpstream(txn.get(), bgsync);
if (!status.isOK()) {
LOG(1) << "The replication progress command (replSetUpdatePosition) failed and will be "
"retried: "
<< status;
}
}
}
void ReplSetImpl::_summarizeStatus(BSONObjBuilder& b) const {
vector<BSONObj> v;
const Member *_self = this->_self;
verify( _self );
MemberState myState = box.getState();
const HostAndPort syncTarget = BackgroundSync::get()->getSyncTarget();
// add self
{
BSONObjBuilder bb;
bb.append("_id", (int) _self->id());
bb.append("name", _self->fullName());
bb.append("health", 1.0);
bb.append("state", (int)myState.s);
bb.append("stateStr", myState.toString());
bb.append("uptime", (unsigned)(time(0) - serverGlobalParams.started));
if (!_self->config().arbiterOnly) {
bb.appendTimestamp("optime", lastOpTimeWritten.asDate());
bb.appendDate("optimeDate", lastOpTimeWritten.getSecs() * 1000LL);
}
int maintenance = _maintenanceMode;
if (maintenance) {
bb.append("maintenanceMode", maintenance);
}
if ( !syncTarget.empty() &&
(myState != MemberState::RS_PRIMARY) &&
(myState != MemberState::RS_REMOVED) ) {
bb.append("syncingTo", syncTarget.toString());
}
if (theReplSet) {
string s = theReplSet->hbmsg();
if( !s.empty() )
bb.append("infoMessage", s);
if (myState == MemberState::RS_PRIMARY) {
bb.appendTimestamp("electionTime", theReplSet->getElectionTime().asDate());
bb.appendDate("electionDate", theReplSet->getElectionTime().getSecs() * 1000LL);
}
}
bb.append("self", true);
v.push_back(bb.obj());
}
Member *m =_members.head();
while( m ) {
BSONObjBuilder bb;
bb.append("_id", (int) m->id());
bb.append("name", m->fullName());
double h = m->hbinfo().health;
bb.append("health", h);
bb.append("state", (int) m->state().s);
if( h == 0 ) {
// if we can't connect the state info is from the past and could be confusing to show
bb.append("stateStr", "(not reachable/healthy)");
}
else {
bb.append("stateStr", m->state().toString());
}
bb.append("uptime", (unsigned) (m->hbinfo().upSince ? (time(0)-m->hbinfo().upSince) : 0));
if (!m->config().arbiterOnly) {
bb.appendTimestamp("optime", m->hbinfo().opTime.asDate());
bb.appendDate("optimeDate", m->hbinfo().opTime.getSecs() * 1000LL);
}
bb.appendTimeT("lastHeartbeat", m->hbinfo().lastHeartbeat);
bb.appendTimeT("lastHeartbeatRecv", m->hbinfo().lastHeartbeatRecv);
bb.append("pingMs", m->hbinfo().ping);
string s = m->lhb();
if( !s.empty() )
bb.append("lastHeartbeatMessage", s);
if (m->hbinfo().authIssue) {
bb.append("authenticated", false);
}
string syncingTo = m->hbinfo().syncingTo;
if (!syncingTo.empty()) {
bb.append("syncingTo", syncingTo);
}
if (m->state() == MemberState::RS_PRIMARY) {
bb.appendTimestamp("electionTime", m->hbinfo().electionTime.asDate());
bb.appendDate("electionDate", m->hbinfo().electionTime.getSecs() * 1000LL);
}
v.push_back(bb.obj());
m = m->next();
}
sort(v.begin(), v.end());
b.append("set", name());
b.appendTimeT("date", time(0));
b.append("myState", myState.s);
if ( !syncTarget.empty() &&
(myState != MemberState::RS_PRIMARY) &&
(myState != MemberState::RS_REMOVED) ) {
b.append("syncingTo", syncTarget.toString());
}
b.append("members", v);
}
void ReplSetImpl::_summarizeStatus(BSONObjBuilder& b) const {
vector<BSONObj> v;
const Member *_self = this->_self;
verify( _self );
MemberState myState = box.getState();
// add self
{
BSONObjBuilder bb;
bb.append("_id", (int) _self->id());
bb.append("name", _self->fullName());
bb.append("health", 1.0);
bb.append("state", (int)myState.s);
bb.append("stateStr", myState.toString());
bb.append("uptime", (unsigned)(time(0) - cmdLine.started));
if (!_self->config().arbiterOnly) {
GTID lastLive;
GTID lastUnapplied;
GTID minLive;
GTID minUnapplied;
gtidManager->getGTIDs(
&lastLive,
&lastUnapplied,
&minLive,
&minUnapplied
);
bb.appendDate("optimeDate", gtidManager->getCurrTimestamp());
bb.append("lastGTID", lastLive.toString());
bb.append("lastUnappliedGTID", lastUnapplied.toString());
bb.append("minLiveGTID", minLive.toString());
bb.append("minUnappliedGTID", minUnapplied.toString());
bb.append("oplogVersion", ReplSetConfig::OPLOG_VERSION);
}
int maintenance = _maintenanceMode;
if (maintenance) {
bb.append("maintenanceMode", maintenance);
}
if (theReplSet) {
string s = theReplSet->hbmsg();
if( !s.empty() )
bb.append("errmsg", s);
}
bb.append("self", true);
v.push_back(bb.obj());
}
Member *m =_members.head();
while( m ) {
BSONObjBuilder bb;
bb.append("_id", (int) m->id());
bb.append("name", m->fullName());
double h = m->hbinfo().health;
bb.append("health", h);
bb.append("state", (int) m->state().s);
if( h == 0 ) {
// if we can't connect the state info is from the past and could be confusing to show
bb.append("stateStr", "(not reachable/healthy)");
}
else {
bb.append("stateStr", m->state().toString());
}
bb.append("uptime", (unsigned) (m->hbinfo().upSince ? (time(0)-m->hbinfo().upSince) : 0));
if (!m->config().arbiterOnly) {
bb.appendDate("optimeDate", m->hbinfo().opTime);
bb.append("lastGTID", m->hbinfo().gtid.toString());
bb.append("lastUnappliedGTID", m->hbinfo().lastUnappliedGTID.toString());
bb.append("minLiveGTID", m->hbinfo().minLiveGTID.toString());
bb.append("minUnappliedGTID", m->hbinfo().minUnappliedGTID.toString());
bb.append("oplogVersion", m->hbinfo().oplogVersion);
}
bb.appendTimeT("lastHeartbeat", m->hbinfo().lastHeartbeat);
bb.appendTimeT("lastHeartbeatRecv", m->getLastRecvHeartbeat());
bb.append("pingMs", m->hbinfo().ping);
string s = m->lhb();
if( !s.empty() )
bb.append("lastHeartbeatMessage", s);
if (m->hbinfo().authIssue) {
bb.append("authenticated", false);
}
string syncingTo = m->hbinfo().syncingTo;
if (!syncingTo.empty()) {
bb.append("syncingTo", syncingTo);
}
v.push_back(bb.obj());
m = m->next();
}
sort(v.begin(), v.end());
b.append("set", name());
b.appendTimeT("date", time(0));
b.append("myState", myState.s);
const Member *syncTarget = BackgroundSync::get()->getSyncTarget();
if ( syncTarget &&
(myState != MemberState::RS_PRIMARY) &&
(myState != MemberState::RS_SHUNNED) ) {
b.append("syncingTo", syncTarget->fullName());
}
b.append("members", v);
if( replSetBlind )
b.append("blind",true); // to avoid confusion if set...normally never set except for testing.
}
void BackgroundSync::producerThread() {
{
boost::unique_lock<boost::mutex> lock(_mutex);
_opSyncInProgress = true;
}
Client::initThread("rsBackgroundSync");
replLocalAuth();
uint32_t timeToSleep = 0;
while (!_opSyncShouldExit) {
try {
if (timeToSleep) {
{
boost::unique_lock<boost::mutex> lck(_mutex);
_opSyncRunning = false;
// notify other threads that we are not running
_opSyncRunningCondVar.notify_all();
}
for (uint32_t i = 0; i < timeToSleep; i++) {
sleepsecs(1);
// get out if we need to
if (_opSyncShouldExit) { break; }
}
timeToSleep = 0;
}
// get out if we need to
if (_opSyncShouldExit) { break; }
{
boost::unique_lock<boost::mutex> lck(_mutex);
_opSyncRunning = false;
while (!_opSyncShouldRun && !_opSyncShouldExit) {
// notify other threads that we are not running
_opSyncRunningCondVar.notify_all();
// wait for permission that we can run
_opSyncCanRunCondVar.wait(lck);
}
// notify other threads that we are running
_opSyncRunningCondVar.notify_all();
_opSyncRunning = true;
}
// get out if we need to
if (_opSyncShouldExit) { break; }
MemberState state = theReplSet->state();
if (state.fatal() || state.startup()) {
timeToSleep = 5;
continue;
}
// this does the work of reading a remote oplog
// and writing it to our oplog
timeToSleep = produce();
}
catch (DBException& e) {
sethbmsg(str::stream() << "db exception in producer: " << e.toString());
timeToSleep = 10;
}
catch (std::exception& e2) {
sethbmsg(str::stream() << "exception in producer: " << e2.what());
timeToSleep = 10;
}
}
cc().shutdown();
{
boost::unique_lock<boost::mutex> lock(_mutex);
_opSyncRunning = false;
_opSyncInProgress = false;
}
}
void ReplSetImpl::_summarizeStatus(BSONObjBuilder& b) const {
vector<BSONObj> v;
const Member *_self = this->_self;
verify( _self );
MemberState myState = box.getState();
// add self
{
BSONObjBuilder bb;
bb.append("_id", (int) _self->id());
bb.append("name", _self->fullName());
bb.append("health", 1.0);
bb.append("state", (int)myState.s);
bb.append("stateStr", myState.toString());
bb.append("uptime", (unsigned)(time(0) - cmdLine.started));
if (!_self->config().arbiterOnly) {
bb.appendTimestamp("optime", lastOpTimeWritten.asDate());
bb.appendDate("optimeDate", lastOpTimeWritten.getSecs() * 1000LL);
}
int maintenance = _maintenanceMode;
if (maintenance) {
bb.append("maintenanceMode", maintenance);
}
if (theReplSet) {
string s = theReplSet->hbmsg();
if( !s.empty() )
bb.append("errmsg", s);
}
bb.append("self", true);
v.push_back(bb.obj());
}
Member *m =_members.head();
while( m ) {
BSONObjBuilder bb;
bb.append("_id", (int) m->id());
bb.append("name", m->fullName());
double h = m->hbinfo().health;
bb.append("health", h);
bb.append("state", (int) m->state().s);
if( h == 0 ) {
// if we can't connect the state info is from the past and could be confusing to show
bb.append("stateStr", "(not reachable/healthy)");
}
else {
bb.append("stateStr", m->state().toString());
}
bb.append("uptime", (unsigned) (m->hbinfo().upSince ? (time(0)-m->hbinfo().upSince) : 0));
if (!m->config().arbiterOnly) {
bb.appendTimestamp("optime", m->hbinfo().opTime.asDate());
bb.appendDate("optimeDate", m->hbinfo().opTime.getSecs() * 1000LL);
}
bb.appendTimeT("lastHeartbeat", m->hbinfo().lastHeartbeat);
bb.append("pingMs", m->hbinfo().ping);
string s = m->lhb();
if( !s.empty() )
bb.append("errmsg", s);
if (m->hbinfo().authIssue) {
bb.append("authenticated", false);
}
v.push_back(bb.obj());
m = m->next();
}
sort(v.begin(), v.end());
b.append("set", name());
b.appendTimeT("date", time(0));
b.append("myState", myState.s);
const Member *syncTarget = _currentSyncTarget;
if ( syncTarget &&
(myState != MemberState::RS_PRIMARY) &&
(myState != MemberState::RS_SHUNNED) ) {
b.append("syncingTo", syncTarget->fullName());
}
b.append("members", v);
if( replSetBlind )
b.append("blind",true); // to avoid confusion if set...normally never set except for testing.
}
void runSyncThread() {
Client::initThread("rsSync");
AuthorizationSession::get(cc())->grantInternalAuthorization();
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
// Set initial indexPrefetch setting
const std::string& prefetch = replCoord->getSettings().rsIndexPrefetch;
if (!prefetch.empty()) {
BackgroundSync::IndexPrefetchConfig prefetchConfig = BackgroundSync::PREFETCH_ALL;
if (prefetch == "none")
prefetchConfig = BackgroundSync::PREFETCH_NONE;
else if (prefetch == "_id_only")
prefetchConfig = BackgroundSync::PREFETCH_ID_ONLY;
else if (prefetch == "all")
prefetchConfig = BackgroundSync::PREFETCH_ALL;
else {
warning() << "unrecognized indexPrefetch setting " << prefetch << ", defaulting "
<< "to \"all\"";
}
BackgroundSync::get()->setIndexPrefetchConfig(prefetchConfig);
}
while (!inShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (replCoord->getMemberState().startup()) {
warning() << "did not receive a valid config yet";
sleepsecs(1);
continue;
}
const MemberState memberState = replCoord->getMemberState();
// An arbiter can never transition to any other state, and doesn't replicate, ever
if (memberState.arbiter()) {
break;
}
// If we are removed then we don't belong to the set anymore
if (memberState.removed()) {
sleepsecs(5);
continue;
}
try {
if (memberState.primary() && !replCoord->isWaitingForApplierToDrain()) {
sleepsecs(1);
continue;
}
bool initialSyncRequested = BackgroundSync::get()->getInitialSyncRequestedFlag();
// Check criteria for doing an initial sync:
// 1. If the oplog is empty, do an initial sync
// 2. If minValid has _initialSyncFlag set, do an initial sync
// 3. If initialSyncRequested is true
if (getGlobalReplicationCoordinator()->getMyLastOptime().isNull() ||
getInitialSyncFlag() || initialSyncRequested) {
syncDoInitialSync();
continue; // start from top again in case sync failed.
}
if (!replCoord->setFollowerMode(MemberState::RS_RECOVERING)) {
continue;
}
/* we have some data. continue tailing. */
SyncTail tail(BackgroundSync::get(), multiSyncApply);
tail.oplogApplication();
} catch (const DBException& e) {
log() << "Received exception while syncing: " << e.toString();
sleepsecs(10);
} catch (const std::exception& e) {
log() << "Received exception while syncing: " << e.what();
sleepsecs(10);
}
}
}
void runSyncThread() {
Client::initThread("rsSync");
replLocalAuth();
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
// Set initial indexPrefetch setting
std::string& prefetch = replCoord->getSettings().rsIndexPrefetch;
if (!prefetch.empty()) {
BackgroundSync::IndexPrefetchConfig prefetchConfig = BackgroundSync::PREFETCH_ALL;
if (prefetch == "none")
prefetchConfig = BackgroundSync::PREFETCH_NONE;
else if (prefetch == "_id_only")
prefetchConfig = BackgroundSync::PREFETCH_ID_ONLY;
else if (prefetch == "all")
prefetchConfig = BackgroundSync::PREFETCH_ALL;
else {
warning() << "unrecognized indexPrefetch setting " << prefetch << ", defaulting "
<< "to \"all\"";
}
BackgroundSync::get()->setIndexPrefetchConfig(prefetchConfig);
}
while (!inShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (replCoord->getReplicationMode() != ReplicationCoordinator::modeReplSet) {
log() << "replSet warning did not receive a valid config yet, sleeping 5 seconds "
<< rsLog;
sleepsecs(5);
continue;
}
const MemberState memberState = replCoord->getCurrentMemberState();
if (replCoord->getCurrentMemberState().arbiter()) {
break;
}
try {
if (memberState.primary()) {
sleepsecs(1);
continue;
}
bool initialSyncRequested = BackgroundSync::get()->getInitialSyncRequestedFlag();
// Check criteria for doing an initial sync:
// 1. If the oplog is empty, do an initial sync
// 2. If minValid has _initialSyncFlag set, do an initial sync
// 3. If initialSyncRequested is true
if (getGlobalReplicationCoordinator()->getMyLastOptime().isNull() ||
getInitialSyncFlag() ||
initialSyncRequested) {
syncDoInitialSync();
continue; // start from top again in case sync failed.
}
replCoord->setFollowerMode(MemberState::RS_RECOVERING);
/* we have some data. continue tailing. */
SyncTail tail(BackgroundSync::get(), multiSyncApply);
tail.oplogApplication();
}
catch(const DBException& e) {
log() << "Received exception while syncing: " << e.toString();
sleepsecs(10);
}
catch(...) {
sethbmsg("unexpected exception in syncThread()");
// TODO : SET NOT SECONDARY here?
sleepsecs(60);
}
}
cc().shutdown();
}
void runSyncThread() {
Client::initThread("rsSync");
AuthorizationSession::get(cc())->grantInternalAuthorization();
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
// Overwrite prefetch index mode in BackgroundSync if ReplSettings has a mode set.
ReplSettings replSettings = replCoord->getSettings();
if (replSettings.isPrefetchIndexModeSet())
BackgroundSync::get()->setIndexPrefetchConfig(replSettings.getPrefetchIndexMode());
while (!inShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (replCoord->getMemberState().startup()) {
warning() << "did not receive a valid config yet";
sleepsecs(1);
continue;
}
const MemberState memberState = replCoord->getMemberState();
// An arbiter can never transition to any other state, and doesn't replicate, ever
if (memberState.arbiter()) {
break;
}
// If we are removed then we don't belong to the set anymore
if (memberState.removed()) {
sleepsecs(5);
continue;
}
try {
if (memberState.primary() && !replCoord->isWaitingForApplierToDrain()) {
sleepsecs(1);
continue;
}
bool initialSyncRequested = BackgroundSync::get()->getInitialSyncRequestedFlag();
// Check criteria for doing an initial sync:
// 1. If the oplog is empty, do an initial sync
// 2. If minValid has _initialSyncFlag set, do an initial sync
// 3. If initialSyncRequested is true
if (getGlobalReplicationCoordinator()->getMyLastOptime().isNull() ||
getInitialSyncFlag() || initialSyncRequested) {
syncDoInitialSync();
continue; // start from top again in case sync failed.
}
if (!replCoord->setFollowerMode(MemberState::RS_RECOVERING)) {
continue;
}
/* we have some data. continue tailing. */
SyncTail tail(BackgroundSync::get(), multiSyncApply);
tail.oplogApplication();
} catch (...) {
std::terminate();
}
}
}
