void runSyncThread() {
Client::initThread("rsSync");
AuthorizationSession::get(cc())->grantInternalAuthorization();
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
// Set initial indexPrefetch setting
const std::string& prefetch = replCoord->getSettings().rsIndexPrefetch;
if (!prefetch.empty()) {
BackgroundSync::IndexPrefetchConfig prefetchConfig = BackgroundSync::PREFETCH_ALL;
if (prefetch == "none")
prefetchConfig = BackgroundSync::PREFETCH_NONE;
else if (prefetch == "_id_only")
prefetchConfig = BackgroundSync::PREFETCH_ID_ONLY;
else if (prefetch == "all")
prefetchConfig = BackgroundSync::PREFETCH_ALL;
else {
warning() << "unrecognized indexPrefetch setting " << prefetch << ", defaulting "
<< "to \"all\"";
}
BackgroundSync::get()->setIndexPrefetchConfig(prefetchConfig);
}
while (!inShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (replCoord->getMemberState().startup()) {
warning() << "did not receive a valid config yet";
sleepsecs(1);
continue;
}
const MemberState memberState = replCoord->getMemberState();
// An arbiter can never transition to any other state, and doesn't replicate, ever
if (memberState.arbiter()) {
break;
}
// If we are removed then we don't belong to the set anymore
if (memberState.removed()) {
sleepsecs(5);
continue;
}
try {
if (memberState.primary() && !replCoord->isWaitingForApplierToDrain()) {
sleepsecs(1);
continue;
}
bool initialSyncRequested = BackgroundSync::get()->getInitialSyncRequestedFlag();
// Check criteria for doing an initial sync:
// 1. If the oplog is empty, do an initial sync
// 2. If minValid has _initialSyncFlag set, do an initial sync
// 3. If initialSyncRequested is true
if (getGlobalReplicationCoordinator()->getMyLastOptime().isNull() ||
getInitialSyncFlag() || initialSyncRequested) {
syncDoInitialSync();
continue; // start from top again in case sync failed.
}
if (!replCoord->setFollowerMode(MemberState::RS_RECOVERING)) {
continue;
}
/* we have some data. continue tailing. */
SyncTail tail(BackgroundSync::get(), multiSyncApply);
tail.oplogApplication();
} catch (const DBException& e) {
log() << "Received exception while syncing: " << e.toString();
sleepsecs(10);
} catch (const std::exception& e) {
log() << "Received exception while syncing: " << e.what();
sleepsecs(10);
}
}
}
void runSyncThread() {
Client::initThread("rsSync");
AuthorizationSession::get(cc())->grantInternalAuthorization();
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
// Overwrite prefetch index mode in BackgroundSync if ReplSettings has a mode set.
ReplSettings replSettings = replCoord->getSettings();
if (replSettings.isPrefetchIndexModeSet())
BackgroundSync::get()->setIndexPrefetchConfig(replSettings.getPrefetchIndexMode());
while (!inShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (replCoord->getMemberState().startup()) {
warning() << "did not receive a valid config yet";
sleepsecs(1);
continue;
}
const MemberState memberState = replCoord->getMemberState();
// An arbiter can never transition to any other state, and doesn't replicate, ever
if (memberState.arbiter()) {
break;
}
// If we are removed then we don't belong to the set anymore
if (memberState.removed()) {
sleepsecs(5);
continue;
}
try {
if (memberState.primary() && !replCoord->isWaitingForApplierToDrain()) {
sleepsecs(1);
continue;
}
bool initialSyncRequested = BackgroundSync::get()->getInitialSyncRequestedFlag();
// Check criteria for doing an initial sync:
// 1. If the oplog is empty, do an initial sync
// 2. If minValid has _initialSyncFlag set, do an initial sync
// 3. If initialSyncRequested is true
if (getGlobalReplicationCoordinator()->getMyLastOptime().isNull() ||
getInitialSyncFlag() || initialSyncRequested) {
syncDoInitialSync();
continue; // start from top again in case sync failed.
}
if (!replCoord->setFollowerMode(MemberState::RS_RECOVERING)) {
continue;
}
/* we have some data. continue tailing. */
SyncTail tail(BackgroundSync::get(), multiSyncApply);
tail.oplogApplication();
} catch (...) {
std::terminate();
}
}
}
void runSyncThread() {
Client::initThread("rsSync");
replLocalAuth();
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
// Set initial indexPrefetch setting
std::string& prefetch = replCoord->getSettings().rsIndexPrefetch;
if (!prefetch.empty()) {
BackgroundSync::IndexPrefetchConfig prefetchConfig = BackgroundSync::PREFETCH_ALL;
if (prefetch == "none")
prefetchConfig = BackgroundSync::PREFETCH_NONE;
else if (prefetch == "_id_only")
prefetchConfig = BackgroundSync::PREFETCH_ID_ONLY;
else if (prefetch == "all")
prefetchConfig = BackgroundSync::PREFETCH_ALL;
else {
warning() << "unrecognized indexPrefetch setting " << prefetch << ", defaulting "
<< "to \"all\"";
}
BackgroundSync::get()->setIndexPrefetchConfig(prefetchConfig);
}
while (!inShutdown()) {
// After a reconfig, we may not be in the replica set anymore, so
// check that we are in the set (and not an arbiter) before
// trying to sync with other replicas.
// TODO(spencer): Use a condition variable to await loading a config
if (replCoord->getReplicationMode() != ReplicationCoordinator::modeReplSet) {
log() << "replSet warning did not receive a valid config yet, sleeping 5 seconds "
<< rsLog;
sleepsecs(5);
continue;
}
const MemberState memberState = replCoord->getCurrentMemberState();
if (replCoord->getCurrentMemberState().arbiter()) {
break;
}
try {
if (memberState.primary()) {
sleepsecs(1);
continue;
}
bool initialSyncRequested = BackgroundSync::get()->getInitialSyncRequestedFlag();
// Check criteria for doing an initial sync:
// 1. If the oplog is empty, do an initial sync
// 2. If minValid has _initialSyncFlag set, do an initial sync
// 3. If initialSyncRequested is true
if (getGlobalReplicationCoordinator()->getMyLastOptime().isNull() ||
getInitialSyncFlag() ||
initialSyncRequested) {
syncDoInitialSync();
continue; // start from top again in case sync failed.
}
replCoord->setFollowerMode(MemberState::RS_RECOVERING);
/* we have some data. continue tailing. */
SyncTail tail(BackgroundSync::get(), multiSyncApply);
tail.oplogApplication();
}
catch(const DBException& e) {
log() << "Received exception while syncing: " << e.toString();
sleepsecs(10);
}
catch(...) {
sethbmsg("unexpected exception in syncThread()");
// TODO : SET NOT SECONDARY here?
sleepsecs(60);
}
}
cc().shutdown();
}
void appendReplicationInfo(OperationContext* txn, BSONObjBuilder& result, int level) {
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (replCoord->getSettings().usingReplSets()) {
IsMasterResponse isMasterResponse;
replCoord->fillIsMasterForReplSet(&isMasterResponse);
result.appendElements(isMasterResponse.toBSON());
if (level) {
replCoord->appendSlaveInfoData(&result);
}
return;
}
// TODO(dannenberg) replAllDead is bad and should be removed when master slave is removed
if (replAllDead) {
result.append("ismaster", 0);
string s = string("dead: ") + replAllDead;
result.append("info", s);
} else {
result.appendBool("ismaster",
getGlobalReplicationCoordinator()->isMasterForReportingPurposes());
}
if (level) {
BSONObjBuilder sources(result.subarrayStart("sources"));
int n = 0;
list<BSONObj> src;
{
const char* localSources = "local.sources";
AutoGetCollectionForRead ctx(txn, localSources);
unique_ptr<PlanExecutor> exec(
InternalPlanner::collectionScan(txn, localSources, ctx.getCollection()));
BSONObj obj;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
src.push_back(obj);
}
}
for (list<BSONObj>::const_iterator i = src.begin(); i != src.end(); i++) {
BSONObj s = *i;
BSONObjBuilder bb;
bb.append(s["host"]);
string sourcename = s["source"].valuestr();
if (sourcename != "main")
bb.append(s["source"]);
{
BSONElement e = s["syncedTo"];
BSONObjBuilder t(bb.subobjStart("syncedTo"));
t.appendDate("time", e.timestampTime());
t.append("inc", e.timestampInc());
t.done();
}
if (level > 1) {
wassert(!txn->lockState()->isLocked());
// note: there is no so-style timeout on this connection; perhaps we should have
// one.
ScopedDbConnection conn(s["host"].valuestr());
DBClientConnection* cliConn = dynamic_cast<DBClientConnection*>(&conn.conn());
if (cliConn && replAuthenticate(cliConn)) {
BSONObj first = conn->findOne((string) "local.oplog.$" + sourcename,
Query().sort(BSON("$natural" << 1)));
BSONObj last = conn->findOne((string) "local.oplog.$" + sourcename,
Query().sort(BSON("$natural" << -1)));
bb.appendDate("masterFirst", first["ts"].timestampTime());
bb.appendDate("masterLast", last["ts"].timestampTime());
const auto lag = (last["ts"].timestampTime() - s["syncedTo"].timestampTime());
bb.append("lagSeconds", durationCount<Milliseconds>(lag) / 1000.0);
}
conn.done();
}
sources.append(BSONObjBuilder::numStr(n++), bb.obj());
}
sources.done();
replCoord->appendSlaveInfoData(&result);
}
}
void appendReplicationInfo(OperationContext* opCtx, BSONObjBuilder& result, int level) {
ReplicationCoordinator* replCoord = ReplicationCoordinator::get(opCtx);
if (replCoord->getSettings().usingReplSets()) {
IsMasterResponse isMasterResponse;
replCoord->fillIsMasterForReplSet(&isMasterResponse);
result.appendElements(isMasterResponse.toBSON());
if (level) {
replCoord->appendSlaveInfoData(&result);
}
return;
}
result.appendBool("ismaster",
ReplicationCoordinator::get(opCtx)->isMasterForReportingPurposes());
if (level) {
BSONObjBuilder sources(result.subarrayStart("sources"));
int n = 0;
list<BSONObj> src;
{
const NamespaceString localSources{"local.sources"};
AutoGetCollectionForReadCommand ctx(opCtx, localSources);
auto exec = InternalPlanner::collectionScan(
opCtx, localSources.ns(), ctx.getCollection(), PlanExecutor::NO_YIELD);
BSONObj obj;
PlanExecutor::ExecState state;
while (PlanExecutor::ADVANCED == (state = exec->getNext(&obj, NULL))) {
src.push_back(obj.getOwned());
}
// Non-yielding collection scans from InternalPlanner will never error.
invariant(PlanExecutor::IS_EOF == state);
}
for (list<BSONObj>::const_iterator i = src.begin(); i != src.end(); i++) {
BSONObj s = *i;
BSONObjBuilder bb;
bb.append(s["host"]);
string sourcename = s["source"].valuestr();
if (sourcename != "main")
bb.append(s["source"]);
{
BSONElement e = s["syncedTo"];
BSONObjBuilder t(bb.subobjStart("syncedTo"));
t.appendDate("time", e.timestampTime());
t.append("inc", e.timestampInc());
t.done();
}
if (level > 1) {
invariant(!opCtx->lockState()->isLocked());
// note: there is no so-style timeout on this connection; perhaps we should have
// one.
ScopedDbConnection conn(s["host"].valuestr());
DBClientConnection* cliConn = dynamic_cast<DBClientConnection*>(&conn.conn());
if (cliConn && replAuthenticate(cliConn)) {
BSONObj first = conn->findOne((string) "local.oplog.$" + sourcename,
Query().sort(BSON("$natural" << 1)));
BSONObj last = conn->findOne((string) "local.oplog.$" + sourcename,
Query().sort(BSON("$natural" << -1)));
bb.appendDate("masterFirst", first["ts"].timestampTime());
bb.appendDate("masterLast", last["ts"].timestampTime());
const auto lag = (last["ts"].timestampTime() - s["syncedTo"].timestampTime());
bb.append("lagSeconds", durationCount<Milliseconds>(lag) / 1000.0);
}
conn.done();
}
sources.append(BSONObjBuilder::numStr(n++), bb.obj());
}
sources.done();
replCoord->appendSlaveInfoData(&result);
}
}
void appendReplicationInfo(OperationContext* txn, BSONObjBuilder& result, int level) {
ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
if (replCoord->getSettings().usingReplSets()) {
if (replCoord->getReplicationMode() != ReplicationCoordinator::modeReplSet
|| replCoord->getCurrentMemberState().shunned()) {
result.append("ismaster", false);
result.append("secondary", false);
result.append("info", ReplSet::startupStatusMsg.get());
result.append( "isreplicaset" , true );
}
else {
theReplSet->fillIsMaster(result);
}
return;
}
if ( replAllDead ) {
result.append("ismaster", 0);
string s = string("dead: ") + replAllDead;
result.append("info", s);
}
else {
result.appendBool("ismaster",
getGlobalReplicationCoordinator()->isMasterForReportingPurposes());
}
if (level && replCoord->getSettings().usingReplSets()) {
result.append( "info" , "is replica set" );
}
else if ( level ) {
BSONObjBuilder sources( result.subarrayStart( "sources" ) );
int n = 0;
list<BSONObj> src;
{
const char* localSources = "local.sources";
Client::ReadContext ctx(txn, localSources);
auto_ptr<PlanExecutor> exec(
InternalPlanner::collectionScan(txn,
localSources,
ctx.ctx().db()->getCollection(txn,
localSources)));
BSONObj obj;
Runner::RunnerState state;
while (Runner::RUNNER_ADVANCED == (state = exec->getNext(&obj, NULL))) {
src.push_back(obj);
}
}
for( list<BSONObj>::const_iterator i = src.begin(); i != src.end(); i++ ) {
BSONObj s = *i;
BSONObjBuilder bb;
bb.append( s["host"] );
string sourcename = s["source"].valuestr();
if ( sourcename != "main" )
bb.append( s["source"] );
{
BSONElement e = s["syncedTo"];
BSONObjBuilder t( bb.subobjStart( "syncedTo" ) );
t.appendDate( "time" , e.timestampTime() );
t.append( "inc" , e.timestampInc() );
t.done();
}
if ( level > 1 ) {
wassert(txn->lockState()->threadState() == 0);
// note: there is no so-style timeout on this connection; perhaps we should have one.
ScopedDbConnection conn(s["host"].valuestr());
DBClientConnection *cliConn = dynamic_cast< DBClientConnection* >( &conn.conn() );
if ( cliConn && replAuthenticate(cliConn) ) {
BSONObj first = conn->findOne( (string)"local.oplog.$" + sourcename,
Query().sort( BSON( "$natural" << 1 ) ) );
BSONObj last = conn->findOne( (string)"local.oplog.$" + sourcename,
Query().sort( BSON( "$natural" << -1 ) ) );
bb.appendDate( "masterFirst" , first["ts"].timestampTime() );
bb.appendDate( "masterLast" , last["ts"].timestampTime() );
double lag = (double) (last["ts"].timestampTime() - s["syncedTo"].timestampTime());
bb.append( "lagSeconds" , lag / 1000 );
}
conn.done();
}
sources.append( BSONObjBuilder::numStr( n++ ) , bb.obj() );
}
sources.done();
}
}