// Initial Sync
Status DatabasesCloner::start() {
_active = true;
if (!_status.isOK() && _status.code() != ErrorCodes::NotYetInitialized) {
return _status;
}
_status = Status::OK();
log() << "starting cloning of all databases";
// Schedule listDatabase command which will kick off the database cloner per result db.
Request listDBsReq(_source,
"admin",
BSON("listDatabases" << true),
rpc::ServerSelectionMetadata(true, boost::none).toBSON());
CBHStatus s = _exec->scheduleRemoteCommand(
listDBsReq,
stdx::bind(&DatabasesCloner::_onListDatabaseFinish, this, stdx::placeholders::_1));
if (!s.isOK()) {
_setStatus(s);
_failed();
}
_doNextActions();
return _status;
}
void ReplicationCoordinatorImpl::_requestRemotePrimaryStepdown(const HostAndPort& target) {
RemoteCommandRequest request(target, "admin", BSON("replSetStepDown" << 1));
log() << "Requesting " << target << " step down from primary";
CBHStatus cbh = _replExecutor.scheduleRemoteCommand(request, remoteStepdownCallback);
if (cbh.getStatus() != ErrorCodes::ShutdownInProgress) {
fassert(18808, cbh.getStatus());
}
}
Status ReplicationCoordinatorImpl::processReplSetFreeze(int secs, BSONObjBuilder* resultObj) {
Status result(ErrorCodes::InternalError, "didn't set status in prepareFreezeResponse");
CBHStatus cbh = _replExecutor->scheduleWork(
stdx::bind(&TopologyCoordinator::prepareFreezeResponse,
_topCoord.get(),
stdx::placeholders::_1,
Date_t(curTimeMillis64()),
secs,
resultObj,
&result));
if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) {
return Status(ErrorCodes::ShutdownInProgress, "replication shutdown in progress");
}
fassert(18641, cbh.getStatus());
_replExecutor->wait(cbh.getValue());
return result;
}
Status ReplicationCoordinatorImpl::processReplSetGetStatus(BSONObjBuilder* response) {
Status result(ErrorCodes::InternalError, "didn't set status in prepareStatusResponse");
CBHStatus cbh = _replExecutor->scheduleWork(
stdx::bind(&TopologyCoordinator::prepareStatusResponse,
_topCoord.get(),
stdx::placeholders::_1,
Date_t(curTimeMillis64()),
time(0) - serverGlobalParams.started,
_getLastOpApplied(),
response,
&result));
if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) {
return Status(ErrorCodes::ShutdownInProgress, "replication shutdown in progress");
}
fassert(18640, cbh.getStatus());
_replExecutor->wait(cbh.getValue());
return result;
}
void ReplicationCoordinatorImpl::_startHeartbeats() {
ReplicaSetConfig::MemberIterator it = _rsConfig.membersBegin();
ReplicaSetConfig::MemberIterator end = _rsConfig.membersBegin();
for(;it != end; it++) {
HostAndPort host = it->getHostAndPort();
CBHStatus status = _replExecutor->scheduleWork(
stdx::bind(
&ReplicationCoordinatorImpl::doMemberHeartbeat,
this,
stdx::placeholders::_1,
host));
if (!status.isOK()) {
log() << "replset: cannot start heartbeats for "
<< host << " due to scheduling error -- "<< status;
continue;
}
_trackHeartbeatHandle(status.getValue());
}
}
Status ReplicationCoordinatorImpl::processHeartbeat(const ReplSetHeartbeatArgs& args,
ReplSetHeartbeatResponse* response) {
Status result(ErrorCodes::InternalError, "didn't set status in prepareHeartbeatResponse");
CBHStatus cbh = _replExecutor->scheduleWork(
stdx::bind(&TopologyCoordinator::prepareHeartbeatResponse,
_topCoord.get(),
stdx::placeholders::_1,
Date_t(curTimeMillis64()),
args,
_settings.ourSetName(),
_getLastOpApplied(),
response,
&result));
if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) {
return Status(ErrorCodes::ShutdownInProgress, "replication shutdown in progress");
}
fassert(18508, cbh.getStatus());
_replExecutor->wait(cbh.getValue());
return result;
}
void InitialSyncState::setStatus(const CBHStatus& s) {
setStatus(s.getStatus());
}
void ReplicationCoordinatorImpl::doMemberHeartbeat(ReplicationExecutor::CallbackData cbData,
const HostAndPort& hap) {
if (cbData.status == ErrorCodes::CallbackCanceled) {
return;
}
// Are we blind, or do we have a failpoint setup to ignore this member?
bool dontHeartbeatMember = false; // TODO: replSetBlind should be here as the default
MONGO_FAIL_POINT_BLOCK(rsHeartbeatRequestNoopByMember, member) {
const StringData& stopMember = member.getData()["member"].valueStringData();
HostAndPort ignoreHAP;
Status status = ignoreHAP.initialize(stopMember);
// Ignore
if (status.isOK()) {
if (hap == ignoreHAP) {
dontHeartbeatMember = true;
}
} else {
log() << "replset: Bad member for rsHeartbeatRequestNoopByMember failpoint "
<< member.getData() << ". 'member' failed to parse into HostAndPort -- "
<< status;
}
}
if (dontHeartbeatMember) {
// Don't issue real heartbeats, just call start again after the timeout.
ReplicationExecutor::CallbackFn restartCB = stdx::bind(
&ReplicationCoordinatorImpl::doMemberHeartbeat,
this,
stdx::placeholders::_1,
hap);
CBHStatus status = _replExecutor->scheduleWorkAt(
Date_t(curTimeMillis64() + heartbeatFrequencyMillis),
restartCB);
if (!status.isOK()) {
log() << "replset: aborting heartbeats for " << hap << " due to scheduling error"
<< " -- "<< status;
return;
}
_trackHeartbeatHandle(status.getValue());
return;
}
// Compose heartbeat command message
BSONObj hbCommandBSON;
{
// take lock to build request
boost::lock_guard<boost::mutex> lock(_mutex);
BSONObjBuilder cmdBuilder;
const MemberConfig me = _rsConfig.getMemberAt(_thisMembersConfigIndex);
cmdBuilder.append("replSetHeartbeat", _rsConfig.getReplSetName());
cmdBuilder.append("v", _rsConfig.getConfigVersion());
cmdBuilder.append("pv", 1);
cmdBuilder.append("checkEmpty", false);
cmdBuilder.append("from", me.getHostAndPort().toString());
cmdBuilder.append("fromId", me.getId());
hbCommandBSON = cmdBuilder.done();
}
const ReplicationExecutor::RemoteCommandRequest request(hap, "admin", hbCommandBSON);
ReplicationExecutor::RemoteCommandCallbackFn callback = stdx::bind(
&ReplicationCoordinatorImpl::_handleHeartbeatResponse,
this,
stdx::placeholders::_1,
hap,
curTimeMillis64(),
heartbeatRetries);
CBHStatus status = _replExecutor->scheduleRemoteCommand(request, callback);
if (!status.isOK()) {
log() << "replset: aborting heartbeats for " << hap << " due to scheduling error"
<< status;
return;
}
_trackHeartbeatHandle(status.getValue());
}