void ReplCoordTest::simulateSuccessfulV1Election() {
OperationContextReplMock txn;
ReplicationCoordinatorImpl* replCoord = getReplCoord();
NetworkInterfaceMock* net = getNet();
auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest();
ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen);
log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)";
ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest();
ASSERT(replCoord->getMemberState().secondary()) << replCoord->getMemberState().toString();
while (!replCoord->getMemberState().primary()) {
log() << "Waiting on network in state " << replCoord->getMemberState();
getNet()->enterNetwork();
if (net->now() < electionTimeoutWhen) {
net->runUntil(electionTimeoutWhen);
}
const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest();
const RemoteCommandRequest& request = noi->getRequest();
log() << request.target.toString() << " processing " << request.cmdObj;
ReplSetHeartbeatArgsV1 hbArgs;
Status status = hbArgs.initialize(request.cmdObj);
if (hbArgs.initialize(request.cmdObj).isOK()) {
ReplSetHeartbeatResponse hbResp;
hbResp.setSetName(rsConfig.getReplSetName());
hbResp.setState(MemberState::RS_SECONDARY);
hbResp.setConfigVersion(rsConfig.getConfigVersion());
net->scheduleResponse(noi, net->now(), makeResponseStatus(hbResp.toBSON(true)));
} else if (request.cmdObj.firstElement().fieldNameStringData() == "replSetRequestVotes") {
net->scheduleResponse(
noi,
net->now(),
makeResponseStatus(BSON("ok" << 1 << "reason"
<< ""
<< "term" << request.cmdObj["term"].Long()
<< "voteGranted" << true)));
} else if (request.cmdObj.firstElement().fieldNameStringData() ==
"replSetDeclareElectionWinner") {
net->scheduleResponse(
noi,
net->now(),
makeResponseStatus(BSON("ok" << 1 << "term" << request.cmdObj["term"].Long())));
} else {
error() << "Black holing unexpected request to " << request.target << ": "
<< request.cmdObj;
net->blackHole(noi);
}
net->runReadyNetworkOperations();
getNet()->exitNetwork();
}
ASSERT(replCoord->isWaitingForApplierToDrain());
ASSERT(replCoord->getMemberState().primary()) << replCoord->getMemberState().toString();
IsMasterResponse imResponse;
replCoord->fillIsMasterForReplSet(&imResponse);
ASSERT_FALSE(imResponse.isMaster()) << imResponse.toBSON().toString();
ASSERT_TRUE(imResponse.isSecondary()) << imResponse.toBSON().toString();
replCoord->signalDrainComplete(&txn);
replCoord->fillIsMasterForReplSet(&imResponse);
ASSERT_TRUE(imResponse.isMaster()) << imResponse.toBSON().toString();
ASSERT_FALSE(imResponse.isSecondary()) << imResponse.toBSON().toString();
ASSERT(replCoord->getMemberState().primary()) << replCoord->getMemberState().toString();
// Consume the notification of election win.
for (int i = 0; i < rsConfig.getNumMembers() - 1; i++) {
replyToReceivedHeartbeatV1();
}
}
void QuorumChecker::_tabulateHeartbeatResponse(const RemoteCommandRequest& request,
const ResponseStatus& response) {
++_numResponses;
if (!response.isOK()) {
warning() << "Failed to complete heartbeat request to " << request.target << "; "
<< response.getStatus();
_badResponses.push_back(std::make_pair(request.target, response.getStatus()));
return;
}
BSONObj resBSON = response.getValue().data;
ReplSetHeartbeatResponse hbResp;
Status hbStatus = hbResp.initialize(resBSON, 0);
if (hbStatus.code() == ErrorCodes::InconsistentReplicaSetNames) {
std::string message = str::stream() << "Our set name did not match that of "
<< request.target.toString();
_vetoStatus = Status(ErrorCodes::NewReplicaSetConfigurationIncompatible, message);
warning() << message;
return;
}
if (!hbStatus.isOK() && hbStatus != ErrorCodes::InvalidReplicaSetConfig) {
warning() << "Got error (" << hbStatus << ") response on heartbeat request to "
<< request.target << "; " << hbResp;
_badResponses.push_back(std::make_pair(request.target, hbStatus));
return;
}
if (!hbResp.getReplicaSetName().empty()) {
if (hbResp.getConfigVersion() >= _rsConfig->getConfigVersion()) {
std::string message = str::stream()
<< "Our config version of " << _rsConfig->getConfigVersion()
<< " is no larger than the version on " << request.target.toString()
<< ", which is " << hbResp.getConfigVersion();
_vetoStatus = Status(ErrorCodes::NewReplicaSetConfigurationIncompatible, message);
warning() << message;
return;
}
}
if (_rsConfig->hasReplicaSetId()) {
StatusWith<rpc::ReplSetMetadata> replMetadata =
rpc::ReplSetMetadata::readFromMetadata(response.getValue().metadata);
if (replMetadata.isOK() && replMetadata.getValue().getReplicaSetId().isSet() &&
_rsConfig->getReplicaSetId() != replMetadata.getValue().getReplicaSetId()) {
std::string message = str::stream()
<< "Our replica set ID of " << _rsConfig->getReplicaSetId()
<< " did not match that of " << request.target.toString() << ", which is "
<< replMetadata.getValue().getReplicaSetId();
_vetoStatus = Status(ErrorCodes::NewReplicaSetConfigurationIncompatible, message);
warning() << message;
}
}
const bool isInitialConfig = _rsConfig->getConfigVersion() == 1;
if (isInitialConfig && hbResp.hasData()) {
std::string message = str::stream() << "'" << request.target.toString()
<< "' has data already, cannot initiate set.";
_vetoStatus = Status(ErrorCodes::CannotInitializeNodeWithData, message);
warning() << message;
return;
}
for (int i = 0; i < _rsConfig->getNumMembers(); ++i) {
const MemberConfig& memberConfig = _rsConfig->getMemberAt(i);
if (memberConfig.getHostAndPort() != request.target) {
continue;
}
if (memberConfig.isElectable()) {
++_numElectable;
}
if (memberConfig.isVoter()) {
_voters.push_back(request.target);
}
return;
}
invariant(false);
}
void ReplicationCoordinatorImpl::_handleHeartbeatResponse(
const ReplicationExecutor::RemoteCommandCallbackData& cbData, int targetIndex) {
// remove handle from queued heartbeats
_untrackHeartbeatHandle(cbData.myHandle);
// Parse and validate the response. At the end of this step, if responseStatus is OK then
// hbResponse is valid.
Status responseStatus = cbData.response.getStatus();
if (responseStatus == ErrorCodes::CallbackCanceled) {
return;
}
const HostAndPort& target = cbData.request.target;
ReplSetHeartbeatResponse hbResponse;
BSONObj resp;
if (responseStatus.isOK()) {
resp = cbData.response.getValue().data;
responseStatus = hbResponse.initialize(resp);
}
const bool isUnauthorized = (responseStatus.code() == ErrorCodes::Unauthorized) ||
(responseStatus.code() == ErrorCodes::AuthenticationFailed);
const Date_t now = _replExecutor.now();
const Timestamp lastApplied = getMyLastOptime(); // Locks and unlocks _mutex.
Milliseconds networkTime(0);
StatusWith<ReplSetHeartbeatResponse> hbStatusResponse(hbResponse);
if (responseStatus.isOK()) {
networkTime = cbData.response.getValue().elapsedMillis;
}
else {
log() << "Error in heartbeat request to " << target << "; " << responseStatus;
if (!resp.isEmpty()) {
LOG(3) << "heartbeat response: " << resp;
}
if (isUnauthorized) {
networkTime = cbData.response.getValue().elapsedMillis;
}
hbStatusResponse = StatusWith<ReplSetHeartbeatResponse>(responseStatus);
}
HeartbeatResponseAction action =
_topCoord->processHeartbeatResponse(
now,
networkTime,
target,
hbStatusResponse,
lastApplied);
if (action.getAction() == HeartbeatResponseAction::NoAction &&
hbStatusResponse.isOK() &&
hbStatusResponse.getValue().hasOpTime() &&
targetIndex >= 0 &&
hbStatusResponse.getValue().hasState() &&
hbStatusResponse.getValue().getState() != MemberState::RS_PRIMARY) {
boost::unique_lock<boost::mutex> lk(_mutex);
if (hbStatusResponse.getValue().getVersion() == _rsConfig.getConfigVersion()) {
_updateOpTimeFromHeartbeat_inlock(targetIndex,
hbStatusResponse.getValue().getOpTime());
// TODO: Enable with Data Replicator
//lk.unlock();
//_dr.slavesHaveProgressed();
}
}
_signalStepDownWaiters();
_scheduleHeartbeatToTarget(
target,
targetIndex,
std::max(now, action.getNextHeartbeatStartDate()));
_handleHeartbeatResponseAction(action, hbStatusResponse);
}