void ReplCoordTest::simulateSuccessfulElection() {
OperationContextReplMock txn;
ReplicationCoordinatorImpl* replCoord = getReplCoord();
NetworkInterfaceMock* net = getNet();
ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest();
ASSERT(replCoord->getMemberState().secondary()) << replCoord->getMemberState().toString();
while (!replCoord->getMemberState().primary()) {
log() << "Waiting on network in state " << replCoord->getMemberState();
getNet()->enterNetwork();
const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest();
const RemoteCommandRequest& request = noi->getRequest();
log() << request.target.toString() << " processing " << request.cmdObj;
ReplSetHeartbeatArgs hbArgs;
if (hbArgs.initialize(request.cmdObj).isOK()) {
ReplSetHeartbeatResponse hbResp;
hbResp.setSetName(rsConfig.getReplSetName());
hbResp.setState(MemberState::RS_SECONDARY);
hbResp.setConfigVersion(rsConfig.getConfigVersion());
BSONObjBuilder respObj;
respObj << "ok" << 1;
hbResp.addToBSON(&respObj, false);
net->scheduleResponse(noi, net->now(), makeResponseStatus(respObj.obj()));
} else if (request.cmdObj.firstElement().fieldNameStringData() == "replSetFresh") {
net->scheduleResponse(
noi,
net->now(),
makeResponseStatus(BSON("ok" << 1 << "fresher" << false << "opTime" << Date_t()
<< "veto" << false)));
} else if (request.cmdObj.firstElement().fieldNameStringData() == "replSetElect") {
net->scheduleResponse(noi,
net->now(),
makeResponseStatus(BSON("ok" << 1 << "vote" << 1 << "round"
<< request.cmdObj["round"].OID())));
} else {
error() << "Black holing unexpected request to " << request.target << ": "
<< request.cmdObj;
net->blackHole(noi);
}
net->runReadyNetworkOperations();
getNet()->exitNetwork();
}
ASSERT(replCoord->isWaitingForApplierToDrain());
ASSERT(replCoord->getMemberState().primary()) << replCoord->getMemberState().toString();
IsMasterResponse imResponse;
replCoord->fillIsMasterForReplSet(&imResponse);
ASSERT_FALSE(imResponse.isMaster()) << imResponse.toBSON().toString();
ASSERT_TRUE(imResponse.isSecondary()) << imResponse.toBSON().toString();
replCoord->signalDrainComplete(&txn);
replCoord->fillIsMasterForReplSet(&imResponse);
ASSERT_TRUE(imResponse.isMaster()) << imResponse.toBSON().toString();
ASSERT_FALSE(imResponse.isSecondary()) << imResponse.toBSON().toString();
ASSERT(replCoord->getMemberState().primary()) << replCoord->getMemberState().toString();
// Consume the notification of election win.
for (int i = 0; i < rsConfig.getNumMembers() - 1; i++) {
replyToReceivedHeartbeat();
}
}
void ReplCoordTest::simulateEnoughHeartbeatsForAllNodesUp() {
ReplicationCoordinatorImpl* replCoord = getReplCoord();
ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest();
NetworkInterfaceMock* net = getNet();
net->enterNetwork();
for (int i = 0; i < rsConfig.getNumMembers() - 1; ++i) {
const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest();
const RemoteCommandRequest& request = noi->getRequest();
log() << request.target.toString() << " processing " << request.cmdObj;
ReplSetHeartbeatArgsV1 hbArgs;
ReplSetHeartbeatArgs hbArgsPV0;
if (hbArgs.initialize(request.cmdObj).isOK() ||
hbArgsPV0.initialize(request.cmdObj).isOK()) {
ReplSetHeartbeatResponse hbResp;
hbResp.setSetName(rsConfig.getReplSetName());
hbResp.setState(MemberState::RS_SECONDARY);
hbResp.setConfigVersion(rsConfig.getConfigVersion());
hbResp.setAppliedOpTime(OpTime(Timestamp(100, 2), 0));
BSONObjBuilder respObj;
net->scheduleResponse(noi, net->now(), makeResponseStatus(hbResp.toBSON(true)));
} else {
error() << "Black holing unexpected request to " << request.target << ": "
<< request.cmdObj;
net->blackHole(noi);
}
net->runReadyNetworkOperations();
}
net->exitNetwork();
}
void ReplCoordTest::simulateStepDownOnIsolation() {
ReplicationCoordinatorImpl* replCoord = getReplCoord();
NetworkInterfaceMock* net = getNet();
ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest();
ASSERT(replCoord->getMemberState().primary()) << replCoord->getMemberState().toString();
while (replCoord->getMemberState().primary()) {
log() << "Waiting on network in state " << replCoord->getMemberState();
getNet()->enterNetwork();
net->runUntil(net->now() + Seconds(10));
const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest();
const RemoteCommandRequest& request = noi->getRequest();
log() << request.target.toString() << " processing " << request.cmdObj;
ReplSetHeartbeatArgs hbArgs;
if (hbArgs.initialize(request.cmdObj).isOK()) {
net->scheduleResponse(
noi, net->now(), ResponseStatus(ErrorCodes::NetworkTimeout, "Nobody's home"));
} else {
error() << "Black holing unexpected request to " << request.target << ": "
<< request.cmdObj;
net->blackHole(noi);
}
net->runReadyNetworkOperations();
getNet()->exitNetwork();
}
}
void ReplCoordTest::simulateSuccessfulV1Election() {
OperationContextReplMock txn;
ReplicationCoordinatorImpl* replCoord = getReplCoord();
NetworkInterfaceMock* net = getNet();
ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest();
ASSERT(replCoord->getMemberState().secondary()) << replCoord->getMemberState().toString();
while (!replCoord->getMemberState().primary()) {
log() << "Waiting on network in state " << replCoord->getMemberState();
getNet()->enterNetwork();
const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest();
const RemoteCommandRequest& request = noi->getRequest();
log() << request.target.toString() << " processing " << request.cmdObj;
ReplSetHeartbeatArgsV1 hbArgs;
Status status = hbArgs.initialize(request.cmdObj);
if (hbArgs.initialize(request.cmdObj).isOK()) {
ReplSetHeartbeatResponse hbResp;
hbResp.setSetName(rsConfig.getReplSetName());
hbResp.setState(MemberState::RS_SECONDARY);
hbResp.setConfigVersion(rsConfig.getConfigVersion());
net->scheduleResponse(noi, net->now(), makeResponseStatus(hbResp.toBSON(true)));
} else if (request.cmdObj.firstElement().fieldNameStringData() == "replSetRequestVotes") {
net->scheduleResponse(
noi,
net->now(),
makeResponseStatus(BSON("ok" << 1 << "reason"
<< ""
<< "term" << request.cmdObj["term"].Long()
<< "voteGranted" << true)));
} else if (request.cmdObj.firstElement().fieldNameStringData() ==
"replSetDeclareElectionWinner") {
net->scheduleResponse(
noi,
net->now(),
makeResponseStatus(BSON("ok" << 1 << "term" << request.cmdObj["term"].Long())));
} else {
error() << "Black holing unexpected request to " << request.target << ": "
<< request.cmdObj;
net->blackHole(noi);
}
net->runReadyNetworkOperations();
getNet()->exitNetwork();
}
ASSERT(replCoord->isWaitingForApplierToDrain());
ASSERT(replCoord->getMemberState().primary()) << replCoord->getMemberState().toString();
IsMasterResponse imResponse;
replCoord->fillIsMasterForReplSet(&imResponse);
ASSERT_FALSE(imResponse.isMaster()) << imResponse.toBSON().toString();
ASSERT_TRUE(imResponse.isSecondary()) << imResponse.toBSON().toString();
replCoord->signalDrainComplete(&txn);
replCoord->fillIsMasterForReplSet(&imResponse);
ASSERT_TRUE(imResponse.isMaster()) << imResponse.toBSON().toString();
ASSERT_FALSE(imResponse.isSecondary()) << imResponse.toBSON().toString();
ASSERT(replCoord->getMemberState().primary()) << replCoord->getMemberState().toString();
}
void ReplCoordTest::simulateSuccessfulDryRun(
stdx::function<void(const RemoteCommandRequest& request)> onDryRunRequest) {
ReplicationCoordinatorImpl* replCoord = getReplCoord();
ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest();
NetworkInterfaceMock* net = getNet();
auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest();
ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen);
log() << "Election timeout scheduled at " << electionTimeoutWhen << " (simulator time)";
int voteRequests = 0;
int votesExpected = rsConfig.getNumMembers() / 2;
log() << "Simulating dry run responses - expecting " << votesExpected
<< " replSetRequestVotes requests";
net->enterNetwork();
while (voteRequests < votesExpected) {
if (net->now() < electionTimeoutWhen) {
net->runUntil(electionTimeoutWhen);
}
const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest();
const RemoteCommandRequest& request = noi->getRequest();
log() << request.target.toString() << " processing " << request.cmdObj;
if (request.cmdObj.firstElement().fieldNameStringData() == "replSetRequestVotes") {
ASSERT_TRUE(request.cmdObj.getBoolField("dryRun"));
onDryRunRequest(request);
net->scheduleResponse(
noi,
net->now(),
makeResponseStatus(BSON("ok" << 1 << "reason"
<< ""
<< "term" << request.cmdObj["term"].Long()
<< "voteGranted" << true)));
voteRequests++;
} else {
error() << "Black holing unexpected request to " << request.target << ": "
<< request.cmdObj;
net->blackHole(noi);
}
net->runReadyNetworkOperations();
}
net->exitNetwork();
log() << "Simulating dry run responses - scheduled " << voteRequests
<< " replSetRequestVotes responses";
getReplCoord()->waitForElectionDryRunFinish_forTest();
log() << "Simulating dry run responses - dry run completed";
}
void ReplCoordTest::replyToReceivedHeartbeatV1() {
NetworkInterfaceMock* net = getNet();
net->enterNetwork();
const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest();
const RemoteCommandRequest& request = noi->getRequest();
const ReplicaSetConfig rsConfig = getReplCoord()->getReplicaSetConfig_forTest();
repl::ReplSetHeartbeatArgsV1 hbArgs;
ASSERT_OK(hbArgs.initialize(request.cmdObj));
repl::ReplSetHeartbeatResponse hbResp;
hbResp.setSetName(rsConfig.getReplSetName());
hbResp.setState(MemberState::RS_SECONDARY);
hbResp.setConfigVersion(rsConfig.getConfigVersion());
BSONObjBuilder respObj;
respObj << "ok" << 1;
hbResp.addToBSON(&respObj, false);
net->scheduleResponse(noi, net->now(), makeResponseStatus(respObj.obj()));
net->runReadyNetworkOperations();
getNet()->exitNetwork();
}