Status MigrationManager::_processRemoteCommandResponse(
const RemoteCommandResponse& remoteCommandResponse,
ScopedMigrationRequest* scopedMigrationRequest) {
stdx::lock_guard<stdx::mutex> lock(_mutex);
Status commandStatus(ErrorCodes::InternalError, "Uninitialized value.");
// Check for local errors sending the remote command caused by stepdown.
if (isErrorDueToConfigStepdown(remoteCommandResponse.status,
_state != State::kEnabled && _state != State::kRecovering)) {
scopedMigrationRequest->keepDocumentOnDestruct();
return {ErrorCodes::BalancerInterrupted,
stream() << "Migration interrupted because the balancer is stopping."
<< " Command status: "
<< remoteCommandResponse.status.toString()};
}
if (!remoteCommandResponse.isOK()) {
commandStatus = remoteCommandResponse.status;
} else {
// TODO: delete in 3.8
commandStatus = extractMigrationStatusFromCommandResponse(remoteCommandResponse.data);
}
if (!Shard::shouldErrorBePropagated(commandStatus.code())) {
commandStatus = {ErrorCodes::OperationFailed,
stream() << "moveChunk command failed on source shard."
<< causedBy(commandStatus)};
}
// Any failure to remove the migration document should be because the config server is
// stepping/shutting down. In this case we must fail the moveChunk command with a retryable
// error so that the caller does not move on to other distlock requiring operations that could
// fail when the balancer recovers and takes distlocks for migration recovery.
Status status = scopedMigrationRequest->tryToRemoveMigration();
if (!status.isOK()) {
commandStatus = {
ErrorCodes::BalancerInterrupted,
stream() << "Migration interrupted because the balancer is stopping"
<< " and failed to remove the config.migrations document."
<< " Command status: "
<< (commandStatus.isOK() ? status.toString() : commandStatus.toString())};
}
return commandStatus;
}
void VoteRequester::Algorithm::processResponse(const RemoteCommandRequest& request,
const RemoteCommandResponse& response) {
auto logLine = log();
logLine << "VoteRequester(term " << _term << (_dryRun ? " dry run" : "") << ") ";
_responsesProcessed++;
if (!response.isOK()) { // failed response
logLine << "failed to receive response from " << request.target << ": " << response.status;
return;
}
_responders.insert(request.target);
// If the primary's vote is a yes, we will set _primaryVote to be Yes.
if (_primaryHost == request.target) {
_primaryVote = PrimaryVote::No;
}
ReplSetRequestVotesResponse voteResponse;
auto status = getStatusFromCommandResult(response.data);
if (status.isOK()) {
status = voteResponse.initialize(response.data);
}
if (!status.isOK()) {
logLine << "received an invalid response from " << request.target << ": " << status;
logLine << "; response message: " << response.data;
return;
}
if (voteResponse.getVoteGranted()) {
logLine << "received a yes vote from " << request.target;
if (_primaryHost == request.target) {
_primaryVote = PrimaryVote::Yes;
}
_votes++;
} else {
logLine << "received a no vote from " << request.target << " with reason \""
<< voteResponse.getReason() << '"';
}
if (voteResponse.getTerm() > _term) {
_staleTerm = true;
}
logLine << "; response message: " << response.data;
}
Shard::HostWithResponse ShardRemote::_runCommand(OperationContext* txn,
const ReadPreferenceSetting& readPref,
const string& dbName,
Milliseconds maxTimeMSOverride,
const BSONObj& cmdObj) {
ReadPreferenceSetting readPrefWithMinOpTime(readPref);
if (getId() == "config") {
readPrefWithMinOpTime.minOpTime = grid.configOpTime();
}
const auto host = _targeter->findHost(txn, readPrefWithMinOpTime);
if (!host.isOK()) {
return Shard::HostWithResponse(boost::none, host.getStatus());
}
const Milliseconds requestTimeout =
std::min(txn->getRemainingMaxTimeMillis(), maxTimeMSOverride);
const RemoteCommandRequest request(
host.getValue(),
dbName,
appendMaxTimeToCmdObj(maxTimeMSOverride, cmdObj),
_appendMetadataForCommand(txn, readPrefWithMinOpTime),
txn,
requestTimeout < Milliseconds::max() ? requestTimeout : RemoteCommandRequest::kNoTimeout);
RemoteCommandResponse swResponse =
Status(ErrorCodes::InternalError, "Internal error running command");
TaskExecutor* executor = Grid::get(txn)->getExecutorPool()->getFixedExecutor();
auto callStatus = executor->scheduleRemoteCommand(
request,
[&swResponse](const RemoteCommandCallbackArgs& args) { swResponse = args.response; });
if (!callStatus.isOK()) {
return Shard::HostWithResponse(host.getValue(), callStatus.getStatus());
}
// Block until the command is carried out
executor->wait(callStatus.getValue());
updateReplSetMonitor(host.getValue(), swResponse.status);
if (!swResponse.isOK()) {
if (swResponse.status.compareCode(ErrorCodes::ExceededTimeLimit)) {
LOG(0) << "Operation timed out with status " << redact(swResponse.status);
}
return Shard::HostWithResponse(host.getValue(), swResponse.status);
}
BSONObj responseObj = swResponse.data.getOwned();
BSONObj responseMetadata = swResponse.metadata.getOwned();
Status commandStatus = getStatusFromCommandResult(responseObj);
Status writeConcernStatus = getWriteConcernStatusFromCommandResult(responseObj);
// Tell the replica set monitor of any errors
updateReplSetMonitor(host.getValue(), commandStatus);
updateReplSetMonitor(host.getValue(), writeConcernStatus);
return Shard::HostWithResponse(host.getValue(),
CommandResponse(std::move(responseObj),
std::move(responseMetadata),
std::move(commandStatus),
std::move(writeConcernStatus)));
}
