void CollectionCloner::_verifyCollectionWasDropped(
const stdx::unique_lock<stdx::mutex>& lk,
Status batchStatus,
std::shared_ptr<OnCompletionGuard> onCompletionGuard,
OperationContext* opCtx) {
// If we already have a _verifyCollectionDroppedScheduler, just return; the existing
// scheduler will take care of cleaning up.
if (_verifyCollectionDroppedScheduler) {
return;
}
BSONObjBuilder cmdObj;
_options.uuid->appendToBuilder(&cmdObj, "find");
cmdObj.append("batchSize", 0);
_verifyCollectionDroppedScheduler = stdx::make_unique<RemoteCommandRetryScheduler>(
_executor,
RemoteCommandRequest(_source,
_sourceNss.db().toString(),
cmdObj.obj(),
ReadPreferenceSetting::secondaryPreferredMetadata(),
opCtx,
RemoteCommandRequest::kNoTimeout),
[this, batchStatus, onCompletionGuard](const RemoteCommandCallbackArgs& args) {
// If the attempt to determine if the collection was dropped fails for any reason other
// than NamespaceNotFound, return the original error code.
//
// Otherwise, if the collection was dropped, either the error will be NamespaceNotFound,
// or it will be a drop-pending collection and the find will succeed and give us a
// collection with a drop-pending name.
UniqueLock lk(_mutex);
Status finalStatus(batchStatus);
if (args.response.isOK()) {
auto response = CursorResponse::parseFromBSON(args.response.data);
if (response.getStatus().code() == ErrorCodes::NamespaceNotFound ||
(response.isOK() && response.getValue().getNSS().isDropPendingNamespace())) {
log() << "CollectionCloner ns: '" << _sourceNss.ns() << "' uuid: UUID(\""
<< *_options.uuid << "\") stopped because collection was dropped.";
finalStatus = Status::OK();
} else if (!response.isOK()) {
log() << "CollectionCloner received an unexpected error when verifying drop of "
"ns: '"
<< _sourceNss.ns() << "' uuid: UUID(\"" << *_options.uuid
<< "\"), status " << response.getStatus();
}
} else {
log() << "CollectionCloner is unable to verify drop of ns: '" << _sourceNss.ns()
<< "' uuid: UUID(\"" << *_options.uuid << "\"), status "
<< args.response.status;
}
onCompletionGuard->setResultAndCancelRemainingWork_inlock(lk, finalStatus);
},
RemoteCommandRetryScheduler::makeNoRetryPolicy());
auto status = _verifyCollectionDroppedScheduler->startup();
if (!status.isOK()) {
log() << "CollectionCloner is unable to start verification of ns: '" << _sourceNss.ns()
<< "' uuid: UUID(\"" << *_options.uuid << "\"), status " << status;
// If we can't run the command, assume this wasn't a drop and just use the original error.
onCompletionGuard->setResultAndCancelRemainingWork_inlock(lk, batchStatus);
}
}
Status Reporter::_schedule_inlock() {
if (!_status.isOK()) {
return _status;
}
if (_active) {
_willRunAgain = true;
return _status;
}
LOG(2) << "Reporter scheduling report to : " << _target;
_willRunAgain = false;
BSONObjBuilder cmd;
_updatePositionSource->prepareReplSetUpdatePositionCommand(&cmd);
StatusWith<ReplicationExecutor::CallbackHandle> scheduleResult =
_executor->scheduleRemoteCommand(
RemoteCommandRequest(_target, "admin", cmd.obj()),
stdx::bind(&Reporter::_callback, this, stdx::placeholders::_1));
if (!scheduleResult.isOK()) {
_status = scheduleResult.getStatus();
LOG(2) << "Reporter failed to schedule with status: " << _status;
return _status;
}
_active = true;
_remoteCommandCallbackHandle = scheduleResult.getValue();
return Status::OK();
}
std::vector<RemoteCommandRequest>
VoteRequester::Algorithm::getRequests() const {
BSONObjBuilder requestVotesCmdBuilder;
requestVotesCmdBuilder.append("replSetRequestVotes", 1);
requestVotesCmdBuilder.append("setName", _rsConfig.getReplSetName());
requestVotesCmdBuilder.append("term", _term);
requestVotesCmdBuilder.append("candidateId", _candidateId);
requestVotesCmdBuilder.append("configVersion", _rsConfig.getConfigVersion());
BSONObjBuilder lastCommittedOp(requestVotesCmdBuilder.subobjStart("lastCommittedOp"));
lastCommittedOp.append("ts", _lastOplogEntry.getTimestamp());
lastCommittedOp.append("term", _lastOplogEntry.getTerm());
lastCommittedOp.done();
const BSONObj requestVotesCmd = requestVotesCmdBuilder.obj();
std::vector<RemoteCommandRequest> requests;
for (const auto& target : _targets) {
requests.push_back(RemoteCommandRequest(
target,
"admin",
requestVotesCmd,
Milliseconds(30*1000))); // trying to match current Socket timeout
}
return requests;
}
std::vector<RemoteCommandRequest>
ElectCmdRunner::Algorithm::getRequests() const {
const MemberConfig& selfConfig = _rsConfig.getMemberAt(_selfIndex);
std::vector<RemoteCommandRequest> requests;
BSONObjBuilder electCmdBuilder;
electCmdBuilder.append("replSetElect", 1);
electCmdBuilder.append("set", _rsConfig.getReplSetName());
electCmdBuilder.append("who", selfConfig.getHostAndPort().toString());
electCmdBuilder.append("whoid", selfConfig.getId());
electCmdBuilder.appendIntOrLL("cfgver", _rsConfig.getConfigVersion());
electCmdBuilder.append("round", _round);
const BSONObj replSetElectCmd = electCmdBuilder.obj();
// Schedule a RemoteCommandRequest for each non-DOWN node
for (std::vector<HostAndPort>::const_iterator it = _targets.begin();
it != _targets.end();
++it) {
invariant(*it != selfConfig.getHostAndPort());
requests.push_back(RemoteCommandRequest(
*it,
"admin",
replSetElectCmd,
Milliseconds(30*1000))); // trying to match current Socket timeout
}
return requests;
}
Fetcher::Fetcher(executor::TaskExecutor* executor,
const HostAndPort& source,
const std::string& dbname,
const BSONObj& findCmdObj,
const CallbackFn& work,
const BSONObj& metadata,
Milliseconds timeout,
std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy)
: _executor(executor),
_source(source),
_dbname(dbname),
_cmdObj(findCmdObj.getOwned()),
_metadata(metadata.getOwned()),
_work(work),
_timeout(timeout),
_firstRemoteCommandScheduler(
_executor,
RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _timeout),
stdx::bind(&Fetcher::_callback, this, stdx::placeholders::_1, kFirstBatchFieldName),
std::move(firstCommandRetryPolicy)) {
uassert(ErrorCodes::BadValue, "callback function cannot be null", work);
}
std::vector<RemoteCommandRequest> QuorumChecker::getRequests() const {
const bool isInitialConfig = _rsConfig->getConfigVersion() == 1;
const MemberConfig& myConfig = _rsConfig->getMemberAt(_myIndex);
std::vector<RemoteCommandRequest> requests;
if (hasReceivedSufficientResponses()) {
return requests;
}
ReplSetHeartbeatArgs hbArgs;
hbArgs.setSetName(_rsConfig->getReplSetName());
hbArgs.setProtocolVersion(1);
hbArgs.setConfigVersion(_rsConfig->getConfigVersion());
hbArgs.setCheckEmpty(isInitialConfig);
hbArgs.setSenderHost(myConfig.getHostAndPort());
hbArgs.setSenderId(myConfig.getId());
const BSONObj hbRequest = hbArgs.toBSON();
// Send a bunch of heartbeat requests.
// Schedule an operation when a "sufficient" number of them have completed, and use that
// to compute the quorum check results.
// Wait for the "completion" callback to finish, and then it's OK to return the results.
for (int i = 0; i < _rsConfig->getNumMembers(); ++i) {
if (_myIndex == i) {
// No need to check self for liveness or unreadiness.
continue;
}
requests.push_back(RemoteCommandRequest(
_rsConfig->getMemberAt(i).getHostAndPort(),
"admin",
hbRequest,
_rsConfig->getHeartbeatTimeoutPeriodMillis()));
}
return requests;
}
void CollectionCloner::_beginCollectionCallback(const executor::TaskExecutor::CallbackArgs& cbd) {
if (!cbd.status.isOK()) {
_finishCallback(cbd.status);
return;
}
MONGO_FAIL_POINT_BLOCK(initialSyncHangCollectionClonerBeforeEstablishingCursor, nssData) {
const BSONObj& data = nssData.getData();
auto nss = data["nss"].str();
// Only hang when cloning the specified collection, or if no collection was specified.
if (nss.empty() || _destNss.toString() == nss) {
while (MONGO_FAIL_POINT(initialSyncHangCollectionClonerBeforeEstablishingCursor) &&
!_isShuttingDown()) {
log() << "initialSyncHangCollectionClonerBeforeEstablishingCursor fail point "
"enabled for "
<< _destNss.toString() << ". Blocking until fail point is disabled.";
mongo::sleepsecs(1);
}
}
}
if (!_idIndexSpec.isEmpty() && _options.autoIndexId == CollectionOptions::NO) {
warning()
<< "Found the _id_ index spec but the collection specified autoIndexId of false on ns:"
<< this->_sourceNss;
}
auto collectionBulkLoader = _storageInterface->createCollectionForBulkLoading(
_destNss, _options, _idIndexSpec, _indexSpecs);
if (!collectionBulkLoader.isOK()) {
_finishCallback(collectionBulkLoader.getStatus());
return;
}
_stats.indexes = _indexSpecs.size();
if (!_idIndexSpec.isEmpty()) {
++_stats.indexes;
}
_collLoader = std::move(collectionBulkLoader.getValue());
BSONObjBuilder cmdObj;
EstablishCursorsCommand cursorCommand;
// The 'find' command is used when the number of cloning cursors is 1 to ensure
// the correctness of the collection cloning process until 'parallelCollectionScan'
// can be tested more extensively in context of initial sync.
if (_maxNumClonerCursors == 1) {
cmdObj.appendElements(
makeCommandWithUUIDorCollectionName("find", _options.uuid, _sourceNss));
cmdObj.append("noCursorTimeout", true);
// Set batchSize to be 0 to establish the cursor without fetching any documents,
// similar to the response format of 'parallelCollectionScan'.
cmdObj.append("batchSize", 0);
cursorCommand = Find;
} else {
cmdObj.appendElements(makeCommandWithUUIDorCollectionName(
"parallelCollectionScan", _options.uuid, _sourceNss));
cmdObj.append("numCursors", _maxNumClonerCursors);
cursorCommand = ParallelCollScan;
}
Client::initThreadIfNotAlready();
auto opCtx = cc().getOperationContext();
MONGO_FAIL_POINT_BLOCK(initialSyncHangBeforeCollectionClone, options) {
const BSONObj& data = options.getData();
if (data["namespace"].String() == _destNss.ns()) {
log() << "initial sync - initialSyncHangBeforeCollectionClone fail point "
"enabled. Blocking until fail point is disabled.";
while (MONGO_FAIL_POINT(initialSyncHangBeforeCollectionClone) && !_isShuttingDown()) {
mongo::sleepsecs(1);
}
}
}
_establishCollectionCursorsScheduler = stdx::make_unique<RemoteCommandRetryScheduler>(
_executor,
RemoteCommandRequest(_source,
_sourceNss.db().toString(),
cmdObj.obj(),
ReadPreferenceSetting::secondaryPreferredMetadata(),
opCtx,
RemoteCommandRequest::kNoTimeout),
[=](const RemoteCommandCallbackArgs& rcbd) {
_establishCollectionCursorsCallback(rcbd, cursorCommand);
},
RemoteCommandRetryScheduler::makeRetryPolicy(
numInitialSyncCollectionFindAttempts.load(),
executor::RemoteCommandRequest::kNoTimeout,
RemoteCommandRetryScheduler::kAllRetriableErrors));
auto scheduleStatus = _establishCollectionCursorsScheduler->startup();
LOG(1) << "Attempting to establish cursors with maxNumClonerCursors: " << _maxNumClonerCursors;
if (!scheduleStatus.isOK()) {
_establishCollectionCursorsScheduler.reset();
_finishCallback(scheduleStatus);
return;
}
}