std::unique_ptr<Fetcher> AbstractOplogFetcher::_makeFetcher(const BSONObj& findCommandObj,
const BSONObj& metadataObj) {
return stdx::make_unique<Fetcher>(
_getExecutor(),
_source,
_nss.db().toString(),
findCommandObj,
[this](const StatusWith<Fetcher::QueryResponse>& resp,
Fetcher::NextAction*,
BSONObjBuilder* builder) { return _callback(resp, builder); },
metadataObj,
_getFindMaxTime() + kNetworkTimeoutBufferMS,
_getGetMoreMaxTime() + kNetworkTimeoutBufferMS);
}
StatusWith<BSONObj> OplogFetcher::_onSuccessfulBatch(const Fetcher::QueryResponse& queryResponse) {
// Stop fetching and return on fail point.
// This fail point makes the oplog fetcher ignore the downloaded batch of operations and not
// error out. The FailPointEnabled error will be caught by the AbstractOplogFetcher.
if (MONGO_FAIL_POINT(stopReplProducer)) {
return Status(ErrorCodes::FailPointEnabled, "stopReplProducer fail point is enabled");
}
const auto& documents = queryResponse.documents;
auto firstDocToApply = documents.cbegin();
if (!documents.empty()) {
LOG(2) << "oplog fetcher read " << documents.size()
<< " operations from remote oplog starting at " << documents.front()["ts"]
<< " and ending at " << documents.back()["ts"];
} else {
LOG(2) << "oplog fetcher read 0 operations from remote oplog";
}
auto oqMetadataResult = parseOplogQueryMetadata(queryResponse);
if (!oqMetadataResult.isOK()) {
error() << "invalid oplog query metadata from sync source " << _getSource() << ": "
<< oqMetadataResult.getStatus() << ": " << queryResponse.otherFields.metadata;
return oqMetadataResult.getStatus();
}
auto oqMetadata = oqMetadataResult.getValue();
// This lastFetched value is the last OpTime from the previous batch.
auto lastFetched = _getLastOpTimeFetched();
// Check start of remote oplog and, if necessary, stop fetcher to execute rollback.
if (queryResponse.first) {
auto remoteRBID = oqMetadata ? boost::make_optional(oqMetadata->getRBID()) : boost::none;
auto remoteLastApplied =
oqMetadata ? boost::make_optional(oqMetadata->getLastOpApplied()) : boost::none;
auto status = checkRemoteOplogStart(documents,
lastFetched,
remoteLastApplied,
_requiredRBID,
remoteRBID,
_requireFresherSyncSource);
if (!status.isOK()) {
// Stop oplog fetcher and execute rollback if necessary.
return status;
}
LOG(1) << "oplog fetcher successfully fetched from " << _getSource();
// If this is the first batch, no rollback is needed and we don't want to enqueue the first
// document, skip it.
if (_startingPoint == StartingPoint::kSkipFirstDoc) {
firstDocToApply++;
}
}
auto validateResult =
OplogFetcher::validateDocuments(documents, queryResponse.first, lastFetched.getTimestamp());
if (!validateResult.isOK()) {
return validateResult.getStatus();
}
auto info = validateResult.getValue();
// Process replset metadata. It is important that this happen after we've validated the
// first batch, so we don't progress our knowledge of the commit point from a
// response that triggers a rollback.
rpc::ReplSetMetadata replSetMetadata;
bool receivedReplMetadata =
queryResponse.otherFields.metadata.hasElement(rpc::kReplSetMetadataFieldName);
if (receivedReplMetadata) {
const auto& metadataObj = queryResponse.otherFields.metadata;
auto metadataResult = rpc::ReplSetMetadata::readFromMetadata(metadataObj);
if (!metadataResult.isOK()) {
error() << "invalid replication metadata from sync source " << _getSource() << ": "
<< metadataResult.getStatus() << ": " << metadataObj;
return metadataResult.getStatus();
}
replSetMetadata = metadataResult.getValue();
// We will only ever have OplogQueryMetadata if we have ReplSetMetadata, so it is safe
// to call processMetadata() in this if block.
_dataReplicatorExternalState->processMetadata(replSetMetadata, oqMetadata);
}
// Increment stats. We read all of the docs in the query.
opsReadStats.increment(info.networkDocumentCount);
networkByteStats.increment(info.networkDocumentBytes);
// Record time for each batch.
getmoreReplStats.recordMillis(durationCount<Milliseconds>(queryResponse.elapsedMillis));
// TODO: back pressure handling will be added in SERVER-23499.
auto status = _enqueueDocumentsFn(firstDocToApply, documents.cend(), info);
if (!status.isOK()) {
return status;
}
if (_dataReplicatorExternalState->shouldStopFetching(
_getSource(), replSetMetadata, oqMetadata)) {
str::stream errMsg;
errMsg << "sync source " << _getSource().toString();
errMsg << " (config version: " << replSetMetadata.getConfigVersion();
// If OplogQueryMetadata was provided, its values were used to determine if we should
// stop fetching from this sync source.
if (oqMetadata) {
errMsg << "; last applied optime: " << oqMetadata->getLastOpApplied().toString();
errMsg << "; sync source index: " << oqMetadata->getSyncSourceIndex();
errMsg << "; primary index: " << oqMetadata->getPrimaryIndex();
} else {
errMsg << "; last visible optime: " << replSetMetadata.getLastOpVisible().toString();
errMsg << "; sync source index: " << replSetMetadata.getSyncSourceIndex();
errMsg << "; primary index: " << replSetMetadata.getPrimaryIndex();
}
errMsg << ") is no longer valid";
return Status(ErrorCodes::InvalidSyncSource, errMsg);
}
auto lastCommittedWithCurrentTerm =
_dataReplicatorExternalState->getCurrentTermAndLastCommittedOpTime();
return makeGetMoreCommandObject(queryResponse.nss,
queryResponse.cursorId,
lastCommittedWithCurrentTerm,
_getGetMoreMaxTime(),
_batchSize);
}
Milliseconds OplogFetcher::getAwaitDataTimeout_forTest() const {
return _getGetMoreMaxTime();
}
