void CollectionCloner::_listIndexesCallback(const Fetcher::QueryResponseStatus& fetchResult, Fetcher::NextAction* nextAction, BSONObjBuilder* getMoreBob) { if (!fetchResult.isOK()) { _finishCallback(nullptr, fetchResult.getStatus()); return; } auto batchData(fetchResult.getValue()); auto&& documents = batchData.documents; if (documents.empty()) { warning() << "No indexes found for collection " << _sourceNss.ns() << " while cloning from " << _source; } // We may be called with multiple batches leading to a need to grow _indexSpecs. _indexSpecs.reserve(_indexSpecs.size() + documents.size()); _indexSpecs.insert(_indexSpecs.end(), documents.begin(), documents.end()); // The fetcher will continue to call with kGetMore until an error or the last batch. if (*nextAction == Fetcher::NextAction::kGetMore) { invariant(getMoreBob); getMoreBob->append("getMore", batchData.cursorId); getMoreBob->append("collection", batchData.nss.coll()); return; } // We have all of the indexes now, so we can start cloning the collection data. auto&& scheduleResult = _scheduleDbWorkFn( stdx::bind(&CollectionCloner::_beginCollectionCallback, this, stdx::placeholders::_1)); if (!scheduleResult.isOK()) { _finishCallback(nullptr, scheduleResult.getStatus()); return; } _dbWorkCallbackHandle = scheduleResult.getValue(); }
void CollectionCloner::_listIndexesCallback(const Fetcher::QueryResponseStatus& fetchResult, Fetcher::NextAction* nextAction, BSONObjBuilder* getMoreBob) { const bool collectionIsEmpty = fetchResult == ErrorCodes::NamespaceNotFound; if (collectionIsEmpty) { // Schedule collection creation and finish callback. auto&& scheduleResult = _scheduleDbWorkFn([this](const executor::TaskExecutor::CallbackArgs& cbd) { if (!cbd.status.isOK()) { _finishCallback(cbd.status); return; } auto txn = cbd.txn; txn->setReplicatedWrites(false); auto&& createStatus = _storageInterface->createCollection(txn, _destNss, _options); _finishCallback(createStatus); }); if (!scheduleResult.isOK()) { _finishCallback(scheduleResult.getStatus()); } return; }; if (!fetchResult.isOK()) { Status newStatus{fetchResult.getStatus().code(), str::stream() << "During listIndexes call on collection '" << _sourceNss.ns() << "' there was an error '" << fetchResult.getStatus().reason() << "'"}; _finishCallback(newStatus); return; } auto batchData(fetchResult.getValue()); auto&& documents = batchData.documents; if (documents.empty()) { warning() << "No indexes found for collection " << _sourceNss.ns() << " while cloning from " << _source; } UniqueLock lk(_mutex); // We may be called with multiple batches leading to a need to grow _indexSpecs. _indexSpecs.reserve(_indexSpecs.size() + documents.size()); for (auto&& doc : documents) { if (StringData("_id_") == doc["name"].str()) { _idIndexSpec = doc; continue; } _indexSpecs.push_back(doc); } lk.unlock(); // The fetcher will continue to call with kGetMore until an error or the last batch. if (*nextAction == Fetcher::NextAction::kGetMore) { invariant(getMoreBob); getMoreBob->append("getMore", batchData.cursorId); getMoreBob->append("collection", batchData.nss.coll()); return; } // We have all of the indexes now, so we can start cloning the collection data. auto&& scheduleResult = _scheduleDbWorkFn( stdx::bind(&CollectionCloner::_beginCollectionCallback, this, stdx::placeholders::_1)); if (!scheduleResult.isOK()) { _finishCallback(scheduleResult.getStatus()); return; } }
void CollectionCloner::_listIndexesCallback(const Fetcher::QueryResponseStatus& fetchResult, Fetcher::NextAction* nextAction, BSONObjBuilder* getMoreBob) { const bool collectionIsEmpty = fetchResult == ErrorCodes::NamespaceNotFound; if (collectionIsEmpty) { // Schedule collection creation and finish callback. auto&& scheduleResult = _scheduleDbWorkFn([this](const executor::TaskExecutor::CallbackArgs& cbd) { if (!cbd.status.isOK()) { _finishCallback(cbd.status); return; } auto opCtx = cbd.opCtx; UnreplicatedWritesBlock uwb(opCtx); auto&& createStatus = _storageInterface->createCollection(opCtx, _destNss, _options); _finishCallback(createStatus); }); if (!scheduleResult.isOK()) { _finishCallback(scheduleResult.getStatus()); } return; }; if (!fetchResult.isOK()) { _finishCallback(fetchResult.getStatus().withContext( str::stream() << "listIndexes call failed on collection '" << _sourceNss.ns() << "'")); return; } auto batchData(fetchResult.getValue()); auto&& documents = batchData.documents; if (documents.empty()) { warning() << "No indexes found for collection " << _sourceNss.ns() << " while cloning from " << _source; } UniqueLock lk(_mutex); // When listing indexes by UUID, the sync source may use a different name for the collection // as result of renaming or two-phase drop. As the index spec also includes a 'ns' field, this // must be rewritten. BSONObjBuilder nsFieldReplacementBuilder; nsFieldReplacementBuilder.append("ns", _sourceNss.ns()); BSONElement nsFieldReplacementElem = nsFieldReplacementBuilder.done().firstElement(); // We may be called with multiple batches leading to a need to grow _indexSpecs. _indexSpecs.reserve(_indexSpecs.size() + documents.size()); for (auto&& doc : documents) { // The addField replaces the 'ns' field with the correct name, see above. if (StringData("_id_") == doc["name"].str()) { _idIndexSpec = doc.addField(nsFieldReplacementElem); continue; } _indexSpecs.push_back(doc.addField(nsFieldReplacementElem)); } lk.unlock(); // The fetcher will continue to call with kGetMore until an error or the last batch. if (*nextAction == Fetcher::NextAction::kGetMore) { invariant(getMoreBob); getMoreBob->append("getMore", batchData.cursorId); getMoreBob->append("collection", batchData.nss.coll()); return; } // We have all of the indexes now, so we can start cloning the collection data. auto&& scheduleResult = _scheduleDbWorkFn( [=](const executor::TaskExecutor::CallbackArgs& cbd) { _beginCollectionCallback(cbd); }); if (!scheduleResult.isOK()) { _finishCallback(scheduleResult.getStatus()); return; } }
void OplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, BSONObjBuilder* getMoreBob) { // if target cut connections between connecting and querying (for // example, because it stepped down) we might not have a cursor if (!result.isOK()) { LOG(2) << "Error returned from oplog query: " << result.getStatus(); _onShutdown(result.getStatus()); return; } const auto& queryResponse = result.getValue(); rpc::ReplSetMetadata metadata; // Forward metadata (containing liveness information) to data replicator external state. bool receivedMetadata = queryResponse.otherFields.metadata.hasElement(rpc::kReplSetMetadataFieldName); if (receivedMetadata) { const auto& metadataObj = queryResponse.otherFields.metadata; auto metadataResult = rpc::ReplSetMetadata::readFromMetadata(metadataObj); if (!metadataResult.isOK()) { error() << "invalid replication metadata from sync source " << _fetcher.getSource() << ": " << metadataResult.getStatus() << ": " << metadataObj; _onShutdown(metadataResult.getStatus()); return; } metadata = metadataResult.getValue(); _dataReplicatorExternalState->processMetadata(metadata); } const auto& documents = queryResponse.documents; auto firstDocToApply = documents.cbegin(); if (!documents.empty()) { LOG(2) << "oplog fetcher read " << documents.size() << " operations from remote oplog starting at " << documents.front()["ts"] << " and ending at " << documents.back()["ts"]; } else { LOG(2) << "oplog fetcher read 0 operations from remote oplog"; } auto opTimeWithHash = getLastOpTimeWithHashFetched(); // Check start of remote oplog and, if necessary, stop fetcher to execute rollback. if (queryResponse.first) { auto status = checkRemoteOplogStart(documents, opTimeWithHash); if (!status.isOK()) { // Stop oplog fetcher and execute rollback. _onShutdown(status, opTimeWithHash); return; } // If this is the first batch and no rollback is needed, skip the first document. firstDocToApply++; } auto validateResult = OplogFetcher::validateDocuments( documents, queryResponse.first, opTimeWithHash.opTime.getTimestamp()); if (!validateResult.isOK()) { _onShutdown(validateResult.getStatus(), opTimeWithHash); return; } auto info = validateResult.getValue(); // TODO: back pressure handling will be added in SERVER-23499. _enqueueDocumentsFn(firstDocToApply, documents.cend(), info, queryResponse.elapsedMillis); // Update last fetched info. if (firstDocToApply != documents.cend()) { opTimeWithHash = info.lastDocument; LOG(3) << "batch resetting last fetched optime: " << opTimeWithHash.opTime << "; hash: " << opTimeWithHash.value; stdx::unique_lock<stdx::mutex> lock(_mutex); _lastFetched = opTimeWithHash; } if (_dataReplicatorExternalState->shouldStopFetching(_fetcher.getSource(), metadata)) { _onShutdown(Status(ErrorCodes::InvalidSyncSource, str::stream() << "sync source " << _fetcher.getSource().toString() << " (last optime: " << metadata.getLastOpVisible().toString() << "; sync source index: " << metadata.getSyncSourceIndex() << "; primary index: " << metadata.getPrimaryIndex() << ") is no longer valid"), opTimeWithHash); return; } // No more data. Stop processing and return Status::OK along with last // fetch info. if (!getMoreBob) { _onShutdown(Status::OK(), opTimeWithHash); return; } getMoreBob->appendElements(makeGetMoreCommandObject(_dataReplicatorExternalState, queryResponse.nss, queryResponse.cursorId, _awaitDataTimeout)); }
void AbstractOplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, BSONObjBuilder* getMoreBob) { Status responseStatus = _checkForShutdownAndConvertStatus(result.getStatus(), "error in fetcher batch callback"); if (ErrorCodes::CallbackCanceled == responseStatus) { LOG(1) << _getComponentName() << " oplog query cancelled to " << _getSource() << ": " << redact(responseStatus); _finishCallback(responseStatus); return; } // If target cut connections between connecting and querying (for // example, because it stepped down) we might not have a cursor. if (!responseStatus.isOK()) { BSONObj findCommandObj = _makeFindCommandObject(_nss, _getLastOpTimeWithHashFetched().opTime); BSONObj metadataObj = _makeMetadataObject(); { stdx::lock_guard<stdx::mutex> lock(_mutex); if (_fetcherRestarts == _maxFetcherRestarts) { log() << "Error returned from oplog query (no more query restarts left): " << redact(responseStatus); } else { log() << "Restarting oplog query due to error: " << redact(responseStatus) << ". Last fetched optime (with hash): " << _lastFetched << ". Restarts remaining: " << (_maxFetcherRestarts - _fetcherRestarts); _fetcherRestarts++; // Destroying current instance in _shuttingDownFetcher will possibly block. _shuttingDownFetcher.reset(); // Move the old fetcher into the shutting down instance. _shuttingDownFetcher.swap(_fetcher); // Create and start fetcher with current term and new starting optime. _fetcher = _makeFetcher(findCommandObj, metadataObj); auto scheduleStatus = _scheduleFetcher_inlock(); if (scheduleStatus.isOK()) { log() << "Scheduled new oplog query " << _fetcher->toString(); return; } error() << "Error scheduling new oplog query: " << redact(scheduleStatus) << ". Returning current oplog query error: " << redact(responseStatus); } } _finishCallback(responseStatus); return; } // Reset fetcher restart counter on successful response. { stdx::lock_guard<stdx::mutex> lock(_mutex); invariant(_isActive_inlock()); _fetcherRestarts = 0; } if (_isShuttingDown()) { _finishCallback( Status(ErrorCodes::CallbackCanceled, _getComponentName() + " shutting down")); return; } // At this point we have a successful batch and can call the subclass's _onSuccessfulBatch. const auto& queryResponse = result.getValue(); auto batchResult = _onSuccessfulBatch(queryResponse); if (!batchResult.isOK()) { // The stopReplProducer fail point expects this to return successfully. If another fail // point wants this to return unsuccessfully, it should use a different error code. if (batchResult.getStatus() == ErrorCodes::FailPointEnabled) { _finishCallback(Status::OK()); return; } _finishCallback(batchResult.getStatus()); return; } // No more data. Stop processing and return Status::OK. if (!getMoreBob) { _finishCallback(Status::OK()); return; } // We have now processed the batch and should move forward our view of _lastFetched. Note that // the _lastFetched value will not be updated until the _onSuccessfulBatch function is // completed. const auto& documents = queryResponse.documents; if (documents.size() > 0) { auto lastDocRes = AbstractOplogFetcher::parseOpTimeWithHash(documents.back()); if (!lastDocRes.isOK()) { _finishCallback(lastDocRes.getStatus()); return; } auto lastDoc = lastDocRes.getValue(); LOG(3) << _getComponentName() << " setting last fetched optime ahead after batch: " << lastDoc.opTime << "; hash: " << lastDoc.value; stdx::lock_guard<stdx::mutex> lock(_mutex); _lastFetched = lastDoc; } // Check for shutdown to save an unnecessary `getMore` request. if (_isShuttingDown()) { _finishCallback( Status(ErrorCodes::CallbackCanceled, _getComponentName() + " shutting down")); return; } // The _onSuccessfulBatch function returns the `getMore` command we want to send. getMoreBob->appendElements(batchResult.getValue()); }