void DatabasesCloner::_onEachDBCloneFinish(const Status& status, const std::string& name) {
UniqueLock lk(_mutex);
if (!status.isOK()) {
warning() << "database '" << name << "' (" << (_stats.databasesCloned + 1) << " of "
<< _databaseCloners.size() << ") clone failed due to " << status.toString();
_fail_inlock(&lk, status);
return;
}
if (StringData(name).equalCaseInsensitive("admin")) {
LOG(1) << "Finished the 'admin' db, now calling isAdminDbValid.";
// Do special checks for the admin database because of auth. collections.
auto adminStatus = Status(ErrorCodes::NotYetInitialized, "");
{
// TODO: Move isAdminDbValid() out of the collection/database cloner code paths.
OperationContext* opCtx = cc().getOperationContext();
ServiceContext::UniqueOperationContext opCtxPtr;
if (!opCtx) {
opCtxPtr = cc().makeOperationContext();
opCtx = opCtxPtr.get();
}
adminStatus = _storage->isAdminDbValid(opCtx);
}
if (!adminStatus.isOK()) {
LOG(1) << "Validation failed on 'admin' db due to " << adminStatus;
_fail_inlock(&lk, adminStatus);
return;
}
}
_stats.databasesCloned++;
if (_stats.databasesCloned == _databaseCloners.size()) {
_succeed_inlock(&lk);
return;
}
// Start next database cloner.
auto&& dbCloner = _databaseCloners[_stats.databasesCloned];
auto startStatus = dbCloner->startup();
if (!startStatus.isOK()) {
warning() << "failed to schedule database '" << name << "' ("
<< (_stats.databasesCloned + 1) << " of " << _databaseCloners.size()
<< ") due to " << startStatus.toString();
_fail_inlock(&lk, startStatus);
return;
}
}
Status DatabasesCloner::startup() {
UniqueLock lk(_mutex);
invariant(!_active);
_active = true;
if (!_status.isOK() && _status.code() != ErrorCodes::NotYetInitialized) {
return _status;
}
_status = Status::OK();
// Schedule listDatabase command which will kick off the database cloner per result db.
Request listDBsReq(_source,
"admin",
BSON("listDatabases" << true),
rpc::ServerSelectionMetadata(true, boost::none).toBSON(),
nullptr);
_listDBsScheduler = stdx::make_unique<RemoteCommandRetryScheduler>(
_exec,
listDBsReq,
stdx::bind(&DatabasesCloner::_onListDatabaseFinish, this, stdx::placeholders::_1),
RemoteCommandRetryScheduler::makeRetryPolicy(
numInitialSyncListDatabasesAttempts,
executor::RemoteCommandRequest::kNoTimeout,
RemoteCommandRetryScheduler::kAllRetriableErrors));
auto s = _listDBsScheduler->startup();
if (!s.isOK()) {
_fail_inlock(&lk, s);
}
return _status;
}
void DatabasesCloner::_onListDatabaseFinish(
const executor::TaskExecutor::RemoteCommandCallbackArgs& cbd) {
Status respStatus = cbd.response.status;
if (respStatus.isOK()) {
respStatus = getStatusFromCommandResult(cbd.response.data);
}
UniqueLock lk(_mutex);
if (!respStatus.isOK()) {
LOG(1) << "'listDatabases' failed: " << respStatus;
_fail_inlock(&lk, respStatus);
return;
}
// There should not be any cloners yet.
invariant(_databaseCloners.size() == 0);
const auto respBSON = cbd.response.data;
auto databasesArray = _parseListDatabasesResponse(respBSON);
if (!databasesArray.isOK()) {
LOG(1) << "'listDatabases' returned a malformed response: "
<< databasesArray.getStatus().toString();
_fail_inlock(&lk, databasesArray.getStatus());
return;
}
auto dbsArray = databasesArray.getValue();
// Ensure that the 'admin' database is the first element in the array of databases so that it
// will be the first to be cloned. This allows users to authenticate against a database while
// initial sync is occurring.
_setAdminAsFirst(dbsArray);
for (BSONElement arrayElement : dbsArray) {
const BSONObj dbBSON = arrayElement.Obj();
// Check to see if we want to exclude this db from the clone.
if (!_includeDbFn(dbBSON)) {
LOG(1) << "Excluding database from the 'listDatabases' response: " << dbBSON;
continue;
}
if (!dbBSON.hasField("name")) {
LOG(1) << "Excluding database due to the 'listDatabases' response not containing a "
"'name' field for this entry: "
<< dbBSON;
}
const std::string dbName = dbBSON["name"].str();
std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
// filters for DatabasesCloner.
const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
const auto collName = collInfo["name"].str();
const NamespaceString ns(dbName, collName);
if (ns.isSystem() && !ns.isLegalClientSystemNS()) {
LOG(1) << "Skipping 'system' collection: " << ns.ns();
return false;
}
if (!ns.isNormal()) {
LOG(1) << "Skipping non-normal collection: " << ns.ns();
return false;
}
LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
return true;
};
const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
if (status.isOK()) {
LOG(1) << "collection clone finished: " << srcNss;
} else {
error() << "collection clone for '" << srcNss << "' failed due to "
<< status.toString();
}
};
const auto onDbFinish = [this, dbName](const Status& status) {
_onEachDBCloneFinish(status, dbName);
};
Status startStatus = Status::OK();
try {
dbCloner.reset(new DatabaseCloner(
_exec,
_dbWorkThreadPool,
_source,
dbName,
BSONObj(), // do not filter collections out during listCollections call.
collectionFilterPred,
_storage, // use storage provided.
onCollectionFinish,
onDbFinish));
if (_scheduleDbWorkFn) {
dbCloner->setScheduleDbWorkFn_forTest(_scheduleDbWorkFn);
}
if (_startCollectionClonerFn) {
dbCloner->setStartCollectionClonerFn(_startCollectionClonerFn);
}
// Start first database cloner.
if (_databaseCloners.empty()) {
startStatus = dbCloner->startup();
}
} catch (...) {
startStatus = exceptionToStatus();
}
if (!startStatus.isOK()) {
std::string err = str::stream() << "could not create cloner for database: " << dbName
<< " due to: " << startStatus.toString();
_setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
error() << err;
break; // exit for_each loop
}
// add cloner to list.
_databaseCloners.push_back(dbCloner);
}
if (_databaseCloners.size() == 0) {
if (_status.isOK()) {
_succeed_inlock(&lk);
} else {
_fail_inlock(&lk, _status);
}
}
}
void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
Status respStatus = cbd.response.status;
if (respStatus.isOK()) {
respStatus = getStatusFromCommandResult(cbd.response.data);
}
UniqueLock lk(_mutex);
if (!respStatus.isOK()) {
LOG(1) << "listDatabases failed: " << respStatus;
_fail_inlock(&lk, respStatus);
return;
}
const auto respBSON = cbd.response.data;
// There should not be any cloners yet
invariant(_databaseCloners.size() == 0);
const auto dbsElem = respBSON["databases"].Obj();
BSONForEach(arrayElement, dbsElem) {
const BSONObj dbBSON = arrayElement.Obj();
// Check to see if we want to exclude this db from the clone.
if (!_includeDbFn(dbBSON)) {
LOG(1) << "excluding db: " << dbBSON;
continue;
}
const std::string dbName = dbBSON["name"].str();
std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
// filters for DatabasesCloner.
const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
const auto collName = collInfo["name"].str();
const NamespaceString ns(dbName, collName);
if (ns.isSystem() && !legalClientSystemNS(ns.ns())) {
LOG(1) << "Skipping 'system' collection: " << ns.ns();
return false;
}
if (!ns.isNormal()) {
LOG(1) << "Skipping non-normal collection: " << ns.ns();
return false;
}
LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
return true;
};
const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
if (status.isOK()) {
LOG(1) << "collection clone finished: " << srcNss;
} else {
warning() << "collection clone for '" << srcNss << "' failed due to "
<< status.toString();
}
};
const auto onDbFinish = [this, dbName](const Status& status) {
_onEachDBCloneFinish(status, dbName);
};
Status startStatus = Status::OK();
try {
dbCloner.reset(new DatabaseCloner(
_exec,
_dbWorkThreadPool,
_source,
dbName,
BSONObj(), // do not filter collections out during listCollections call.
collectionFilterPred,
_storage, // use storage provided.
onCollectionFinish,
onDbFinish));
if (_scheduleDbWorkFn) {
dbCloner->setScheduleDbWorkFn_forTest(_scheduleDbWorkFn);
}
// Start first database cloner.
if (_databaseCloners.empty()) {
startStatus = dbCloner->startup();
}
} catch (...) {
startStatus = exceptionToStatus();
}
if (!startStatus.isOK()) {
std::string err = str::stream() << "could not create cloner for database: " << dbName
<< " due to: " << startStatus.toString();
_setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
error() << err;
break; // exit for_each loop
}
// add cloner to list.
_databaseCloners.push_back(dbCloner);
}
if (_databaseCloners.size() == 0) {
if (_status.isOK()) {
_succeed_inlock(&lk);
} else {
_fail_inlock(&lk, _status);
}
}
}