void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
const Status respStatus = cbd.response.getStatus();
if (!respStatus.isOK()) {
// TODO: retry internally?
_setStatus(respStatus);
_doNextActions();
return;
}
const auto respBSON = cbd.response.getValue().data;
// There should not be any cloners yet
invariant(_databaseCloners.size() == 0);
const auto okElem = respBSON["ok"];
if (okElem.trueValue()) {
const auto dbsElem = respBSON["databases"].Obj();
BSONForEach(arrayElement, dbsElem) {
const BSONObj dbBSON = arrayElement.Obj();
const std::string name = dbBSON["name"].str();
++_clonersActive;
std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
try {
dbCloner.reset(new DatabaseCloner(
_exec,
_source,
name,
BSONObj(), // do not filter database out.
[](const BSONObj&) { return true; }, // clone all dbs.
_storage, // use storage provided.
[](const Status& status, const NamespaceString& srcNss) {
if (status.isOK()) {
log() << "collection clone finished: " << srcNss;
} else {
log() << "collection clone for '" << srcNss << "' failed due to "
<< status.toString();
}
},
[=](const Status& status) { _onEachDBCloneFinish(status, name); }));
} catch (...) {
// error creating, fails below.
}
Status s = dbCloner ? dbCloner->start() : Status(ErrorCodes::UnknownError, "Bad!");
if (!s.isOK()) {
std::string err = str::stream() << "could not create cloner for database: " << name
<< " due to: " << s.toString();
_setStatus(Status(ErrorCodes::InitialSyncFailure, err));
error() << err;
break; // exit for_each loop
}
// add cloner to list.
_databaseCloners.push_back(dbCloner);
}
} else {
void DatabasesCloner::_onListDatabaseFinish(
const executor::TaskExecutor::RemoteCommandCallbackArgs& cbd) {
Status respStatus = cbd.response.status;
if (respStatus.isOK()) {
respStatus = getStatusFromCommandResult(cbd.response.data);
}
UniqueLock lk(_mutex);
if (!respStatus.isOK()) {
LOG(1) << "'listDatabases' failed: " << respStatus;
_fail_inlock(&lk, respStatus);
return;
}
// There should not be any cloners yet.
invariant(_databaseCloners.size() == 0);
const auto respBSON = cbd.response.data;
auto databasesArray = _parseListDatabasesResponse(respBSON);
if (!databasesArray.isOK()) {
LOG(1) << "'listDatabases' returned a malformed response: "
<< databasesArray.getStatus().toString();
_fail_inlock(&lk, databasesArray.getStatus());
return;
}
auto dbsArray = databasesArray.getValue();
// Ensure that the 'admin' database is the first element in the array of databases so that it
// will be the first to be cloned. This allows users to authenticate against a database while
// initial sync is occurring.
_setAdminAsFirst(dbsArray);
for (BSONElement arrayElement : dbsArray) {
const BSONObj dbBSON = arrayElement.Obj();
// Check to see if we want to exclude this db from the clone.
if (!_includeDbFn(dbBSON)) {
LOG(1) << "Excluding database from the 'listDatabases' response: " << dbBSON;
continue;
}
if (!dbBSON.hasField("name")) {
LOG(1) << "Excluding database due to the 'listDatabases' response not containing a "
"'name' field for this entry: "
<< dbBSON;
}
const std::string dbName = dbBSON["name"].str();
std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
// filters for DatabasesCloner.
const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
const auto collName = collInfo["name"].str();
const NamespaceString ns(dbName, collName);
if (ns.isSystem() && !ns.isLegalClientSystemNS()) {
LOG(1) << "Skipping 'system' collection: " << ns.ns();
return false;
}
if (!ns.isNormal()) {
LOG(1) << "Skipping non-normal collection: " << ns.ns();
return false;
}
LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
return true;
};
const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
if (status.isOK()) {
LOG(1) << "collection clone finished: " << srcNss;
} else {
error() << "collection clone for '" << srcNss << "' failed due to "
<< status.toString();
}
};
const auto onDbFinish = [this, dbName](const Status& status) {
_onEachDBCloneFinish(status, dbName);
};
Status startStatus = Status::OK();
try {
dbCloner.reset(new DatabaseCloner(
_exec,
_dbWorkThreadPool,
_source,
dbName,
BSONObj(), // do not filter collections out during listCollections call.
collectionFilterPred,
_storage, // use storage provided.
onCollectionFinish,
onDbFinish));
if (_scheduleDbWorkFn) {
dbCloner->setScheduleDbWorkFn_forTest(_scheduleDbWorkFn);
}
if (_startCollectionClonerFn) {
dbCloner->setStartCollectionClonerFn(_startCollectionClonerFn);
}
// Start first database cloner.
if (_databaseCloners.empty()) {
startStatus = dbCloner->startup();
}
} catch (...) {
startStatus = exceptionToStatus();
}
if (!startStatus.isOK()) {
std::string err = str::stream() << "could not create cloner for database: " << dbName
<< " due to: " << startStatus.toString();
_setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
error() << err;
break; // exit for_each loop
}
// add cloner to list.
_databaseCloners.push_back(dbCloner);
}
if (_databaseCloners.size() == 0) {
if (_status.isOK()) {
_succeed_inlock(&lk);
} else {
_fail_inlock(&lk, _status);
}
}
}
void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
Status respStatus = cbd.response.getStatus();
if (respStatus.isOK()) {
respStatus = getStatusFromCommandResult(cbd.response.getValue().data);
}
UniqueLock lk(_mutex);
if (!respStatus.isOK()) {
LOG(1) << "listDatabases failed: " << respStatus;
_setStatus_inlock(respStatus);
_failed_inlock(lk);
return;
}
const auto respBSON = cbd.response.getValue().data;
// There should not be any cloners yet
invariant(_databaseCloners.size() == 0);
const auto dbsElem = respBSON["databases"].Obj();
BSONForEach(arrayElement, dbsElem) {
const BSONObj dbBSON = arrayElement.Obj();
// Check to see if we want to exclude this db from the clone.
if (!_includeDbFn(dbBSON)) {
LOG(1) << "excluding db: " << dbBSON;
continue;
}
const std::string dbName = dbBSON["name"].str();
++_clonersActive;
std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
Status startStatus(ErrorCodes::NotYetInitialized,
"The DatabasesCloner could not be started.");
// filters for DatabasesCloner.
const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
const auto collName = collInfo["name"].str();
const NamespaceString ns(dbName, collName);
if (ns.isSystem() && !legalClientSystemNS(ns.ns(), true)) {
LOG(1) << "Skipping 'system' collection: " << ns.ns();
return false;
}
if (!ns.isNormal()) {
LOG(1) << "Skipping non-normal collection: " << ns.ns();
return false;
}
LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
return true;
};
const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
if (status.isOK()) {
LOG(1) << "collection clone finished: " << srcNss;
} else {
warning() << "collection clone for '" << srcNss << "' failed due to "
<< status.toString();
}
};
const auto onDbFinish = [this, dbName](const Status& status) {
_onEachDBCloneFinish(status, dbName);
};
try {
dbCloner.reset(new DatabaseCloner(
_exec,
_source,
dbName,
BSONObj(), // do not filter collections out during listCollections call.
collectionFilterPred,
_storage, // use storage provided.
onCollectionFinish,
onDbFinish));
// Start database cloner.
startStatus = dbCloner->start();
} catch (...) {
startStatus = exceptionToStatus();
}
if (!startStatus.isOK()) {
std::string err = str::stream() << "could not create cloner for database: " << dbName
<< " due to: " << startStatus.toString();
_setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
error() << err;
break; // exit for_each loop
}
// add cloner to list.
_databaseCloners.push_back(dbCloner);
}
if (_databaseCloners.size() == 0) {
if (_status.isOK()) {
_active = false;
lk.unlock();
_finishFn(_status);
} else {
_failed_inlock(lk);
}
}
}