void DatabasesCloner::_onEachDBCloneFinish(const Status& status, const std::string& name) {
UniqueLock lk(_mutex);
auto clonersLeft = --_clonersActive;
if (!status.isOK()) {
warning() << "database '" << name << "' clone failed due to " << status.toString();
_setStatus_inlock(status);
if (clonersLeft == 0) {
_failed_inlock(lk);
} else {
// After cancellation this callback will called until clonersLeft = 0.
_cancelCloners_inlock(lk);
}
return;
}
LOG(2) << "Database clone finished: " << name;
if (StringData(name).equalCaseInsensitive("admin")) {
LOG(1) << "Finished the 'admin' db, now calling isAdminDbValid.";
// Do special checks for the admin database because of auth. collections.
const auto adminStatus = _storage->isAdminDbValid(nullptr /* TODO: wire in txn*/);
if (!adminStatus.isOK()) {
_setStatus_inlock(adminStatus);
}
}
if (clonersLeft == 0) {
_active = false;
// All cloners are done, trigger event.
LOG(2) << "All database clones finished, calling _finishFn.";
lk.unlock();
_finishFn(_status);
return;
}
}
Status DatabasesCloner::startup() {
UniqueLock lk(_mutex);
invariant(!_active);
_active = true;
if (!_status.isOK() && _status.code() != ErrorCodes::NotYetInitialized) {
return _status;
}
_status = Status::OK();
// Schedule listDatabase command which will kick off the database cloner per result db.
Request listDBsReq(_source,
"admin",
BSON("listDatabases" << true),
rpc::ServerSelectionMetadata(true, boost::none).toBSON());
_listDBsScheduler = stdx::make_unique<RemoteCommandRetryScheduler>(
_exec,
listDBsReq,
stdx::bind(&DatabasesCloner::_onListDatabaseFinish, this, stdx::placeholders::_1),
RemoteCommandRetryScheduler::makeRetryPolicy(
numListDatabasesRetries,
executor::RemoteCommandRequest::kNoTimeout,
RemoteCommandRetryScheduler::kAllRetriableErrors));
auto s = _listDBsScheduler->startup();
if (!s.isOK()) {
_setStatus_inlock(s);
_failed_inlock(lk);
}
return _status;
}
void DatabasesCloner::_succeed_inlock(UniqueLock* lk) {
LOG(3) << "DatabasesCloner::_succeed_inlock called";
const auto status = Status::OK();
_setStatus_inlock(status);
auto finish = _finishFn;
lk->unlock();
LOG(3) << "DatabasesCloner - calling _finishFn with status OK";
finish(status);
lk->lock();
_active = false;
}
void DatabasesCloner::_fail_inlock(UniqueLock* lk, Status status) {
LOG(3) << "DatabasesCloner::_fail_inlock called";
if (!_active) {
return;
}
_setStatus_inlock(status);
// TODO: shutdown outstanding work, like any cloners active
auto finish = _finishFn;
lk->unlock();
LOG(3) << "DatabasesCloner - calling _finishFn with status: " << _status;
finish(status);
lk->lock();
_active = false;
}
void DatabasesCloner::shutdown() {
if (auto listDatabaseScheduler = _getListDatabasesScheduler()) {
listDatabaseScheduler->shutdown();
}
auto databaseCloners = _getDatabaseCloners();
for (auto&& cloner : databaseCloners) {
cloner->shutdown();
}
LockGuard lk(_mutex);
if (!_active) {
return;
}
_active = false;
_setStatus_inlock({ErrorCodes::CallbackCanceled, "Initial Sync Cancelled."});
}
void DatabasesCloner::_succeed_inlock(UniqueLock* lk) {
LOG(3) << "DatabasesCloner::_succeed_inlock called";
const auto status = Status::OK();
_setStatus_inlock(status);
invariant(_finishFn);
auto finish = _finishFn;
_finishFn = OnFinishFn();
lk->unlock();
LOG(3) << "DatabasesCloner - calling _finishFn with status OK";
finish(status);
// Release any resources that might be held by the '_finishFn' (moved to 'finish') function
// object.
finish = OnFinishFn();
lk->lock();
invariant(_state != State::kComplete);
_state = State::kComplete;
}
void DatabasesCloner::_fail_inlock(UniqueLock* lk, Status status) {
LOG(3) << "DatabasesCloner::_fail_inlock called";
if (!_isActive_inlock()) {
return;
}
_setStatus_inlock(status);
// TODO: shutdown outstanding work, like any cloners active
invariant(_finishFn);
auto finish = _finishFn;
_finishFn = {};
lk->unlock();
LOG(3) << "DatabasesCloner - calling _finishFn with status: " << _status;
finish(status);
// Release any resources that might be held by the '_finishFn' (moved to 'finish') function
// object.
finish = OnFinishFn();
lk->lock();
invariant(_state != State::kComplete);
_state = State::kComplete;
}
void DatabasesCloner::_onListDatabaseFinish(
const executor::TaskExecutor::RemoteCommandCallbackArgs& cbd) {
Status respStatus = cbd.response.status;
if (respStatus.isOK()) {
respStatus = getStatusFromCommandResult(cbd.response.data);
}
UniqueLock lk(_mutex);
if (!respStatus.isOK()) {
LOG(1) << "'listDatabases' failed: " << respStatus;
_fail_inlock(&lk, respStatus);
return;
}
// There should not be any cloners yet.
invariant(_databaseCloners.size() == 0);
const auto respBSON = cbd.response.data;
auto databasesArray = _parseListDatabasesResponse(respBSON);
if (!databasesArray.isOK()) {
LOG(1) << "'listDatabases' returned a malformed response: "
<< databasesArray.getStatus().toString();
_fail_inlock(&lk, databasesArray.getStatus());
return;
}
auto dbsArray = databasesArray.getValue();
// Ensure that the 'admin' database is the first element in the array of databases so that it
// will be the first to be cloned. This allows users to authenticate against a database while
// initial sync is occurring.
_setAdminAsFirst(dbsArray);
for (BSONElement arrayElement : dbsArray) {
const BSONObj dbBSON = arrayElement.Obj();
// Check to see if we want to exclude this db from the clone.
if (!_includeDbFn(dbBSON)) {
LOG(1) << "Excluding database from the 'listDatabases' response: " << dbBSON;
continue;
}
if (!dbBSON.hasField("name")) {
LOG(1) << "Excluding database due to the 'listDatabases' response not containing a "
"'name' field for this entry: "
<< dbBSON;
}
const std::string dbName = dbBSON["name"].str();
std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
// filters for DatabasesCloner.
const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
const auto collName = collInfo["name"].str();
const NamespaceString ns(dbName, collName);
if (ns.isSystem() && !ns.isLegalClientSystemNS()) {
LOG(1) << "Skipping 'system' collection: " << ns.ns();
return false;
}
if (!ns.isNormal()) {
LOG(1) << "Skipping non-normal collection: " << ns.ns();
return false;
}
LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
return true;
};
const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
if (status.isOK()) {
LOG(1) << "collection clone finished: " << srcNss;
} else {
error() << "collection clone for '" << srcNss << "' failed due to "
<< status.toString();
}
};
const auto onDbFinish = [this, dbName](const Status& status) {
_onEachDBCloneFinish(status, dbName);
};
Status startStatus = Status::OK();
try {
dbCloner.reset(new DatabaseCloner(
_exec,
_dbWorkThreadPool,
_source,
dbName,
BSONObj(), // do not filter collections out during listCollections call.
collectionFilterPred,
_storage, // use storage provided.
onCollectionFinish,
onDbFinish));
if (_scheduleDbWorkFn) {
dbCloner->setScheduleDbWorkFn_forTest(_scheduleDbWorkFn);
}
if (_startCollectionClonerFn) {
dbCloner->setStartCollectionClonerFn(_startCollectionClonerFn);
}
// Start first database cloner.
if (_databaseCloners.empty()) {
startStatus = dbCloner->startup();
}
} catch (...) {
startStatus = exceptionToStatus();
}
if (!startStatus.isOK()) {
std::string err = str::stream() << "could not create cloner for database: " << dbName
<< " due to: " << startStatus.toString();
_setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
error() << err;
break; // exit for_each loop
}
// add cloner to list.
_databaseCloners.push_back(dbCloner);
}
if (_databaseCloners.size() == 0) {
if (_status.isOK()) {
_succeed_inlock(&lk);
} else {
_fail_inlock(&lk, _status);
}
}
}
void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
Status respStatus = cbd.response.getStatus();
if (respStatus.isOK()) {
respStatus = getStatusFromCommandResult(cbd.response.getValue().data);
}
UniqueLock lk(_mutex);
if (!respStatus.isOK()) {
LOG(1) << "listDatabases failed: " << respStatus;
_setStatus_inlock(respStatus);
_failed_inlock(lk);
return;
}
const auto respBSON = cbd.response.getValue().data;
// There should not be any cloners yet
invariant(_databaseCloners.size() == 0);
const auto dbsElem = respBSON["databases"].Obj();
BSONForEach(arrayElement, dbsElem) {
const BSONObj dbBSON = arrayElement.Obj();
// Check to see if we want to exclude this db from the clone.
if (!_includeDbFn(dbBSON)) {
LOG(1) << "excluding db: " << dbBSON;
continue;
}
const std::string dbName = dbBSON["name"].str();
++_clonersActive;
std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
Status startStatus(ErrorCodes::NotYetInitialized,
"The DatabasesCloner could not be started.");
// filters for DatabasesCloner.
const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
const auto collName = collInfo["name"].str();
const NamespaceString ns(dbName, collName);
if (ns.isSystem() && !legalClientSystemNS(ns.ns(), true)) {
LOG(1) << "Skipping 'system' collection: " << ns.ns();
return false;
}
if (!ns.isNormal()) {
LOG(1) << "Skipping non-normal collection: " << ns.ns();
return false;
}
LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
return true;
};
const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
if (status.isOK()) {
LOG(1) << "collection clone finished: " << srcNss;
} else {
warning() << "collection clone for '" << srcNss << "' failed due to "
<< status.toString();
}
};
const auto onDbFinish = [this, dbName](const Status& status) {
_onEachDBCloneFinish(status, dbName);
};
try {
dbCloner.reset(new DatabaseCloner(
_exec,
_source,
dbName,
BSONObj(), // do not filter collections out during listCollections call.
collectionFilterPred,
_storage, // use storage provided.
onCollectionFinish,
onDbFinish));
// Start database cloner.
startStatus = dbCloner->start();
} catch (...) {
startStatus = exceptionToStatus();
}
if (!startStatus.isOK()) {
std::string err = str::stream() << "could not create cloner for database: " << dbName
<< " due to: " << startStatus.toString();
_setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
error() << err;
break; // exit for_each loop
}
// add cloner to list.
_databaseCloners.push_back(dbCloner);
}
if (_databaseCloners.size() == 0) {
if (_status.isOK()) {
_active = false;
lk.unlock();
_finishFn(_status);
} else {
_failed_inlock(lk);
}
}
}
