예제 #1
0
void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
    const Status respStatus = cbd.response.getStatus();
    if (!respStatus.isOK()) {
        // TODO: retry internally?
        _setStatus(respStatus);
        _doNextActions();
        return;
    }

    const auto respBSON = cbd.response.getValue().data;

    // There should not be any cloners yet
    invariant(_databaseCloners.size() == 0);

    const auto okElem = respBSON["ok"];
    if (okElem.trueValue()) {
        const auto dbsElem = respBSON["databases"].Obj();
        BSONForEach(arrayElement, dbsElem) {
            const BSONObj dbBSON = arrayElement.Obj();
            const std::string name = dbBSON["name"].str();
            ++_clonersActive;
            std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
            try {
                dbCloner.reset(new DatabaseCloner(
                    _exec,
                    _source,
                    name,
                    BSONObj(),                            // do not filter database out.
                    [](const BSONObj&) { return true; },  // clone all dbs.
                    _storage,                             // use storage provided.
                    [](const Status& status, const NamespaceString& srcNss) {
                        if (status.isOK()) {
                            log() << "collection clone finished: " << srcNss;
                        } else {
                            log() << "collection clone for '" << srcNss << "' failed due to "
                                  << status.toString();
                        }
                    },
                    [=](const Status& status) { _onEachDBCloneFinish(status, name); }));
            } catch (...) {
                // error creating, fails below.
            }

            Status s = dbCloner ? dbCloner->start() : Status(ErrorCodes::UnknownError, "Bad!");

            if (!s.isOK()) {
                std::string err = str::stream() << "could not create cloner for database: " << name
                                                << " due to: " << s.toString();
                _setStatus(Status(ErrorCodes::InitialSyncFailure, err));
                error() << err;
                break;  // exit for_each loop
            }

            // add cloner to list.
            _databaseCloners.push_back(dbCloner);
        }
    } else {
void DatabasesCloner::_onListDatabaseFinish(
    const executor::TaskExecutor::RemoteCommandCallbackArgs& cbd) {
    Status respStatus = cbd.response.status;
    if (respStatus.isOK()) {
        respStatus = getStatusFromCommandResult(cbd.response.data);
    }

    UniqueLock lk(_mutex);
    if (!respStatus.isOK()) {
        LOG(1) << "'listDatabases' failed: " << respStatus;
        _fail_inlock(&lk, respStatus);
        return;
    }

    // There should not be any cloners yet.
    invariant(_databaseCloners.size() == 0);
    const auto respBSON = cbd.response.data;

    auto databasesArray = _parseListDatabasesResponse(respBSON);
    if (!databasesArray.isOK()) {
        LOG(1) << "'listDatabases' returned a malformed response: "
               << databasesArray.getStatus().toString();
        _fail_inlock(&lk, databasesArray.getStatus());
        return;
    }

    auto dbsArray = databasesArray.getValue();
    // Ensure that the 'admin' database is the first element in the array of databases so that it
    // will be the first to be cloned. This allows users to authenticate against a database while
    // initial sync is occurring.
    _setAdminAsFirst(dbsArray);

    for (BSONElement arrayElement : dbsArray) {
        const BSONObj dbBSON = arrayElement.Obj();

        // Check to see if we want to exclude this db from the clone.
        if (!_includeDbFn(dbBSON)) {
            LOG(1) << "Excluding database from the 'listDatabases' response: " << dbBSON;
            continue;
        }

        if (!dbBSON.hasField("name")) {
            LOG(1) << "Excluding database due to the 'listDatabases' response not containing a "
                      "'name' field for this entry: "
                   << dbBSON;
        }

        const std::string dbName = dbBSON["name"].str();
        std::shared_ptr<DatabaseCloner> dbCloner{nullptr};

        // filters for DatabasesCloner.
        const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
            const auto collName = collInfo["name"].str();
            const NamespaceString ns(dbName, collName);
            if (ns.isSystem() && !ns.isLegalClientSystemNS()) {
                LOG(1) << "Skipping 'system' collection: " << ns.ns();
                return false;
            }
            if (!ns.isNormal()) {
                LOG(1) << "Skipping non-normal collection: " << ns.ns();
                return false;
            }

            LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
            return true;
        };
        const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
            if (status.isOK()) {
                LOG(1) << "collection clone finished: " << srcNss;
            } else {
                error() << "collection clone for '" << srcNss << "' failed due to "
                        << status.toString();
            }
        };
        const auto onDbFinish = [this, dbName](const Status& status) {
            _onEachDBCloneFinish(status, dbName);
        };
        Status startStatus = Status::OK();
        try {
            dbCloner.reset(new DatabaseCloner(
                _exec,
                _dbWorkThreadPool,
                _source,
                dbName,
                BSONObj(),  // do not filter collections out during listCollections call.
                collectionFilterPred,
                _storage,  // use storage provided.
                onCollectionFinish,
                onDbFinish));
            if (_scheduleDbWorkFn) {
                dbCloner->setScheduleDbWorkFn_forTest(_scheduleDbWorkFn);
            }
            if (_startCollectionClonerFn) {
                dbCloner->setStartCollectionClonerFn(_startCollectionClonerFn);
            }
            // Start first database cloner.
            if (_databaseCloners.empty()) {
                startStatus = dbCloner->startup();
            }
        } catch (...) {
            startStatus = exceptionToStatus();
        }

        if (!startStatus.isOK()) {
            std::string err = str::stream() << "could not create cloner for database: " << dbName
                                            << " due to: " << startStatus.toString();
            _setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
            error() << err;
            break;  // exit for_each loop
        }

        // add cloner to list.
        _databaseCloners.push_back(dbCloner);
    }
    if (_databaseCloners.size() == 0) {
        if (_status.isOK()) {
            _succeed_inlock(&lk);
        } else {
            _fail_inlock(&lk, _status);
        }
    }
}
예제 #3
0
void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) {
    Status respStatus = cbd.response.getStatus();
    if (respStatus.isOK()) {
        respStatus = getStatusFromCommandResult(cbd.response.getValue().data);
    }

    UniqueLock lk(_mutex);
    if (!respStatus.isOK()) {
        LOG(1) << "listDatabases failed: " << respStatus;
        _setStatus_inlock(respStatus);
        _failed_inlock(lk);
        return;
    }

    const auto respBSON = cbd.response.getValue().data;
    // There should not be any cloners yet
    invariant(_databaseCloners.size() == 0);
    const auto dbsElem = respBSON["databases"].Obj();
    BSONForEach(arrayElement, dbsElem) {
        const BSONObj dbBSON = arrayElement.Obj();

        // Check to see if we want to exclude this db from the clone.
        if (!_includeDbFn(dbBSON)) {
            LOG(1) << "excluding db: " << dbBSON;
            continue;
        }

        const std::string dbName = dbBSON["name"].str();
        ++_clonersActive;
        std::shared_ptr<DatabaseCloner> dbCloner{nullptr};
        Status startStatus(ErrorCodes::NotYetInitialized,
                           "The DatabasesCloner could not be started.");

        // filters for DatabasesCloner.
        const auto collectionFilterPred = [dbName](const BSONObj& collInfo) {
            const auto collName = collInfo["name"].str();
            const NamespaceString ns(dbName, collName);
            if (ns.isSystem() && !legalClientSystemNS(ns.ns(), true)) {
                LOG(1) << "Skipping 'system' collection: " << ns.ns();
                return false;
            }
            if (!ns.isNormal()) {
                LOG(1) << "Skipping non-normal collection: " << ns.ns();
                return false;
            }

            LOG(2) << "Allowing cloning of collectionInfo: " << collInfo;
            return true;
        };
        const auto onCollectionFinish = [](const Status& status, const NamespaceString& srcNss) {
            if (status.isOK()) {
                LOG(1) << "collection clone finished: " << srcNss;
            } else {
                warning() << "collection clone for '" << srcNss << "' failed due to "
                          << status.toString();
            }
        };
        const auto onDbFinish = [this, dbName](const Status& status) {
            _onEachDBCloneFinish(status, dbName);
        };
        try {
            dbCloner.reset(new DatabaseCloner(
                _exec,
                _source,
                dbName,
                BSONObj(),  // do not filter collections out during listCollections call.
                collectionFilterPred,
                _storage,  // use storage provided.
                onCollectionFinish,
                onDbFinish));
            // Start database cloner.
            startStatus = dbCloner->start();
        } catch (...) {
            startStatus = exceptionToStatus();
        }

        if (!startStatus.isOK()) {
            std::string err = str::stream() << "could not create cloner for database: " << dbName
                                            << " due to: " << startStatus.toString();
            _setStatus_inlock({ErrorCodes::InitialSyncFailure, err});
            error() << err;
            break;  // exit for_each loop
        }

        // add cloner to list.
        _databaseCloners.push_back(dbCloner);
    }

    if (_databaseCloners.size() == 0) {
        if (_status.isOK()) {
            _active = false;
            lk.unlock();
            _finishFn(_status);
        } else {
            _failed_inlock(lk);
        }
    }
}