StatusWith<std::vector<std::string>> ShardingCatalogManager::_getDBNamesListFromShard(
OperationContext* opCtx, std::shared_ptr<RemoteCommandTargeter> targeter) {
auto swCommandResponse =
_runCommandForAddShard(opCtx,
targeter.get(),
NamespaceString::kAdminDb,
BSON("listDatabases" << 1 << "nameOnly" << true));
if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
auto cmdStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!cmdStatus.isOK()) {
return cmdStatus;
}
auto cmdResult = std::move(swCommandResponse.getValue().response);
std::vector<std::string> dbNames;
for (const auto& dbEntry : cmdResult["databases"].Obj()) {
const auto& dbName = dbEntry["name"].String();
if (!(dbName == NamespaceString::kAdminDb || dbName == NamespaceString::kLocalDb ||
dbName == NamespaceString::kConfigDb)) {
dbNames.push_back(dbName);
}
}
return dbNames;
}
StatusWith<std::vector<std::string>> ShardingCatalogManagerImpl::_getDBNamesListFromShard(
OperationContext* txn, std::shared_ptr<RemoteCommandTargeter> targeter) {
auto swCommandResponse =
_runCommandForAddShard(txn, targeter.get(), "admin", BSON("listDatabases" << 1));
if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
auto cmdStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!cmdStatus.isOK()) {
return cmdStatus;
}
auto cmdResult = std::move(swCommandResponse.getValue().response);
vector<string> dbNames;
for (const auto& dbEntry : cmdResult["databases"].Obj()) {
const string& dbName = dbEntry["name"].String();
if (!(dbName == "local" || dbName == "admin")) {
dbNames.push_back(dbName);
}
}
return dbNames;
}
Status ShardingCatalogManager::_dropSessionsCollection(
OperationContext* opCtx, std::shared_ptr<RemoteCommandTargeter> targeter) {
BSONObjBuilder builder;
builder.append("drop", NamespaceString::kLogicalSessionsNamespace.coll());
{
BSONObjBuilder wcBuilder(builder.subobjStart("writeConcern"));
wcBuilder.append("w", "majority");
}
auto swCommandResponse = _runCommandForAddShard(
opCtx, targeter.get(), NamespaceString::kLogicalSessionsNamespace.db(), builder.done());
if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
auto cmdStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!cmdStatus.isOK() && cmdStatus.code() != ErrorCodes::NamespaceNotFound) {
return cmdStatus;
}
return Status::OK();
}
StatusWith<std::string> ShardingCatalogManager::addShard(
OperationContext* opCtx,
const std::string* shardProposedName,
const ConnectionString& shardConnectionString,
const long long maxSize) {
if (shardConnectionString.type() == ConnectionString::INVALID) {
return {ErrorCodes::BadValue, "Invalid connection string"};
}
if (shardProposedName && shardProposedName->empty()) {
return {ErrorCodes::BadValue, "shard name cannot be empty"};
}
// Only one addShard operation can be in progress at a time.
Lock::ExclusiveLock lk(opCtx->lockState(), _kShardMembershipLock);
// Check if this shard has already been added (can happen in the case of a retry after a network
// error, for example) and thus this addShard request should be considered a no-op.
auto existingShard =
_checkIfShardExists(opCtx, shardConnectionString, shardProposedName, maxSize);
if (!existingShard.isOK()) {
return existingShard.getStatus();
}
if (existingShard.getValue()) {
// These hosts already belong to an existing shard, so report success and terminate the
// addShard request. Make sure to set the last optime for the client to the system last
// optime so that we'll still wait for replication so that this state is visible in the
// committed snapshot.
repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx);
return existingShard.getValue()->getName();
}
// Force a reload of the ShardRegistry to ensure that, in case this addShard is to re-add a
// replica set that has recently been removed, we have detached the ReplicaSetMonitor for the
// set with that setName from the ReplicaSetMonitorManager and will create a new
// ReplicaSetMonitor when targeting the set below.
// Note: This is necessary because as of 3.4, removeShard is performed by mongos (unlike
// addShard), so the ShardRegistry is not synchronously reloaded on the config server when a
// shard is removed.
if (!Grid::get(opCtx)->shardRegistry()->reload(opCtx)) {
// If the first reload joined an existing one, call reload again to ensure the reload is
// fresh.
Grid::get(opCtx)->shardRegistry()->reload(opCtx);
}
// TODO: Don't create a detached Shard object, create a detached RemoteCommandTargeter instead.
const std::shared_ptr<Shard> shard{
Grid::get(opCtx)->shardRegistry()->createConnection(shardConnectionString)};
invariant(shard);
auto targeter = shard->getTargeter();
auto stopMonitoringGuard = MakeGuard([&] {
if (shardConnectionString.type() == ConnectionString::SET) {
// This is a workaround for the case were we could have some bad shard being
// requested to be added and we put that bad connection string on the global replica set
// monitor registry. It needs to be cleaned up so that when a correct replica set is
// added, it will be recreated.
ReplicaSetMonitor::remove(shardConnectionString.getSetName());
}
});
// Validate the specified connection string may serve as shard at all
auto shardStatus =
_validateHostAsShard(opCtx, targeter, shardProposedName, shardConnectionString);
if (!shardStatus.isOK()) {
return shardStatus.getStatus();
}
ShardType& shardType = shardStatus.getValue();
// Check that none of the existing shard candidate's dbs exist already
auto dbNamesStatus = _getDBNamesListFromShard(opCtx, targeter);
if (!dbNamesStatus.isOK()) {
return dbNamesStatus.getStatus();
}
for (const auto& dbName : dbNamesStatus.getValue()) {
auto dbt = Grid::get(opCtx)->catalogClient()->getDatabase(
opCtx, dbName, repl::ReadConcernLevel::kLocalReadConcern);
if (dbt.isOK()) {
const auto& dbDoc = dbt.getValue().value;
return Status(ErrorCodes::OperationFailed,
str::stream() << "can't add shard "
<< "'"
<< shardConnectionString.toString()
<< "'"
<< " because a local database '"
<< dbName
<< "' exists in another "
<< dbDoc.getPrimary());
} else if (dbt != ErrorCodes::NamespaceNotFound) {
return dbt.getStatus();
}
}
// Check that the shard candidate does not have a local config.system.sessions collection
auto res = _dropSessionsCollection(opCtx, targeter);
if (!res.isOK()) {
return res.withContext(
"can't add shard with a local copy of config.system.sessions, please drop this "
"collection from the shard manually and try again.");
}
// If a name for a shard wasn't provided, generate one
if (shardType.getName().empty()) {
auto result = generateNewShardName(opCtx);
if (!result.isOK()) {
return result.getStatus();
}
shardType.setName(result.getValue());
}
if (maxSize > 0) {
shardType.setMaxSizeMB(maxSize);
}
// Helper function that runs a command on the to-be shard and returns the status
auto runCmdOnNewShard = [this, &opCtx, &targeter](const BSONObj& cmd) -> Status {
auto swCommandResponse =
_runCommandForAddShard(opCtx, targeter.get(), NamespaceString::kAdminDb, cmd);
if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
// Grabs the underlying status from a StatusWith object by taking the first
// non-OK status, if there is one. This is needed due to the semantics of
// _runCommandForAddShard.
auto commandResponse = std::move(swCommandResponse.getValue());
BatchedCommandResponse batchResponse;
return Shard::CommandResponse::processBatchWriteResponse(commandResponse, &batchResponse);
};
AddShard addShardCmd = add_shard_util::createAddShardCmd(opCtx, shardType.getName());
auto addShardCmdBSON = [&]() {
// In 4.2, use the _addShard command to add the shard, which in turn inserts a
// shardIdentity document into the shard and triggers sharding state initialization.
// In the unlikely scenario that there's a downgrade to 4.0 between the
// construction of this command object and the issuing of the command
// on the receiving shard, the user will receive a rather harmless
// CommandNotFound error for _addShard, and can simply retry.
if (serverGlobalParams.featureCompatibility.getVersion() ==
ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo42) {
// Needed for IDL toBSON method
BSONObj passthroughFields;
return addShardCmd.toBSON(passthroughFields);
} else {
// To support backwards compatibility with v4.0 shards, insert a shardIdentity document
// directly.
return add_shard_util::createShardIdentityUpsertForAddShard(addShardCmd);
}
}();
auto addShardStatus = runCmdOnNewShard(addShardCmdBSON);
if (!addShardStatus.isOK()) {
return addShardStatus;
}
{
// Hold the fcvLock across checking the FCV, sending setFCV to the new shard, and
// writing the entry for the new shard to config.shards. This ensures the FCV doesn't change
// after we send setFCV to the new shard, but before we write its entry to config.shards.
// (Note, we don't use a Global IX lock here, because we don't want to hold the global lock
// while blocking on the network).
invariant(!opCtx->lockState()->isLocked());
Lock::SharedLock lk(opCtx->lockState(), FeatureCompatibilityVersion::fcvLock);
BSONObj setFCVCmd;
switch (serverGlobalParams.featureCompatibility.getVersion()) {
case ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo42:
case ServerGlobalParams::FeatureCompatibility::Version::kUpgradingTo42:
setFCVCmd = BSON(FeatureCompatibilityVersionCommandParser::kCommandName
<< FeatureCompatibilityVersionParser::kVersion42
<< WriteConcernOptions::kWriteConcernField
<< opCtx->getWriteConcern().toBSON());
break;
default:
setFCVCmd = BSON(FeatureCompatibilityVersionCommandParser::kCommandName
<< FeatureCompatibilityVersionParser::kVersion40
<< WriteConcernOptions::kWriteConcernField
<< opCtx->getWriteConcern().toBSON());
break;
}
auto versionResponse =
_runCommandForAddShard(opCtx, targeter.get(), NamespaceString::kAdminDb, setFCVCmd);
if (!versionResponse.isOK()) {
return versionResponse.getStatus();
}
if (!versionResponse.getValue().commandStatus.isOK()) {
return versionResponse.getValue().commandStatus;
}
log() << "going to insert new entry for shard into config.shards: " << shardType.toString();
Status result = Grid::get(opCtx)->catalogClient()->insertConfigDocument(
opCtx,
ShardType::ConfigNS,
shardType.toBSON(),
ShardingCatalogClient::kLocalWriteConcern);
if (!result.isOK()) {
log() << "error adding shard: " << shardType.toBSON() << " err: " << result.reason();
return result;
}
}
// Add all databases which were discovered on the new shard
for (const auto& dbName : dbNamesStatus.getValue()) {
DatabaseType dbt(dbName, shardType.getName(), false, databaseVersion::makeNew());
{
const auto status = Grid::get(opCtx)->catalogClient()->updateConfigDocument(
opCtx,
DatabaseType::ConfigNS,
BSON(DatabaseType::name(dbName)),
dbt.toBSON(),
true,
ShardingCatalogClient::kLocalWriteConcern);
if (!status.isOK()) {
log() << "adding shard " << shardConnectionString.toString()
<< " even though could not add database " << dbName;
}
}
}
// Record in changelog
BSONObjBuilder shardDetails;
shardDetails.append("name", shardType.getName());
shardDetails.append("host", shardConnectionString.toString());
Grid::get(opCtx)->catalogClient()->logChange(
opCtx, "addShard", "", shardDetails.obj(), ShardingCatalogClient::kMajorityWriteConcern);
// Ensure the added shard is visible to this process.
auto shardRegistry = Grid::get(opCtx)->shardRegistry();
if (!shardRegistry->getShard(opCtx, shardType.getName()).isOK()) {
return {ErrorCodes::OperationFailed,
"Could not find shard metadata for shard after adding it. This most likely "
"indicates that the shard was removed immediately after it was added."};
}
stopMonitoringGuard.Dismiss();
return shardType.getName();
}
StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard(
OperationContext* opCtx,
std::shared_ptr<RemoteCommandTargeter> targeter,
const std::string* shardProposedName,
const ConnectionString& connectionString) {
auto swCommandResponse = _runCommandForAddShard(
opCtx, targeter.get(), NamespaceString::kAdminDb, BSON("isMaster" << 1));
if (swCommandResponse.getStatus() == ErrorCodes::IncompatibleServerVersion) {
return swCommandResponse.getStatus().withReason(
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard because its binary version is not compatible with "
"the cluster's featureCompatibilityVersion.");
} else if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
// Check for a command response error
auto resIsMasterStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!resIsMasterStatus.isOK()) {
return resIsMasterStatus.withContext(str::stream()
<< "Error running isMaster against "
<< targeter->connectionString().toString());
}
auto resIsMaster = std::move(swCommandResponse.getValue().response);
// Fail if the node being added is a mongos.
const std::string msg = resIsMaster.getStringField("msg");
if (msg == "isdbgrid") {
return {ErrorCodes::IllegalOperation, "cannot add a mongos as a shard"};
}
// Extract the maxWireVersion so we can verify that the node being added has a binary version
// greater than or equal to the cluster's featureCompatibilityVersion. We expect an incompatible
// binary node to be unable to communicate, returning an IncompatibleServerVersion error,
// because of our internal wire version protocol. So we can safely invariant here that the node
// is compatible.
long long maxWireVersion;
Status status = bsonExtractIntegerField(resIsMaster, "maxWireVersion", &maxWireVersion);
if (!status.isOK()) {
return status.withContext(str::stream() << "isMaster returned invalid 'maxWireVersion' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard");
}
if (serverGlobalParams.featureCompatibility.getVersion() >
ServerGlobalParams::FeatureCompatibility::Version::kFullyDowngradedTo40) {
// If the cluster's FCV is 4.2, or upgrading to / downgrading from, the node being added
// must be a v4.2 binary.
invariant(maxWireVersion == WireVersion::LATEST_WIRE_VERSION);
} else {
// If the cluster's FCV is 4.0, the node being added must be a v4.0 or v4.2 binary.
invariant(serverGlobalParams.featureCompatibility.getVersion() ==
ServerGlobalParams::FeatureCompatibility::Version::kFullyDowngradedTo40);
invariant(maxWireVersion >= WireVersion::LATEST_WIRE_VERSION - 1);
}
// Check whether there is a master. If there isn't, the replica set may not have been
// initiated. If the connection is a standalone, it will return true for isMaster.
bool isMaster;
status = bsonExtractBooleanField(resIsMaster, "ismaster", &isMaster);
if (!status.isOK()) {
return status.withContext(str::stream() << "isMaster returned invalid 'ismaster' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard");
}
if (!isMaster) {
return {ErrorCodes::NotMaster,
str::stream()
<< connectionString.toString()
<< " does not have a master. If this is a replica set, ensure that it has a"
<< " healthy primary and that the set has been properly initiated."};
}
const std::string providedSetName = connectionString.getSetName();
const std::string foundSetName = resIsMaster["setName"].str();
// Make sure the specified replica set name (if any) matches the actual shard's replica set
if (providedSetName.empty() && !foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host is part of set " << foundSetName << "; "
<< "use replica set url format "
<< "<setname>/<server1>,<server2>, ..."};
}
if (!providedSetName.empty() && foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host did not return a set name; "
<< "is the replica set still initializing? "
<< resIsMaster};
}
// Make sure the set name specified in the connection string matches the one where its hosts
// belong into
if (!providedSetName.empty() && (providedSetName != foundSetName)) {
return {ErrorCodes::OperationFailed,
str::stream() << "the provided connection string (" << connectionString.toString()
<< ") does not match the actual set name "
<< foundSetName};
}
// Is it a config server?
if (resIsMaster.hasField("configsvr")) {
return {ErrorCodes::OperationFailed,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard since it is a config server"};
}
// If the shard is part of a replica set, make sure all the hosts mentioned in the connection
// string are part of the set. It is fine if not all members of the set are mentioned in the
// connection string, though.
if (!providedSetName.empty()) {
std::set<std::string> hostSet;
BSONObjIterator iter(resIsMaster["hosts"].Obj());
while (iter.more()) {
hostSet.insert(iter.next().String()); // host:port
}
if (resIsMaster["passives"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["passives"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
if (resIsMaster["arbiters"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["arbiters"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
for (const auto& hostEntry : connectionString.getServers()) {
const auto& host = hostEntry.toString(); // host:port
if (hostSet.find(host) == hostSet.end()) {
return {ErrorCodes::OperationFailed,
str::stream() << "in seed list " << connectionString.toString() << ", host "
<< host
<< " does not belong to replica set "
<< foundSetName
<< "; found "
<< resIsMaster.toString()};
}
}
}
std::string actualShardName;
if (shardProposedName) {
actualShardName = *shardProposedName;
} else if (!foundSetName.empty()) {
// Default it to the name of the replica set
actualShardName = foundSetName;
}
// Disallow adding shard replica set with name 'config'
if (actualShardName == NamespaceString::kConfigDb) {
return {ErrorCodes::BadValue, "use of shard replica set with name 'config' is not allowed"};
}
// Retrieve the most up to date connection string that we know from the replica set monitor (if
// this is a replica set shard, otherwise it will be the same value as connectionString).
ConnectionString actualShardConnStr = targeter->connectionString();
ShardType shard;
shard.setName(actualShardName);
shard.setHost(actualShardConnStr.toString());
shard.setState(ShardType::ShardState::kShardAware);
return shard;
}
StatusWith<string> ShardingCatalogManagerImpl::addShard(
OperationContext* txn,
const std::string* shardProposedName,
const ConnectionString& shardConnectionString,
const long long maxSize) {
if (shardConnectionString.type() == ConnectionString::INVALID) {
return {ErrorCodes::BadValue, "Invalid connection string"};
}
if (shardProposedName && shardProposedName->empty()) {
return {ErrorCodes::BadValue, "shard name cannot be empty"};
}
// TODO: Don't create a detached Shard object, create a detached RemoteCommandTargeter instead.
const std::shared_ptr<Shard> shard{
Grid::get(txn)->shardRegistry()->createConnection(shardConnectionString)};
invariant(shard);
auto targeter = shard->getTargeter();
// Validate the specified connection string may serve as shard at all
auto shardStatus =
_validateHostAsShard(txn, targeter, shardProposedName, shardConnectionString);
if (!shardStatus.isOK()) {
// TODO: This is a workaround for the case were we could have some bad shard being
// requested to be added and we put that bad connection string on the global replica set
// monitor registry. It needs to be cleaned up so that when a correct replica set is added,
// it will be recreated.
ReplicaSetMonitor::remove(shardConnectionString.getSetName());
return shardStatus.getStatus();
}
ShardType& shardType = shardStatus.getValue();
auto dbNamesStatus = _getDBNamesListFromShard(txn, targeter);
if (!dbNamesStatus.isOK()) {
return dbNamesStatus.getStatus();
}
// Check that none of the existing shard candidate's dbs exist already
for (const string& dbName : dbNamesStatus.getValue()) {
auto dbt = _catalogClient->getDatabase(txn, dbName);
if (dbt.isOK()) {
const auto& dbDoc = dbt.getValue().value;
return Status(ErrorCodes::OperationFailed,
str::stream() << "can't add shard "
<< "'"
<< shardConnectionString.toString()
<< "'"
<< " because a local database '"
<< dbName
<< "' exists in another "
<< dbDoc.getPrimary());
} else if (dbt != ErrorCodes::NamespaceNotFound) {
return dbt.getStatus();
}
}
// If a name for a shard wasn't provided, generate one
if (shardType.getName().empty()) {
StatusWith<string> result = _generateNewShardName(txn);
if (!result.isOK()) {
return result.getStatus();
}
shardType.setName(result.getValue());
}
if (maxSize > 0) {
shardType.setMaxSizeMB(maxSize);
}
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
Grid::get(txn)->shardRegistry()->getConfigServerConnectionString());
shardIdentity.setShardName(shardType.getName());
shardIdentity.setClusterId(Grid::get(txn)->shardRegistry()->getClusterId());
auto validateStatus = shardIdentity.validate();
if (!validateStatus.isOK()) {
return validateStatus;
}
log() << "going to insert shardIdentity document into shard: " << shardIdentity.toString();
auto updateRequest = shardIdentity.createUpsertForAddShard();
BatchedCommandRequest commandRequest(updateRequest.release());
commandRequest.setNS(NamespaceString::kConfigCollectionNamespace);
commandRequest.setWriteConcern(kMajorityWriteConcern.toBSON());
auto swCommandResponse =
_runCommandForAddShard(txn, targeter.get(), "admin", commandRequest.toBSON());
if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
auto commandResponse = std::move(swCommandResponse.getValue());
BatchedCommandResponse batchResponse;
auto batchResponseStatus =
Shard::CommandResponse::processBatchWriteResponse(commandResponse, &batchResponse);
if (!batchResponseStatus.isOK()) {
return batchResponseStatus;
}
log() << "going to insert new entry for shard into config.shards: " << shardType.toString();
Status result =
_catalogClient->insertConfigDocument(txn, ShardType::ConfigNS, shardType.toBSON());
if (!result.isOK()) {
log() << "error adding shard: " << shardType.toBSON() << " err: " << result.reason();
if (result == ErrorCodes::DuplicateKey) {
// TODO(SERVER-24213): adding a shard that already exists should be considered success,
// however this approach does no validation that we are adding the shard with the same
// options. It also does not protect against adding the same shard with a different
// shard name and slightly different connection string. This is a temporary hack to
// get the continuous stepdown suite passing.
warning() << "Received duplicate key error when inserting new shard with name "
<< shardType.getName() << " and connection string "
<< shardConnectionString.toString()
<< " to config.shards collection. This most likely means that there was an "
"attempt to add a shard that already exists in the cluster";
return shardType.getName();
}
return result;
}
// Add all databases which were discovered on the new shard
for (const string& dbName : dbNamesStatus.getValue()) {
DatabaseType dbt;
dbt.setName(dbName);
dbt.setPrimary(shardType.getName());
dbt.setSharded(false);
Status status = _catalogClient->updateDatabase(txn, dbName, dbt);
if (!status.isOK()) {
log() << "adding shard " << shardConnectionString.toString()
<< " even though could not add database " << dbName;
}
}
// Record in changelog
BSONObjBuilder shardDetails;
shardDetails.append("name", shardType.getName());
shardDetails.append("host", shardConnectionString.toString());
_catalogClient->logChange(txn, "addShard", "", shardDetails.obj());
// Ensure the added shard is visible to this process.
auto shardRegistry = Grid::get(txn)->shardRegistry();
if (!shardRegistry->getShard(txn, shardType.getName())) {
return {ErrorCodes::OperationFailed,
"Could not find shard metadata for shard after adding it. This most likely "
"indicates that the shard was removed immediately after it was added."};
}
return shardType.getName();
}
StatusWith<ShardType> ShardingCatalogManagerImpl::_validateHostAsShard(
OperationContext* txn,
std::shared_ptr<RemoteCommandTargeter> targeter,
const std::string* shardProposedName,
const ConnectionString& connectionString) {
// Check whether any host in the connection is already part of the cluster.
Grid::get(txn)->shardRegistry()->reload(txn);
for (const auto& hostAndPort : connectionString.getServers()) {
std::shared_ptr<Shard> shard;
shard = Grid::get(txn)->shardRegistry()->getShardNoReload(hostAndPort.toString());
if (shard) {
return {ErrorCodes::OperationFailed,
str::stream() << "'" << hostAndPort.toString() << "' "
<< "is already a member of the existing shard '"
<< shard->getConnString().toString()
<< "' ("
<< shard->getId()
<< ")."};
}
}
// Check for mongos and older version mongod connections, and whether the hosts
// can be found for the user specified replset.
auto swCommandResponse =
_runCommandForAddShard(txn, targeter.get(), "admin", BSON("isMaster" << 1));
if (!swCommandResponse.isOK()) {
if (swCommandResponse.getStatus() == ErrorCodes::RPCProtocolNegotiationFailed) {
// Mongos to mongos commands are no longer supported in the wire protocol
// (because mongos does not support OP_COMMAND), similarly for a new mongos
// and an old mongod. So the call will fail in such cases.
// TODO: If/When mongos ever supports opCommands, this logic will break because
// cmdStatus will be OK.
return {ErrorCodes::RPCProtocolNegotiationFailed,
str::stream() << targeter->connectionString().toString()
<< " does not recognize the RPC protocol being used. This is"
<< " likely because it contains a node that is a mongos or an old"
<< " version of mongod."};
} else {
return swCommandResponse.getStatus();
}
}
// Check for a command response error
auto resIsMasterStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!resIsMasterStatus.isOK()) {
return {resIsMasterStatus.code(),
str::stream() << "Error running isMaster against "
<< targeter->connectionString().toString()
<< ": "
<< causedBy(resIsMasterStatus)};
}
auto resIsMaster = std::move(swCommandResponse.getValue().response);
// Check whether there is a master. If there isn't, the replica set may not have been
// initiated. If the connection is a standalone, it will return true for isMaster.
bool isMaster;
Status status = bsonExtractBooleanField(resIsMaster, "ismaster", &isMaster);
if (!status.isOK()) {
return Status(status.code(),
str::stream() << "isMaster returned invalid 'ismaster' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard: "
<< status.reason());
}
if (!isMaster) {
return {ErrorCodes::NotMaster,
str::stream()
<< connectionString.toString()
<< " does not have a master. If this is a replica set, ensure that it has a"
<< " healthy primary and that the set has been properly initiated."};
}
const string providedSetName = connectionString.getSetName();
const string foundSetName = resIsMaster["setName"].str();
// Make sure the specified replica set name (if any) matches the actual shard's replica set
if (providedSetName.empty() && !foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host is part of set " << foundSetName << "; "
<< "use replica set url format "
<< "<setname>/<server1>,<server2>, ..."};
}
if (!providedSetName.empty() && foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host did not return a set name; "
<< "is the replica set still initializing? "
<< resIsMaster};
}
// Make sure the set name specified in the connection string matches the one where its hosts
// belong into
if (!providedSetName.empty() && (providedSetName != foundSetName)) {
return {ErrorCodes::OperationFailed,
str::stream() << "the provided connection string (" << connectionString.toString()
<< ") does not match the actual set name "
<< foundSetName};
}
// Is it a config server?
if (resIsMaster.hasField("configsvr")) {
return {ErrorCodes::OperationFailed,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard since it is a config server"};
}
// If the shard is part of a replica set, make sure all the hosts mentioned in the connection
// string are part of the set. It is fine if not all members of the set are mentioned in the
// connection string, though.
if (!providedSetName.empty()) {
std::set<string> hostSet;
BSONObjIterator iter(resIsMaster["hosts"].Obj());
while (iter.more()) {
hostSet.insert(iter.next().String()); // host:port
}
if (resIsMaster["passives"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["passives"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
if (resIsMaster["arbiters"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["arbiters"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
vector<HostAndPort> hosts = connectionString.getServers();
for (size_t i = 0; i < hosts.size(); i++) {
const string host = hosts[i].toString(); // host:port
if (hostSet.find(host) == hostSet.end()) {
return {ErrorCodes::OperationFailed,
str::stream() << "in seed list " << connectionString.toString() << ", host "
<< host
<< " does not belong to replica set "
<< foundSetName
<< "; found "
<< resIsMaster.toString()};
}
}
}
string actualShardName;
if (shardProposedName) {
actualShardName = *shardProposedName;
} else if (!foundSetName.empty()) {
// Default it to the name of the replica set
actualShardName = foundSetName;
}
// Disallow adding shard replica set with name 'config'
if (actualShardName == "config") {
return {ErrorCodes::BadValue, "use of shard replica set with name 'config' is not allowed"};
}
// Retrieve the most up to date connection string that we know from the replica set monitor (if
// this is a replica set shard, otherwise it will be the same value as connectionString).
ConnectionString actualShardConnStr = targeter->connectionString();
ShardType shard;
shard.setName(actualShardName);
shard.setHost(actualShardConnStr.toString());
return shard;
}
StatusWith<std::string> ShardingCatalogManagerImpl::addShard(
OperationContext* opCtx,
const std::string* shardProposedName,
const ConnectionString& shardConnectionString,
const long long maxSize) {
if (shardConnectionString.type() == ConnectionString::INVALID) {
return {ErrorCodes::BadValue, "Invalid connection string"};
}
if (shardProposedName && shardProposedName->empty()) {
return {ErrorCodes::BadValue, "shard name cannot be empty"};
}
// Only one addShard operation can be in progress at a time.
Lock::ExclusiveLock lk(opCtx->lockState(), _kShardMembershipLock);
// Check if this shard has already been added (can happen in the case of a retry after a network
// error, for example) and thus this addShard request should be considered a no-op.
auto existingShard =
_checkIfShardExists(opCtx, shardConnectionString, shardProposedName, maxSize);
if (!existingShard.isOK()) {
return existingShard.getStatus();
}
if (existingShard.getValue()) {
// These hosts already belong to an existing shard, so report success and terminate the
// addShard request. Make sure to set the last optime for the client to the system last
// optime so that we'll still wait for replication so that this state is visible in the
// committed snapshot.
repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx);
return existingShard.getValue()->getName();
}
// Force a reload of the ShardRegistry to ensure that, in case this addShard is to re-add a
// replica set that has recently been removed, we have detached the ReplicaSetMonitor for the
// set with that setName from the ReplicaSetMonitorManager and will create a new
// ReplicaSetMonitor when targeting the set below.
// Note: This is necessary because as of 3.4, removeShard is performed by mongos (unlike
// addShard), so the ShardRegistry is not synchronously reloaded on the config server when a
// shard is removed.
if (!Grid::get(opCtx)->shardRegistry()->reload(opCtx)) {
// If the first reload joined an existing one, call reload again to ensure the reload is
// fresh.
Grid::get(opCtx)->shardRegistry()->reload(opCtx);
}
// TODO: Don't create a detached Shard object, create a detached RemoteCommandTargeter instead.
const std::shared_ptr<Shard> shard{
Grid::get(opCtx)->shardRegistry()->createConnection(shardConnectionString)};
invariant(shard);
auto targeter = shard->getTargeter();
auto stopMonitoringGuard = MakeGuard([&] {
if (shardConnectionString.type() == ConnectionString::SET) {
// This is a workaround for the case were we could have some bad shard being
// requested to be added and we put that bad connection string on the global replica set
// monitor registry. It needs to be cleaned up so that when a correct replica set is
// added, it will be recreated.
ReplicaSetMonitor::remove(shardConnectionString.getSetName());
}
});
// Validate the specified connection string may serve as shard at all
auto shardStatus =
_validateHostAsShard(opCtx, targeter, shardProposedName, shardConnectionString);
if (!shardStatus.isOK()) {
return shardStatus.getStatus();
}
ShardType& shardType = shardStatus.getValue();
// Check that none of the existing shard candidate's dbs exist already
auto dbNamesStatus = _getDBNamesListFromShard(opCtx, targeter);
if (!dbNamesStatus.isOK()) {
return dbNamesStatus.getStatus();
}
for (const auto& dbName : dbNamesStatus.getValue()) {
auto dbt = Grid::get(opCtx)->catalogClient(opCtx)->getDatabase(opCtx, dbName);
if (dbt.isOK()) {
const auto& dbDoc = dbt.getValue().value;
return Status(ErrorCodes::OperationFailed,
str::stream() << "can't add shard "
<< "'"
<< shardConnectionString.toString()
<< "'"
<< " because a local database '"
<< dbName
<< "' exists in another "
<< dbDoc.getPrimary());
} else if (dbt != ErrorCodes::NamespaceNotFound) {
return dbt.getStatus();
}
}
// If a name for a shard wasn't provided, generate one
if (shardType.getName().empty()) {
auto result = generateNewShardName(opCtx);
if (!result.isOK()) {
return result.getStatus();
}
shardType.setName(result.getValue());
}
if (maxSize > 0) {
shardType.setMaxSizeMB(maxSize);
}
// If the minimum allowed version for the cluster is 3.4, set the featureCompatibilityVersion to
// 3.4 on the shard.
if (serverGlobalParams.featureCompatibility.version.load() ==
ServerGlobalParams::FeatureCompatibility::Version::k34) {
auto versionResponse =
_runCommandForAddShard(opCtx,
targeter.get(),
"admin",
BSON(FeatureCompatibilityVersion::kCommandName
<< FeatureCompatibilityVersionCommandParser::kVersion34));
if (!versionResponse.isOK()) {
return versionResponse.getStatus();
}
if (!versionResponse.getValue().commandStatus.isOK()) {
if (versionResponse.getStatus().code() == ErrorCodes::CommandNotFound) {
return {ErrorCodes::OperationFailed,
"featureCompatibilityVersion for cluster is 3.4, cannot add a shard with "
"version below 3.4. See "
"http://dochub.mongodb.org/core/3.4-feature-compatibility."};
}
return versionResponse.getValue().commandStatus;
}
}
if (!MONGO_FAIL_POINT(dontUpsertShardIdentityOnNewShards)) {
auto commandRequest = createShardIdentityUpsertForAddShard(opCtx, shardType.getName());
LOG(2) << "going to insert shardIdentity document into shard: " << shardType;
auto swCommandResponse =
_runCommandForAddShard(opCtx, targeter.get(), "admin", commandRequest);
if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
auto commandResponse = std::move(swCommandResponse.getValue());
BatchedCommandResponse batchResponse;
auto batchResponseStatus =
Shard::CommandResponse::processBatchWriteResponse(commandResponse, &batchResponse);
if (!batchResponseStatus.isOK()) {
return batchResponseStatus;
}
}
log() << "going to insert new entry for shard into config.shards: " << shardType.toString();
Status result = Grid::get(opCtx)->catalogClient(opCtx)->insertConfigDocument(
opCtx,
ShardType::ConfigNS,
shardType.toBSON(),
ShardingCatalogClient::kMajorityWriteConcern);
if (!result.isOK()) {
log() << "error adding shard: " << shardType.toBSON() << " err: " << result.reason();
return result;
}
// Add all databases which were discovered on the new shard
for (const auto& dbName : dbNamesStatus.getValue()) {
DatabaseType dbt;
dbt.setName(dbName);
dbt.setPrimary(shardType.getName());
dbt.setSharded(false);
Status status = Grid::get(opCtx)->catalogClient(opCtx)->updateDatabase(opCtx, dbName, dbt);
if (!status.isOK()) {
log() << "adding shard " << shardConnectionString.toString()
<< " even though could not add database " << dbName;
}
}
// Record in changelog
BSONObjBuilder shardDetails;
shardDetails.append("name", shardType.getName());
shardDetails.append("host", shardConnectionString.toString());
Grid::get(opCtx)->catalogClient(opCtx)->logChange(
opCtx, "addShard", "", shardDetails.obj(), ShardingCatalogClient::kMajorityWriteConcern);
// Ensure the added shard is visible to this process.
auto shardRegistry = Grid::get(opCtx)->shardRegistry();
if (!shardRegistry->getShard(opCtx, shardType.getName()).isOK()) {
return {ErrorCodes::OperationFailed,
"Could not find shard metadata for shard after adding it. This most likely "
"indicates that the shard was removed immediately after it was added."};
}
stopMonitoringGuard.Dismiss();
return shardType.getName();
}
StatusWith<ShardType> ShardingCatalogManagerImpl::_validateHostAsShard(
OperationContext* opCtx,
std::shared_ptr<RemoteCommandTargeter> targeter,
const std::string* shardProposedName,
const ConnectionString& connectionString) {
// Check if the node being added is a mongos or a version of mongod too old to speak the current
// communication protocol.
auto swCommandResponse =
_runCommandForAddShard(opCtx, targeter.get(), "admin", BSON("isMaster" << 1));
if (!swCommandResponse.isOK()) {
if (swCommandResponse.getStatus() == ErrorCodes::RPCProtocolNegotiationFailed) {
// Mongos to mongos commands are no longer supported in the wire protocol
// (because mongos does not support OP_COMMAND), similarly for a new mongos
// and an old mongod. So the call will fail in such cases.
// TODO: If/When mongos ever supports opCommands, this logic will break because
// cmdStatus will be OK.
return {ErrorCodes::RPCProtocolNegotiationFailed,
str::stream() << targeter->connectionString().toString()
<< " does not recognize the RPC protocol being used. This is"
<< " likely because it contains a node that is a mongos or an old"
<< " version of mongod."};
} else {
return swCommandResponse.getStatus();
}
}
// Check for a command response error
auto resIsMasterStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!resIsMasterStatus.isOK()) {
return {resIsMasterStatus.code(),
str::stream() << "Error running isMaster against "
<< targeter->connectionString().toString()
<< ": "
<< causedBy(resIsMasterStatus)};
}
auto resIsMaster = std::move(swCommandResponse.getValue().response);
// Check that the node being added is a new enough version.
// If we're running this code, that means the mongos that the addShard request originated from
// must be at least version 3.4 (since 3.2 mongoses don't know about the _configsvrAddShard
// command). Since it is illegal to have v3.4 mongoses with v3.2 shards, we should reject
// adding any shards that are not v3.4. We can determine this by checking that the
// maxWireVersion reported in isMaster is at least COMMANDS_ACCEPT_WRITE_CONCERN.
// TODO(SERVER-25623): This approach won't work to prevent v3.6 mongoses from adding v3.4
// shards, so we'll have to rethink this during the 3.5 development cycle.
long long maxWireVersion;
Status status = bsonExtractIntegerField(resIsMaster, "maxWireVersion", &maxWireVersion);
if (!status.isOK()) {
return Status(status.code(),
str::stream() << "isMaster returned invalid 'maxWireVersion' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard: "
<< status.reason());
}
if (maxWireVersion < WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN) {
return Status(ErrorCodes::IncompatibleServerVersion,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard because we detected a mongod with server "
"version older than 3.4.0. It is invalid to add v3.2 and "
"older shards through a v3.4 mongos.");
}
// Check whether there is a master. If there isn't, the replica set may not have been
// initiated. If the connection is a standalone, it will return true for isMaster.
bool isMaster;
status = bsonExtractBooleanField(resIsMaster, "ismaster", &isMaster);
if (!status.isOK()) {
return Status(status.code(),
str::stream() << "isMaster returned invalid 'ismaster' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard: "
<< status.reason());
}
if (!isMaster) {
return {ErrorCodes::NotMaster,
str::stream()
<< connectionString.toString()
<< " does not have a master. If this is a replica set, ensure that it has a"
<< " healthy primary and that the set has been properly initiated."};
}
const std::string providedSetName = connectionString.getSetName();
const std::string foundSetName = resIsMaster["setName"].str();
// Make sure the specified replica set name (if any) matches the actual shard's replica set
if (providedSetName.empty() && !foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host is part of set " << foundSetName << "; "
<< "use replica set url format "
<< "<setname>/<server1>,<server2>, ..."};
}
if (!providedSetName.empty() && foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host did not return a set name; "
<< "is the replica set still initializing? "
<< resIsMaster};
}
// Make sure the set name specified in the connection string matches the one where its hosts
// belong into
if (!providedSetName.empty() && (providedSetName != foundSetName)) {
return {ErrorCodes::OperationFailed,
str::stream() << "the provided connection string (" << connectionString.toString()
<< ") does not match the actual set name "
<< foundSetName};
}
// Is it a config server?
if (resIsMaster.hasField("configsvr")) {
return {ErrorCodes::OperationFailed,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard since it is a config server"};
}
// If the shard is part of a replica set, make sure all the hosts mentioned in the connection
// string are part of the set. It is fine if not all members of the set are mentioned in the
// connection string, though.
if (!providedSetName.empty()) {
std::set<std::string> hostSet;
BSONObjIterator iter(resIsMaster["hosts"].Obj());
while (iter.more()) {
hostSet.insert(iter.next().String()); // host:port
}
if (resIsMaster["passives"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["passives"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
if (resIsMaster["arbiters"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["arbiters"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
for (const auto& hostEntry : connectionString.getServers()) {
const auto& host = hostEntry.toString(); // host:port
if (hostSet.find(host) == hostSet.end()) {
return {ErrorCodes::OperationFailed,
str::stream() << "in seed list " << connectionString.toString() << ", host "
<< host
<< " does not belong to replica set "
<< foundSetName
<< "; found "
<< resIsMaster.toString()};
}
}
}
std::string actualShardName;
if (shardProposedName) {
actualShardName = *shardProposedName;
} else if (!foundSetName.empty()) {
// Default it to the name of the replica set
actualShardName = foundSetName;
}
// Disallow adding shard replica set with name 'config'
if (actualShardName == NamespaceString::kConfigDb) {
return {ErrorCodes::BadValue, "use of shard replica set with name 'config' is not allowed"};
}
// Retrieve the most up to date connection string that we know from the replica set monitor (if
// this is a replica set shard, otherwise it will be the same value as connectionString).
ConnectionString actualShardConnStr = targeter->connectionString();
ShardType shard;
shard.setName(actualShardName);
shard.setHost(actualShardConnStr.toString());
shard.setState(ShardType::ShardState::kShardAware);
return shard;
}
