void ShardRegistry::reload() {
vector<ShardType> shards;
Status status = _catalogManager->getAllShards(&shards);
massert(13632, "couldn't get updated shard list from config server", status.isOK());
int numShards = shards.size();
LOG(1) << "found " << numShards << " shards listed on config server(s)";
boost::lock_guard<boost::mutex> lk(_mutex);
_lookup.clear();
_rsLookup.clear();
ShardType configServerShard;
configServerShard.setName("config");
configServerShard.setHost(_catalogManager->connectionString().toString());
_addShard_inlock(configServerShard);
for (const ShardType& shardType : shards) {
uassertStatusOK(shardType.validate());
// Skip the config host even if there is one left over from legacy installations. The
// config host is installed manually from the catalog manager data.
if (shardType.getName() == "config") {
continue;
}
_addShard_inlock(shardType);
}
}
/**
* Stores sample shard and ping information at the current version.
*/
void storeShardsAndPings(int numShards, int numPings) {
DBDirectClient client(&_txn);
for (int i = 0; i < numShards; i++) {
ShardType shard;
shard.setName(OID::gen().toString());
shard.setHost((string) (str::stream() << "$dummyShard:" << (i + 1) << "0000"));
client.insert(ShardType::ConfigNS, shard.toBSON());
}
for (int i = 0; i < numPings; i++) {
MongosType ping;
ping.setName((string) (str::stream() << "$dummyMongos:" << (i + 1) << "0000"));
ping.setPing(jsTime());
ping.setMongoVersion(versionString);
ping.setConfigVersion(CURRENT_CONFIG_VERSION);
if (i % 2 == 0) {
ping.setPing(ping.getPing() - Minutes(10));
}
client.insert(MongosType::ConfigNS, ping.toBSON());
}
}
TEST_F(EnableShardingTest, succeedsWhenTheDatabaseIsAlreadySharded) {
ShardType shard;
shard.setName("shard0");
shard.setHost("shard0:12");
ASSERT_OK(setupShards(vector<ShardType>{shard}));
setupDatabase("db5", shard.getName(), true);
auto status =
ShardingCatalogManager::get(operationContext())->enableSharding(operationContext(), "db5");
ASSERT_OK(status);
}
/**
* Stores a legacy { version : X } config server entry
*/
void storeLegacyConfigVersion(int version) {
if (version == 0) return;
if (version == 1) {
ShardType shard;
shard.setName("test");
shard.setHost("$dummy:10000");
client().insert(ShardType::ConfigNS, shard.toBSON());
return;
}
client().insert(VersionType::ConfigNS, BSON("_id" << 1 << "version" << version));
}
TEST_F(EnableShardingTest, dbExistsInvalidFormat) {
ShardType shard;
shard.setName("shard0");
shard.setHost("shard0:12");
ASSERT_OK(setupShards(vector<ShardType>{shard}));
// Set up database with bad type for primary field.
ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(),
DatabaseType::ConfigNS,
BSON("_id"
<< "db6"
<< "primary"
<< 12
<< "partitioned"
<< false),
ShardingCatalogClient::kMajorityWriteConcern));
auto status =
ShardingCatalogManager::get(operationContext())->enableSharding(operationContext(), "db6");
ASSERT_EQ(ErrorCodes::TypeMismatch, status.code());
}
void ShardRegistry::_addConfigShard_inlock() {
ShardType configServerShard;
configServerShard.setName("config");
configServerShard.setHost(_catalogManager->connectionString().toString());
_addShard_inlock(configServerShard);
}
StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard(
OperationContext* opCtx,
std::shared_ptr<RemoteCommandTargeter> targeter,
const std::string* shardProposedName,
const ConnectionString& connectionString) {
auto swCommandResponse = _runCommandForAddShard(
opCtx, targeter.get(), NamespaceString::kAdminDb, BSON("isMaster" << 1));
if (swCommandResponse.getStatus() == ErrorCodes::IncompatibleServerVersion) {
return swCommandResponse.getStatus().withReason(
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard because its binary version is not compatible with "
"the cluster's featureCompatibilityVersion.");
} else if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
// Check for a command response error
auto resIsMasterStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!resIsMasterStatus.isOK()) {
return resIsMasterStatus.withContext(str::stream()
<< "Error running isMaster against "
<< targeter->connectionString().toString());
}
auto resIsMaster = std::move(swCommandResponse.getValue().response);
// Fail if the node being added is a mongos.
const std::string msg = resIsMaster.getStringField("msg");
if (msg == "isdbgrid") {
return {ErrorCodes::IllegalOperation, "cannot add a mongos as a shard"};
}
// Extract the maxWireVersion so we can verify that the node being added has a binary version
// greater than or equal to the cluster's featureCompatibilityVersion. We expect an incompatible
// binary node to be unable to communicate, returning an IncompatibleServerVersion error,
// because of our internal wire version protocol. So we can safely invariant here that the node
// is compatible.
long long maxWireVersion;
Status status = bsonExtractIntegerField(resIsMaster, "maxWireVersion", &maxWireVersion);
if (!status.isOK()) {
return status.withContext(str::stream() << "isMaster returned invalid 'maxWireVersion' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard");
}
if (serverGlobalParams.featureCompatibility.getVersion() >
ServerGlobalParams::FeatureCompatibility::Version::kFullyDowngradedTo40) {
// If the cluster's FCV is 4.2, or upgrading to / downgrading from, the node being added
// must be a v4.2 binary.
invariant(maxWireVersion == WireVersion::LATEST_WIRE_VERSION);
} else {
// If the cluster's FCV is 4.0, the node being added must be a v4.0 or v4.2 binary.
invariant(serverGlobalParams.featureCompatibility.getVersion() ==
ServerGlobalParams::FeatureCompatibility::Version::kFullyDowngradedTo40);
invariant(maxWireVersion >= WireVersion::LATEST_WIRE_VERSION - 1);
}
// Check whether there is a master. If there isn't, the replica set may not have been
// initiated. If the connection is a standalone, it will return true for isMaster.
bool isMaster;
status = bsonExtractBooleanField(resIsMaster, "ismaster", &isMaster);
if (!status.isOK()) {
return status.withContext(str::stream() << "isMaster returned invalid 'ismaster' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard");
}
if (!isMaster) {
return {ErrorCodes::NotMaster,
str::stream()
<< connectionString.toString()
<< " does not have a master. If this is a replica set, ensure that it has a"
<< " healthy primary and that the set has been properly initiated."};
}
const std::string providedSetName = connectionString.getSetName();
const std::string foundSetName = resIsMaster["setName"].str();
// Make sure the specified replica set name (if any) matches the actual shard's replica set
if (providedSetName.empty() && !foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host is part of set " << foundSetName << "; "
<< "use replica set url format "
<< "<setname>/<server1>,<server2>, ..."};
}
if (!providedSetName.empty() && foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host did not return a set name; "
<< "is the replica set still initializing? "
<< resIsMaster};
}
// Make sure the set name specified in the connection string matches the one where its hosts
// belong into
if (!providedSetName.empty() && (providedSetName != foundSetName)) {
return {ErrorCodes::OperationFailed,
str::stream() << "the provided connection string (" << connectionString.toString()
<< ") does not match the actual set name "
<< foundSetName};
}
// Is it a config server?
if (resIsMaster.hasField("configsvr")) {
return {ErrorCodes::OperationFailed,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard since it is a config server"};
}
// If the shard is part of a replica set, make sure all the hosts mentioned in the connection
// string are part of the set. It is fine if not all members of the set are mentioned in the
// connection string, though.
if (!providedSetName.empty()) {
std::set<std::string> hostSet;
BSONObjIterator iter(resIsMaster["hosts"].Obj());
while (iter.more()) {
hostSet.insert(iter.next().String()); // host:port
}
if (resIsMaster["passives"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["passives"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
if (resIsMaster["arbiters"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["arbiters"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
for (const auto& hostEntry : connectionString.getServers()) {
const auto& host = hostEntry.toString(); // host:port
if (hostSet.find(host) == hostSet.end()) {
return {ErrorCodes::OperationFailed,
str::stream() << "in seed list " << connectionString.toString() << ", host "
<< host
<< " does not belong to replica set "
<< foundSetName
<< "; found "
<< resIsMaster.toString()};
}
}
}
std::string actualShardName;
if (shardProposedName) {
actualShardName = *shardProposedName;
} else if (!foundSetName.empty()) {
// Default it to the name of the replica set
actualShardName = foundSetName;
}
// Disallow adding shard replica set with name 'config'
if (actualShardName == NamespaceString::kConfigDb) {
return {ErrorCodes::BadValue, "use of shard replica set with name 'config' is not allowed"};
}
// Retrieve the most up to date connection string that we know from the replica set monitor (if
// this is a replica set shard, otherwise it will be the same value as connectionString).
ConnectionString actualShardConnStr = targeter->connectionString();
ShardType shard;
shard.setName(actualShardName);
shard.setHost(actualShardConnStr.toString());
shard.setState(ShardType::ShardState::kShardAware);
return shard;
}
void ShardRegistry::_addConfigShard_inlock() {
ShardType configServerShard;
configServerShard.setName("config");
configServerShard.setHost(_configServerCS.toString());
_addShard_inlock(configServerShard);
}
StatusWith<ShardType> ShardingCatalogManagerImpl::_validateHostAsShard(
OperationContext* txn,
std::shared_ptr<RemoteCommandTargeter> targeter,
const std::string* shardProposedName,
const ConnectionString& connectionString) {
// Check whether any host in the connection is already part of the cluster.
Grid::get(txn)->shardRegistry()->reload(txn);
for (const auto& hostAndPort : connectionString.getServers()) {
std::shared_ptr<Shard> shard;
shard = Grid::get(txn)->shardRegistry()->getShardNoReload(hostAndPort.toString());
if (shard) {
return {ErrorCodes::OperationFailed,
str::stream() << "'" << hostAndPort.toString() << "' "
<< "is already a member of the existing shard '"
<< shard->getConnString().toString()
<< "' ("
<< shard->getId()
<< ")."};
}
}
// Check for mongos and older version mongod connections, and whether the hosts
// can be found for the user specified replset.
auto swCommandResponse =
_runCommandForAddShard(txn, targeter.get(), "admin", BSON("isMaster" << 1));
if (!swCommandResponse.isOK()) {
if (swCommandResponse.getStatus() == ErrorCodes::RPCProtocolNegotiationFailed) {
// Mongos to mongos commands are no longer supported in the wire protocol
// (because mongos does not support OP_COMMAND), similarly for a new mongos
// and an old mongod. So the call will fail in such cases.
// TODO: If/When mongos ever supports opCommands, this logic will break because
// cmdStatus will be OK.
return {ErrorCodes::RPCProtocolNegotiationFailed,
str::stream() << targeter->connectionString().toString()
<< " does not recognize the RPC protocol being used. This is"
<< " likely because it contains a node that is a mongos or an old"
<< " version of mongod."};
} else {
return swCommandResponse.getStatus();
}
}
// Check for a command response error
auto resIsMasterStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!resIsMasterStatus.isOK()) {
return {resIsMasterStatus.code(),
str::stream() << "Error running isMaster against "
<< targeter->connectionString().toString()
<< ": "
<< causedBy(resIsMasterStatus)};
}
auto resIsMaster = std::move(swCommandResponse.getValue().response);
// Check whether there is a master. If there isn't, the replica set may not have been
// initiated. If the connection is a standalone, it will return true for isMaster.
bool isMaster;
Status status = bsonExtractBooleanField(resIsMaster, "ismaster", &isMaster);
if (!status.isOK()) {
return Status(status.code(),
str::stream() << "isMaster returned invalid 'ismaster' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard: "
<< status.reason());
}
if (!isMaster) {
return {ErrorCodes::NotMaster,
str::stream()
<< connectionString.toString()
<< " does not have a master. If this is a replica set, ensure that it has a"
<< " healthy primary and that the set has been properly initiated."};
}
const string providedSetName = connectionString.getSetName();
const string foundSetName = resIsMaster["setName"].str();
// Make sure the specified replica set name (if any) matches the actual shard's replica set
if (providedSetName.empty() && !foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host is part of set " << foundSetName << "; "
<< "use replica set url format "
<< "<setname>/<server1>,<server2>, ..."};
}
if (!providedSetName.empty() && foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host did not return a set name; "
<< "is the replica set still initializing? "
<< resIsMaster};
}
// Make sure the set name specified in the connection string matches the one where its hosts
// belong into
if (!providedSetName.empty() && (providedSetName != foundSetName)) {
return {ErrorCodes::OperationFailed,
str::stream() << "the provided connection string (" << connectionString.toString()
<< ") does not match the actual set name "
<< foundSetName};
}
// Is it a config server?
if (resIsMaster.hasField("configsvr")) {
return {ErrorCodes::OperationFailed,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard since it is a config server"};
}
// If the shard is part of a replica set, make sure all the hosts mentioned in the connection
// string are part of the set. It is fine if not all members of the set are mentioned in the
// connection string, though.
if (!providedSetName.empty()) {
std::set<string> hostSet;
BSONObjIterator iter(resIsMaster["hosts"].Obj());
while (iter.more()) {
hostSet.insert(iter.next().String()); // host:port
}
if (resIsMaster["passives"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["passives"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
if (resIsMaster["arbiters"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["arbiters"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
vector<HostAndPort> hosts = connectionString.getServers();
for (size_t i = 0; i < hosts.size(); i++) {
const string host = hosts[i].toString(); // host:port
if (hostSet.find(host) == hostSet.end()) {
return {ErrorCodes::OperationFailed,
str::stream() << "in seed list " << connectionString.toString() << ", host "
<< host
<< " does not belong to replica set "
<< foundSetName
<< "; found "
<< resIsMaster.toString()};
}
}
}
string actualShardName;
if (shardProposedName) {
actualShardName = *shardProposedName;
} else if (!foundSetName.empty()) {
// Default it to the name of the replica set
actualShardName = foundSetName;
}
// Disallow adding shard replica set with name 'config'
if (actualShardName == "config") {
return {ErrorCodes::BadValue, "use of shard replica set with name 'config' is not allowed"};
}
// Retrieve the most up to date connection string that we know from the replica set monitor (if
// this is a replica set shard, otherwise it will be the same value as connectionString).
ConnectionString actualShardConnStr = targeter->connectionString();
ShardType shard;
shard.setName(actualShardName);
shard.setHost(actualShardConnStr.toString());
return shard;
}
StatusWith<ShardType> ShardingCatalogManagerImpl::_validateHostAsShard(
OperationContext* opCtx,
std::shared_ptr<RemoteCommandTargeter> targeter,
const std::string* shardProposedName,
const ConnectionString& connectionString) {
// Check if the node being added is a mongos or a version of mongod too old to speak the current
// communication protocol.
auto swCommandResponse =
_runCommandForAddShard(opCtx, targeter.get(), "admin", BSON("isMaster" << 1));
if (!swCommandResponse.isOK()) {
if (swCommandResponse.getStatus() == ErrorCodes::RPCProtocolNegotiationFailed) {
// Mongos to mongos commands are no longer supported in the wire protocol
// (because mongos does not support OP_COMMAND), similarly for a new mongos
// and an old mongod. So the call will fail in such cases.
// TODO: If/When mongos ever supports opCommands, this logic will break because
// cmdStatus will be OK.
return {ErrorCodes::RPCProtocolNegotiationFailed,
str::stream() << targeter->connectionString().toString()
<< " does not recognize the RPC protocol being used. This is"
<< " likely because it contains a node that is a mongos or an old"
<< " version of mongod."};
} else {
return swCommandResponse.getStatus();
}
}
// Check for a command response error
auto resIsMasterStatus = std::move(swCommandResponse.getValue().commandStatus);
if (!resIsMasterStatus.isOK()) {
return {resIsMasterStatus.code(),
str::stream() << "Error running isMaster against "
<< targeter->connectionString().toString()
<< ": "
<< causedBy(resIsMasterStatus)};
}
auto resIsMaster = std::move(swCommandResponse.getValue().response);
// Check that the node being added is a new enough version.
// If we're running this code, that means the mongos that the addShard request originated from
// must be at least version 3.4 (since 3.2 mongoses don't know about the _configsvrAddShard
// command). Since it is illegal to have v3.4 mongoses with v3.2 shards, we should reject
// adding any shards that are not v3.4. We can determine this by checking that the
// maxWireVersion reported in isMaster is at least COMMANDS_ACCEPT_WRITE_CONCERN.
// TODO(SERVER-25623): This approach won't work to prevent v3.6 mongoses from adding v3.4
// shards, so we'll have to rethink this during the 3.5 development cycle.
long long maxWireVersion;
Status status = bsonExtractIntegerField(resIsMaster, "maxWireVersion", &maxWireVersion);
if (!status.isOK()) {
return Status(status.code(),
str::stream() << "isMaster returned invalid 'maxWireVersion' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard: "
<< status.reason());
}
if (maxWireVersion < WireVersion::COMMANDS_ACCEPT_WRITE_CONCERN) {
return Status(ErrorCodes::IncompatibleServerVersion,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard because we detected a mongod with server "
"version older than 3.4.0. It is invalid to add v3.2 and "
"older shards through a v3.4 mongos.");
}
// Check whether there is a master. If there isn't, the replica set may not have been
// initiated. If the connection is a standalone, it will return true for isMaster.
bool isMaster;
status = bsonExtractBooleanField(resIsMaster, "ismaster", &isMaster);
if (!status.isOK()) {
return Status(status.code(),
str::stream() << "isMaster returned invalid 'ismaster' "
<< "field when attempting to add "
<< connectionString.toString()
<< " as a shard: "
<< status.reason());
}
if (!isMaster) {
return {ErrorCodes::NotMaster,
str::stream()
<< connectionString.toString()
<< " does not have a master. If this is a replica set, ensure that it has a"
<< " healthy primary and that the set has been properly initiated."};
}
const std::string providedSetName = connectionString.getSetName();
const std::string foundSetName = resIsMaster["setName"].str();
// Make sure the specified replica set name (if any) matches the actual shard's replica set
if (providedSetName.empty() && !foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host is part of set " << foundSetName << "; "
<< "use replica set url format "
<< "<setname>/<server1>,<server2>, ..."};
}
if (!providedSetName.empty() && foundSetName.empty()) {
return {ErrorCodes::OperationFailed,
str::stream() << "host did not return a set name; "
<< "is the replica set still initializing? "
<< resIsMaster};
}
// Make sure the set name specified in the connection string matches the one where its hosts
// belong into
if (!providedSetName.empty() && (providedSetName != foundSetName)) {
return {ErrorCodes::OperationFailed,
str::stream() << "the provided connection string (" << connectionString.toString()
<< ") does not match the actual set name "
<< foundSetName};
}
// Is it a config server?
if (resIsMaster.hasField("configsvr")) {
return {ErrorCodes::OperationFailed,
str::stream() << "Cannot add " << connectionString.toString()
<< " as a shard since it is a config server"};
}
// If the shard is part of a replica set, make sure all the hosts mentioned in the connection
// string are part of the set. It is fine if not all members of the set are mentioned in the
// connection string, though.
if (!providedSetName.empty()) {
std::set<std::string> hostSet;
BSONObjIterator iter(resIsMaster["hosts"].Obj());
while (iter.more()) {
hostSet.insert(iter.next().String()); // host:port
}
if (resIsMaster["passives"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["passives"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
if (resIsMaster["arbiters"].isABSONObj()) {
BSONObjIterator piter(resIsMaster["arbiters"].Obj());
while (piter.more()) {
hostSet.insert(piter.next().String()); // host:port
}
}
for (const auto& hostEntry : connectionString.getServers()) {
const auto& host = hostEntry.toString(); // host:port
if (hostSet.find(host) == hostSet.end()) {
return {ErrorCodes::OperationFailed,
str::stream() << "in seed list " << connectionString.toString() << ", host "
<< host
<< " does not belong to replica set "
<< foundSetName
<< "; found "
<< resIsMaster.toString()};
}
}
}
std::string actualShardName;
if (shardProposedName) {
actualShardName = *shardProposedName;
} else if (!foundSetName.empty()) {
// Default it to the name of the replica set
actualShardName = foundSetName;
}
// Disallow adding shard replica set with name 'config'
if (actualShardName == NamespaceString::kConfigDb) {
return {ErrorCodes::BadValue, "use of shard replica set with name 'config' is not allowed"};
}
// Retrieve the most up to date connection string that we know from the replica set monitor (if
// this is a replica set shard, otherwise it will be the same value as connectionString).
ConnectionString actualShardConnStr = targeter->connectionString();
ShardType shard;
shard.setName(actualShardName);
shard.setHost(actualShardConnStr.toString());
shard.setState(ShardType::ShardState::kShardAware);
return shard;
}