TEST_F(ShardingStateTest, InitializeAgainWithDifferentClusterIdFails) {
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
ConnectionString(ConnectionString::SET, "a:1,b:2", "config"));
shardIdentity.setShardName("a");
shardIdentity.setClusterId(OID::gen());
ASSERT_OK(shardingState()->initializeFromShardIdentity(operationContext(), shardIdentity));
ShardIdentityType shardIdentity2;
shardIdentity2.setConfigsvrConnString(
ConnectionString(ConnectionString::SET, "a:1,b:2", "config"));
shardIdentity2.setShardName("a");
shardIdentity2.setClusterId(OID::gen());
shardingState()->setGlobalInitMethodForTest(
[](OperationContext* txn, const ConnectionString& connStr, StringData distLockProcessId) {
return Status{ErrorCodes::InternalError, "should not reach here"};
});
auto status = shardingState()->initializeFromShardIdentity(operationContext(), shardIdentity2);
ASSERT_EQ(ErrorCodes::InconsistentShardIdentity, status);
ASSERT_TRUE(shardingState()->enabled());
ASSERT_EQ("a", shardingState()->getShardName());
ASSERT_EQ("config/a:1,b:2", shardingState()->getConfigServer(operationContext()).toString());
}
TEST_F(ShardingStateTest, InitWhilePreviouslyInErrorStateWillStayInErrorState) {
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
ConnectionString(ConnectionString::SET, "a:1,b:2", "config"));
shardIdentity.setShardName("a");
shardIdentity.setClusterId(OID::gen());
shardingState()->setGlobalInitMethodForTest(
[](OperationContext* txn, const ConnectionString& connStr, StringData distLockProcessId) {
return Status{ErrorCodes::ShutdownInProgress, "shutting down"};
});
{
auto status =
shardingState()->initializeFromShardIdentity(operationContext(), shardIdentity);
ASSERT_EQ(ErrorCodes::ShutdownInProgress, status);
}
// ShardingState is now in error state, attempting to call it again will still result in error.
shardingState()->setGlobalInitMethodForTest(
[](OperationContext* txn, const ConnectionString& connStr, StringData distLockProcessId) {
return Status::OK();
});
{
auto status =
shardingState()->initializeFromShardIdentity(operationContext(), shardIdentity);
ASSERT_EQ(ErrorCodes::ManualInterventionRequired, status);
}
ASSERT_FALSE(shardingState()->enabled());
}
TEST_F(ShardingStateTest, ValidShardIdentitySucceeds) {
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
ConnectionString(ConnectionString::SET, "a:1,b:2", "config"));
shardIdentity.setShardName("a");
shardIdentity.setClusterId(OID::gen());
ASSERT_OK(shardingState()->initializeFromShardIdentity(operationContext(), shardIdentity));
ASSERT_TRUE(shardingState()->enabled());
ASSERT_EQ("a", shardingState()->getShardName());
ASSERT_EQ("config/a:1,b:2", shardingState()->getConfigServer(operationContext()).toString());
}
TEST_F(ShardingStateTest,
InitializeShardingAwarenessIfNeededNotReadOnlyAndShardServerAndValidShardIdentity) {
// Insert the shardIdentity doc to disk before setting the clusterRole, since if the clusterRole
// is ShardServer, the OpObserver for inserts will trigger sharding initialization from the
// inserted doc.
serverGlobalParams.clusterRole = ClusterRole::None;
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
ConnectionString(ConnectionString::SET, "a:1,b:2", "config"));
shardIdentity.setShardName("a");
shardIdentity.setClusterId(OID::gen());
ASSERT_OK(shardIdentity.validate());
BSONObj validShardIdentity = shardIdentity.toBSON();
_dbDirectClient->insert(NamespaceString::kConfigCollectionNamespace.toString(),
validShardIdentity);
storageGlobalParams.readOnly = false;
serverGlobalParams.clusterRole = ClusterRole::ShardServer;
serverGlobalParams.overrideShardIdentity = BSONObj();
auto swShardingInitialized =
shardingState()->initializeShardingAwarenessIfNeeded(operationContext());
ASSERT_OK(swShardingInitialized);
ASSERT_TRUE(swShardingInitialized.getValue());
}
TEST_F(ShardingStateTest,
InitializeShardingAwarenessIfNeededReadOnlyAndNotShardServerAndValidOverrideShardIdentity) {
storageGlobalParams.readOnly = true;
serverGlobalParams.clusterRole = ClusterRole::None;
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
ConnectionString(ConnectionString::SET, "a:1,b:2", "config"));
shardIdentity.setShardName("a");
shardIdentity.setClusterId(OID::gen());
ASSERT_OK(shardIdentity.validate());
serverGlobalParams.overrideShardIdentity = shardIdentity.toBSON();
auto swShardingInitialized =
shardingState()->initializeShardingAwarenessIfNeeded(operationContext());
ASSERT_EQUALS(ErrorCodes::InvalidOptions, swShardingInitialized.getStatus().code());
}
void ShardingInitializationMongoD::initializeFromShardIdentity(
OperationContext* opCtx, const ShardIdentityType& shardIdentity) {
invariant(serverGlobalParams.clusterRole == ClusterRole::ShardServer);
invariant(opCtx->lockState()->isLocked());
uassertStatusOKWithContext(
shardIdentity.validate(),
"Invalid shard identity document found when initializing sharding state");
log() << "initializing sharding state with: " << shardIdentity;
const auto& configSvrConnStr = shardIdentity.getConfigsvrConnectionString();
auto const shardingState = ShardingState::get(opCtx);
auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
stdx::unique_lock<stdx::mutex> ul(_initSynchronizationMutex);
if (shardingState->enabled()) {
uassert(40371, "", shardingState->shardId() == shardIdentity.getShardName());
uassert(40372, "", shardingState->clusterId() == shardIdentity.getClusterId());
auto prevConfigsvrConnStr = shardRegistry->getConfigServerConnectionString();
uassert(40373, "", prevConfigsvrConnStr.type() == ConnectionString::SET);
uassert(40374, "", prevConfigsvrConnStr.getSetName() == configSvrConnStr.getSetName());
return;
}
auto initializationStatus = shardingState->initializationStatus();
uassert(ErrorCodes::ManualInterventionRequired,
str::stream() << "Server's sharding metadata manager failed to initialize and will "
"remain in this state until the instance is manually reset"
<< causedBy(*initializationStatus),
!initializationStatus);
try {
_initFunc(opCtx, shardIdentity, generateDistLockProcessId(opCtx));
shardingState->setInitialized(shardIdentity.getShardName().toString(),
shardIdentity.getClusterId());
} catch (const DBException& ex) {
shardingState->setInitialized(ex.toStatus());
}
}
TEST_F(ShardingStateTest,
InitializeShardingAwarenessIfNeededNotReadOnlyAndNotShardServerAndValidShardIdentity) {
storageGlobalParams.readOnly = false;
serverGlobalParams.clusterRole = ClusterRole::None;
serverGlobalParams.overrideShardIdentity = BSONObj();
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
ConnectionString(ConnectionString::SET, "a:1,b:2", "config"));
shardIdentity.setShardName("a");
shardIdentity.setClusterId(OID::gen());
ASSERT_OK(shardIdentity.validate());
BSONObj validShardIdentity = shardIdentity.toBSON();
_dbDirectClient->insert(NamespaceString::kConfigCollectionNamespace.toString(),
validShardIdentity);
// The shardIdentity doc on disk is ignored if ClusterRole is None.
auto swShardingInitialized =
shardingState()->initializeShardingAwarenessIfNeeded(operationContext());
ASSERT_OK(swShardingInitialized);
ASSERT_FALSE(swShardingInitialized.getValue());
}
StatusWith<ShardIdentityType> ShardIdentityType::fromBSON(const BSONObj& source) {
if (!source.hasField("_id")) {
return {ErrorCodes::NoSuchKey,
str::stream() << "missing _id field for shardIdentity document"};
}
ShardIdentityType shardIdentity;
{
std::string docId;
Status status = bsonExtractStringField(source, "_id", &docId);
if (!status.isOK()) {
return status;
}
if (docId != IdName) {
return {ErrorCodes::FailedToParse,
str::stream() << "got _id: " << docId << " instead of " << IdName};
}
}
{
std::string connString;
Status status = bsonExtractStringField(source, configsvrConnString(), &connString);
if (!status.isOK()) {
return status;
}
try {
// Note: ConnectionString::parse can uassert from HostAndPort constructor.
auto parsedConfigConnStrStatus = ConnectionString::parse(connString);
if (!parsedConfigConnStrStatus.isOK()) {
return parsedConfigConnStrStatus.getStatus();
}
auto configSvrConnStr = parsedConfigConnStrStatus.getValue();
if (configSvrConnStr.type() != ConnectionString::SET) {
return Status(ErrorCodes::UnsupportedFormat,
str::stream()
<< "config server connection string can only be replica sets: "
<< configSvrConnStr.toString());
}
shardIdentity.setConfigsvrConnString(std::move(configSvrConnStr));
} catch (const UserException& parseException) {
return parseException.toStatus();
}
}
{
std::string name;
Status status = bsonExtractStringField(source, shardName(), &name);
if (!status.isOK()) {
return status;
}
shardIdentity.setShardName(name);
}
{
OID oid;
Status status = bsonExtractOIDField(source, clusterId(), &oid);
if (!status.isOK()) {
return status;
}
shardIdentity.setClusterId(oid);
}
return shardIdentity;
}
StatusWith<string> ShardingCatalogManagerImpl::addShard(
OperationContext* txn,
const std::string* shardProposedName,
const ConnectionString& shardConnectionString,
const long long maxSize) {
if (shardConnectionString.type() == ConnectionString::INVALID) {
return {ErrorCodes::BadValue, "Invalid connection string"};
}
if (shardProposedName && shardProposedName->empty()) {
return {ErrorCodes::BadValue, "shard name cannot be empty"};
}
// TODO: Don't create a detached Shard object, create a detached RemoteCommandTargeter instead.
const std::shared_ptr<Shard> shard{
Grid::get(txn)->shardRegistry()->createConnection(shardConnectionString)};
invariant(shard);
auto targeter = shard->getTargeter();
// Validate the specified connection string may serve as shard at all
auto shardStatus =
_validateHostAsShard(txn, targeter, shardProposedName, shardConnectionString);
if (!shardStatus.isOK()) {
// TODO: This is a workaround for the case were we could have some bad shard being
// requested to be added and we put that bad connection string on the global replica set
// monitor registry. It needs to be cleaned up so that when a correct replica set is added,
// it will be recreated.
ReplicaSetMonitor::remove(shardConnectionString.getSetName());
return shardStatus.getStatus();
}
ShardType& shardType = shardStatus.getValue();
auto dbNamesStatus = _getDBNamesListFromShard(txn, targeter);
if (!dbNamesStatus.isOK()) {
return dbNamesStatus.getStatus();
}
// Check that none of the existing shard candidate's dbs exist already
for (const string& dbName : dbNamesStatus.getValue()) {
auto dbt = _catalogClient->getDatabase(txn, dbName);
if (dbt.isOK()) {
const auto& dbDoc = dbt.getValue().value;
return Status(ErrorCodes::OperationFailed,
str::stream() << "can't add shard "
<< "'"
<< shardConnectionString.toString()
<< "'"
<< " because a local database '"
<< dbName
<< "' exists in another "
<< dbDoc.getPrimary());
} else if (dbt != ErrorCodes::NamespaceNotFound) {
return dbt.getStatus();
}
}
// If a name for a shard wasn't provided, generate one
if (shardType.getName().empty()) {
StatusWith<string> result = _generateNewShardName(txn);
if (!result.isOK()) {
return result.getStatus();
}
shardType.setName(result.getValue());
}
if (maxSize > 0) {
shardType.setMaxSizeMB(maxSize);
}
ShardIdentityType shardIdentity;
shardIdentity.setConfigsvrConnString(
Grid::get(txn)->shardRegistry()->getConfigServerConnectionString());
shardIdentity.setShardName(shardType.getName());
shardIdentity.setClusterId(Grid::get(txn)->shardRegistry()->getClusterId());
auto validateStatus = shardIdentity.validate();
if (!validateStatus.isOK()) {
return validateStatus;
}
log() << "going to insert shardIdentity document into shard: " << shardIdentity.toString();
auto updateRequest = shardIdentity.createUpsertForAddShard();
BatchedCommandRequest commandRequest(updateRequest.release());
commandRequest.setNS(NamespaceString::kConfigCollectionNamespace);
commandRequest.setWriteConcern(kMajorityWriteConcern.toBSON());
auto swCommandResponse =
_runCommandForAddShard(txn, targeter.get(), "admin", commandRequest.toBSON());
if (!swCommandResponse.isOK()) {
return swCommandResponse.getStatus();
}
auto commandResponse = std::move(swCommandResponse.getValue());
BatchedCommandResponse batchResponse;
auto batchResponseStatus =
Shard::CommandResponse::processBatchWriteResponse(commandResponse, &batchResponse);
if (!batchResponseStatus.isOK()) {
return batchResponseStatus;
}
log() << "going to insert new entry for shard into config.shards: " << shardType.toString();
Status result =
_catalogClient->insertConfigDocument(txn, ShardType::ConfigNS, shardType.toBSON());
if (!result.isOK()) {
log() << "error adding shard: " << shardType.toBSON() << " err: " << result.reason();
if (result == ErrorCodes::DuplicateKey) {
// TODO(SERVER-24213): adding a shard that already exists should be considered success,
// however this approach does no validation that we are adding the shard with the same
// options. It also does not protect against adding the same shard with a different
// shard name and slightly different connection string. This is a temporary hack to
// get the continuous stepdown suite passing.
warning() << "Received duplicate key error when inserting new shard with name "
<< shardType.getName() << " and connection string "
<< shardConnectionString.toString()
<< " to config.shards collection. This most likely means that there was an "
"attempt to add a shard that already exists in the cluster";
return shardType.getName();
}
return result;
}
// Add all databases which were discovered on the new shard
for (const string& dbName : dbNamesStatus.getValue()) {
DatabaseType dbt;
dbt.setName(dbName);
dbt.setPrimary(shardType.getName());
dbt.setSharded(false);
Status status = _catalogClient->updateDatabase(txn, dbName, dbt);
if (!status.isOK()) {
log() << "adding shard " << shardConnectionString.toString()
<< " even though could not add database " << dbName;
}
}
// Record in changelog
BSONObjBuilder shardDetails;
shardDetails.append("name", shardType.getName());
shardDetails.append("host", shardConnectionString.toString());
_catalogClient->logChange(txn, "addShard", "", shardDetails.obj());
// Ensure the added shard is visible to this process.
auto shardRegistry = Grid::get(txn)->shardRegistry();
if (!shardRegistry->getShard(txn, shardType.getName())) {
return {ErrorCodes::OperationFailed,
"Could not find shard metadata for shard after adding it. This most likely "
"indicates that the shard was removed immediately after it was added."};
}
return shardType.getName();
}
