bool initShardVersion( DBClientBase& conn_in, BSONObj& result ){
WriteBackListener::init( conn_in );
DBClientBase* conn = getVersionable( &conn_in );
assert( conn ); // errors thrown above
BSONObjBuilder cmdBuilder;
cmdBuilder.append( "setShardVersion" , "" );
cmdBuilder.appendBool( "init", true );
cmdBuilder.append( "configdb" , configServer.modelServer() );
cmdBuilder.appendOID( "serverID" , &serverID );
cmdBuilder.appendBool( "authoritative" , true );
BSONObj cmd = cmdBuilder.obj();
LOG(1) << "initializing shard connection to " << conn->toString() << endl;
LOG(2) << "initial sharding settings : " << cmd << endl;
bool ok = conn->runCommand( "admin" , cmd , result );
LOG(3) << "initial sharding result : " << result << endl;
return ok;
}
bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){
WriteBackListener::init( *conn_in );
DBClientBase* conn = getVersionable( conn_in );
verify( conn ); // errors thrown above
BSONObjBuilder cmdBuilder;
cmdBuilder.append( "setShardVersion" , "" );
cmdBuilder.appendBool( "init", true );
cmdBuilder.append( "configdb" , configServer.modelServer() );
cmdBuilder.appendOID( "serverID" , &serverID );
cmdBuilder.appendBool( "authoritative" , true );
BSONObj cmd = cmdBuilder.obj();
LOG(1) << "initializing shard connection to " << conn->toString() << endl;
LOG(2) << "initial sharding settings : " << cmd << endl;
bool ok = conn->runCommand( "admin" , cmd , result );
// HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
// Result is false, but will still initialize serverID and configdb
if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
{
ok = true;
}
LOG(3) << "initial sharding result : " << result << endl;
return ok;
}
bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){
WriteBackListener::init( *conn_in );
bool ok;
DBClientBase* conn = NULL;
try {
// May throw if replica set primary is down
conn = getVersionable( conn_in );
dassert( conn ); // errors thrown above
BSONObjBuilder cmdBuilder;
cmdBuilder.append( "setShardVersion" , "" );
cmdBuilder.appendBool( "init", true );
cmdBuilder.append( "configdb" , configServer.modelServer() );
cmdBuilder.appendOID( "serverID" , &serverID );
cmdBuilder.appendBool( "authoritative" , true );
BSONObj cmd = cmdBuilder.obj();
LOG(1) << "initializing shard connection to " << conn->toString() << endl;
LOG(2) << "initial sharding settings : " << cmd << endl;
ok = conn->runCommand("admin", cmd, result, 0);
}
catch( const DBException& ex ) {
bool ignoreFailure = ShardConnection::ignoreInitialVersionFailure
&& conn_in->type() == ConnectionString::SET;
if ( !ignoreFailure )
throw;
// Using initShardVersion is not strictly required when talking to replica sets - it is
// preferred to do so because it registers mongos early with the mongod. This info is
// also sent by checkShardVersion before a connection is used for a write or read.
OCCASIONALLY {
warning() << "failed to initialize new replica set connection version, "
<< "will initialize on first use" << endl;
}
return true;
}
// HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
// Result is false, but will still initialize serverID and configdb
if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
{
ok = true;
}
LOG(3) << "initial sharding result : " << result << endl;
return ok;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase * conn_in , const string& ns , ChunkManagerPtr refManager, bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( *conn_in );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = getVersionable( conn_in );
verify(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManagerIfExists( ns , authoritative );
// It's possible the chunk manager was reset since we checked whether sharded was true,
// so must check this here.
if( manager ) officialSequenceNumber = manager->getSequenceNumber();
}
// Check this manager against the reference manager
if( isSharded && manager ){
Shard shard = Shard::make( conn->getServerAddress() );
if( refManager && ! refManager->compatibleWith( manager, shard ) ){
throw SendStaleConfigException( ns, str::stream() << "manager (" << manager->getVersion( shard ).toString() << " : " << manager->getSequenceNumber() << ") "
<< "not compatible with reference manager (" << refManager->getVersion( shard ).toString() << " : " << refManager->getSequenceNumber() << ") "
<< "on shard " << shard.getName() << " (" << shard.getAddress().toString() << ")",
refManager->getVersion( shard ), manager->getVersion( shard ) );
}
}
else if( refManager ){
Shard shard = Shard::make( conn->getServerAddress() );
string msg( str::stream() << "not sharded ("
<< ( (manager.get() == 0) ? string( "<none>" ) :
str::stream() << manager->getSequenceNumber() )
<< ") but has reference manager ("
<< refManager->getSequenceNumber() << ") "
<< "on conn " << conn->getServerAddress() << " ("
<< conn_in->getServerAddress() << ")" );
throw SendStaleConfigException( ns, msg,
refManager->getVersion( shard ), ShardChunkVersion( 0, OID() ));
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie., last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = ShardChunkVersion( 0, OID() );
if ( isSharded && manager ) {
version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
}
if( ! version.isSet() ){
LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
( ! isSharded ? "no longer sharded" :
( ! manager ? "no chunk manager found" :
"version is zero" ) ) << endl;
}
LOG(2) << " have to set shard version for conn: " << conn->getServerAddress() << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
const string versionableServerAddress(conn->getServerAddress());
BSONObj result;
if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
warning() << "reloading full configuration for " << conf->getName()
<< ", connection state indicates significant version changes" << endl;
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion host: " << versionableServerAddress << " " << result << endl;
sleepmillis( 10 * tryNumber );
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
return true;
}
string errmsg = str::stream() << "setShardVersion failed host: " << versionableServerAddress << " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
/**
* Updates the remote cached version on the remote shard host (primary, in the case of replica
* sets) if needed with a fully-qualified shard version for the given namespace:
* config server(s) + shard name + shard version
*
* If no remote cached version has ever been set, an initial shard version is sent.
*
* If the namespace is empty and no version has ever been sent, the config server + shard name
* is sent to the remote shard host to initialize the connection as coming from mongos.
* NOTE: This initialization is *best-effort only*. Operations which wish to correctly version
* must send the namespace.
*
* Config servers are special and are not (unless otherwise a shard) kept up to date with this
* protocol. This is safe so long as config servers only contain unversioned collections.
*
* It is an error to call checkShardVersion with an unversionable connection (isVersionableCB).
*
* @return true if we contacted the remote host
*/
bool checkShardVersion(DBClientBase* conn_in,
const string& ns,
ChunkManagerPtr refManager,
bool authoritative,
int tryNumber) {
// TODO: cache, optimize, etc...
// Empty namespaces are special - we require initialization but not versioning
if (ns.size() == 0) {
return initShardVersionEmptyNS(conn_in);
}
auto status = grid.catalogCache()->getDatabase(nsToDatabase(ns));
if (!status.isOK()) {
return false;
}
shared_ptr<DBConfig> conf = status.getValue();
DBClientBase* conn = getVersionable(conn_in);
verify(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ShardPtr primary;
ChunkManagerPtr manager;
if (authoritative)
conf->getChunkManagerIfExists(ns, true);
conf->getChunkManagerOrPrimary(ns, manager, primary);
if (manager) {
officialSequenceNumber = manager->getSequenceNumber();
}
const auto shard = grid.shardRegistry()->getShard(conn->getServerAddress());
uassert(ErrorCodes::ShardNotFound,
str::stream() << conn->getServerAddress() << " is not recognized as a shard",
shard);
// Check this manager against the reference manager
if (manager) {
if (refManager && !refManager->compatibleWith(*manager, shard->getId())) {
const ChunkVersion refVersion(refManager->getVersion(shard->getId()));
const ChunkVersion currentVersion(manager->getVersion(shard->getId()));
string msg(str::stream()
<< "manager (" << currentVersion.toString() << " : "
<< manager->getSequenceNumber() << ") "
<< "not compatible with reference manager (" << refVersion.toString()
<< " : " << refManager->getSequenceNumber() << ") "
<< "on shard " << shard->getId() << " (" << shard->getConnString().toString()
<< ")");
throw SendStaleConfigException(ns, msg, refVersion, currentVersion);
}
} else if (refManager) {
string msg(str::stream() << "not sharded ("
<< ((manager.get() == 0) ? string("<none>") : str::stream()
<< manager->getSequenceNumber())
<< ") but has reference manager ("
<< refManager->getSequenceNumber() << ") "
<< "on conn " << conn->getServerAddress() << " ("
<< conn_in->getServerAddress() << ")");
throw SendStaleConfigException(
ns, msg, refManager->getVersion(shard->getId()), ChunkVersion::UNSHARDED());
}
// Do not send setShardVersion to collections on the config servers - this causes problems
// when config servers are also shards and get SSV with conflicting names.
// TODO: Make config servers regular shards
if (primary && primary->getId() == "config") {
return false;
}
// Has the ChunkManager been reloaded since the last time we updated the shard version over
// this connection? If we've never updated the shard version, do so now.
unsigned long long sequenceNumber = 0;
if (connectionShardStatus.getSequence(conn, ns, &sequenceNumber)) {
if (sequenceNumber == officialSequenceNumber) {
return false;
}
}
ChunkVersion version = ChunkVersion(0, 0, OID());
if (manager) {
version = manager->getVersion(shard->getId());
}
LOG(1) << "setting shard version of " << version << " for " << ns << " on shard "
<< shard->toString();
LOG(3) << "last version sent with chunk manager iteration " << sequenceNumber
<< ", current chunk manager iteration is " << officialSequenceNumber;
BSONObj result;
if (setShardVersion(*conn,
ns,
grid.catalogManager()->connectionString().toString(),
version,
manager.get(),
authoritative,
result)) {
LOG(1) << " setShardVersion success: " << result;
connectionShardStatus.setSequence(conn, ns, officialSequenceNumber);
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if (result["need_authoritative"].trueValue())
massert(10428, "need_authoritative set but in authoritative mode already", !authoritative);
if (!authoritative) {
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
return true;
}
if (result["reloadConfig"].trueValue()) {
if (result["version"].timestampTime() == Date_t()) {
warning() << "reloading full configuration for " << conf->name()
<< ", connection state indicates significant version changes";
// reload db
conf->reload();
} else {
// reload config
conf->getChunkManager(ns, true);
}
}
const int maxNumTries = 7;
if (tryNumber < maxNumTries) {
LOG(tryNumber < (maxNumTries / 2) ? 1 : 0)
<< "going to retry checkShardVersion shard: " << shard->toString() << " " << result;
sleepmillis(10 * tryNumber);
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
return true;
}
string errmsg = str::stream() << "setShardVersion failed shard: " << shard->toString() << " "
<< result;
log() << " " << errmsg << endl;
massert(10429, errmsg, 0);
return true;
}
/**
* Special internal logic to run reduced version handshake for empty namespace operations to
* shards.
*
* Eventually this should go completely away, but for now many commands rely on unversioned but
* mongos-specific behavior on mongod (auditing and replication information in commands)
*/
static bool initShardVersionEmptyNS(DBClientBase* conn_in) {
bool ok;
BSONObj result;
DBClientBase* conn = NULL;
try {
// May throw if replica set primary is down
conn = getVersionable(conn_in);
dassert(conn); // errors thrown above
// Check to see if we've already initialized this connection
if (connectionShardStatus.hasAnySequenceSet(conn))
return false;
// Check to see if this is actually a shard and not a single config server
// NOTE: Config servers are registered only by the name "config" in the shard cache, not
// by host, so lookup by host will fail unless the host is also a shard.
const auto shard = grid.shardRegistry()->getShard(conn->getServerAddress());
if (!shard) {
return false;
}
LOG(1) << "initializing shard connection to " << shard->toString() << endl;
ok = setShardVersion(*conn,
"",
grid.catalogManager()->connectionString().toString(),
ChunkVersion(),
NULL,
true,
result);
} catch (const DBException&) {
// NOTE: Replica sets may fail to initShardVersion because future calls relying on
// correct versioning must later call checkShardVersion on the primary.
// Secondary queries and commands may not call checkShardVersion, but secondary ops
// aren't versioned at all.
if (conn_in->type() != ConnectionString::SET) {
throw;
}
// NOTE: Only old-style cluster operations will talk via DBClientReplicaSets - using
// checkShardVersion is required (which includes initShardVersion information) if these
// connections are used.
OCCASIONALLY {
warning() << "failed to initialize new replica set connection version, "
<< "will initialize on first use" << endl;
}
return false;
}
// Record the connection wire version if sent in the response, initShardVersion is a
// handshake for mongos->mongod connections.
if (!result["minWireVersion"].eoo()) {
int minWireVersion = result["minWireVersion"].numberInt();
int maxWireVersion = result["maxWireVersion"].numberInt();
conn->setWireVersions(minWireVersion, maxWireVersion);
}
LOG(3) << "initial sharding result : " << result << endl;
connectionShardStatus.setSequence(conn, "", 0);
return ok;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase& conn_in , const string& ns , bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( conn_in );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = getVersionable( &conn_in );
assert(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManagerIfExists( ns , authoritative );
// It's possible the chunk manager was reset since we checked whether sharded was true,
// so must check this here.
if( manager ) officialSequenceNumber = manager->getSequenceNumber();
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie., last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = 0;
if ( isSharded && manager ) {
version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
}
if( version == 0 ){
LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
( ! isSharded ? "no longer sharded" :
( ! manager ? "no chunk manager found" :
"version is zero" ) ) << endl;
}
LOG(2) << " have to set shard version for conn: " << conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
checkShardVersion( *conn , ns , 1 , tryNumber + 1 );
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl;
sleepmillis( 10 * tryNumber );
checkShardVersion( *conn , ns , true , tryNumber + 1 );
return true;
}
string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
bool VersionManager::initShardVersionCB( DBClientBase * conn_in, BSONObj& result ){
WriteBackListener::init( *conn_in );
bool ok;
DBClientBase* conn = NULL;
try {
// May throw if replica set primary is down
conn = getVersionable( conn_in );
dassert( conn ); // errors thrown above
BSONObjBuilder cmdBuilder;
cmdBuilder.append( "setShardVersion" , "" );
cmdBuilder.appendBool( "init", true );
cmdBuilder.append( "configdb" , configServer.modelServer() );
cmdBuilder.appendOID( "serverID" , &serverID );
cmdBuilder.appendBool( "authoritative" , true );
BSONObj cmd = cmdBuilder.obj();
LOG(1) << "initializing shard connection to " << conn->toString() << endl;
LOG(2) << "initial sharding settings : " << cmd << endl;
ok = conn->runCommand("admin", cmd, result, 0);
}
catch( const DBException& ) {
if ( conn_in->type() != ConnectionString::SET ) {
throw;
}
// NOTE: Only old-style cluster operations will talk via DBClientReplicaSets - using
// checkShardVersion is required (which includes initShardVersion information) if these
// connections are used.
OCCASIONALLY {
warning() << "failed to initialize new replica set connection version, "
<< "will initialize on first use" << endl;
}
return true;
}
// HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1
// Result is false, but will still initialize serverID and configdb
if( ! ok && ! result["errmsg"].eoo() && ( result["errmsg"].String() == "need to specify namespace"/* 2.0.1/2 */ ||
result["errmsg"].String() == "need to speciy namespace" /* 1.8 */ ))
{
ok = true;
}
// Record the connection wire version if sent in the response, initShardVersion is a
// handshake for mongos->mongod connections.
if ( !result["minWireVersion"].eoo() ) {
int minWireVersion = result["minWireVersion"].numberInt();
int maxWireVersion = result["maxWireVersion"].numberInt();
conn->setWireVersions( minWireVersion, maxWireVersion );
}
LOG(3) << "initial sharding result : " << result << endl;
return ok;
}