void checkShardVersion( DBClientBase& conn , const string& ns , bool authoritative ){
// TODO: cache, optimize, etc...
WriteBackListener::init( conn );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return;
ShardChunkVersion version = 0;
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ){
manager = conf->getChunkManager( ns , authoritative );
officialSequenceNumber = manager->getSequenceNumber();
}
unsigned long long & sequenceNumber = checkShardVersionLastSequence[ make_pair(&conn,ns) ];
if ( sequenceNumber == officialSequenceNumber )
return;
if ( isSharded ){
version = manager->getVersion( Shard::make( conn.getServerAddress() ) );
}
log(2) << " have to set shard version for conn: " << &conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( conn , ns , version , authoritative , result ) ){
// success!
log(1) << " setShardVersion success!" << endl;
sequenceNumber = officialSequenceNumber;
dassert( sequenceNumber == checkShardVersionLastSequence[ make_pair(&conn,ns) ] );
return;
}
log(1) << " setShardVersion failed!\n" << result << endl;
if ( result.getBoolField( "need_authoritative" ) )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ){
checkShardVersion( conn , ns , 1 );
return;
}
log() << " setShardVersion failed: " << result << endl;
massert( 10429 , (string)"setShardVersion failed! " + result.jsonString() , 0 );
}
bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {
string ns = cmdObj.firstElement().valuestrsafe();
if ( ns.size() == 0 ) {
errmsg = "need to specify fully namespace";
return false;
}
DBConfigPtr config = grid.getDBConfig( ns );
if ( ! config->isSharded( ns ) ) {
errmsg = "ns not sharded.";
return false;
}
ChunkManagerPtr cm = config->getChunkManagerIfExists( ns );
if ( ! cm ) {
errmsg = "no chunk manager?";
return false;
}
cm->_printChunks();
result.appendTimestamp( "version" , cm->getVersion().toLong() );
return 1;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase * conn_in , const string& ns , ChunkManagerPtr refManager, bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( *conn_in );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = getVersionable( conn_in );
verify(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManagerIfExists( ns , authoritative );
// It's possible the chunk manager was reset since we checked whether sharded was true,
// so must check this here.
if( manager ) officialSequenceNumber = manager->getSequenceNumber();
}
// Check this manager against the reference manager
if( isSharded && manager ){
Shard shard = Shard::make( conn->getServerAddress() );
if( refManager && ! refManager->compatibleWith( manager, shard ) ){
throw SendStaleConfigException( ns, str::stream() << "manager (" << manager->getVersion( shard ).toString() << " : " << manager->getSequenceNumber() << ") "
<< "not compatible with reference manager (" << refManager->getVersion( shard ).toString() << " : " << refManager->getSequenceNumber() << ") "
<< "on shard " << shard.getName() << " (" << shard.getAddress().toString() << ")",
refManager->getVersion( shard ), manager->getVersion( shard ) );
}
}
else if( refManager ){
Shard shard = Shard::make( conn->getServerAddress() );
string msg( str::stream() << "not sharded ("
<< ( (manager.get() == 0) ? string( "<none>" ) :
str::stream() << manager->getSequenceNumber() )
<< ") but has reference manager ("
<< refManager->getSequenceNumber() << ") "
<< "on conn " << conn->getServerAddress() << " ("
<< conn_in->getServerAddress() << ")" );
throw SendStaleConfigException( ns, msg,
refManager->getVersion( shard ), ShardChunkVersion( 0, OID() ));
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie., last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = ShardChunkVersion( 0, OID() );
if ( isSharded && manager ) {
version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
}
if( ! version.isSet() ){
LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
( ! isSharded ? "no longer sharded" :
( ! manager ? "no chunk manager found" :
"version is zero" ) ) << endl;
}
LOG(2) << " have to set shard version for conn: " << conn->getServerAddress() << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
const string versionableServerAddress(conn->getServerAddress());
BSONObj result;
if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
warning() << "reloading full configuration for " << conf->getName()
<< ", connection state indicates significant version changes" << endl;
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion host: " << versionableServerAddress << " " << result << endl;
sleepmillis( 10 * tryNumber );
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
return true;
}
string errmsg = str::stream() << "setShardVersion failed host: " << versionableServerAddress << " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase& conn_in , const string& ns , bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( conn_in );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = 0;
switch ( conn_in.type() ) {
case ConnectionString::INVALID:
assert(0);
break;
case ConnectionString::MASTER:
// great
conn = &conn_in;
break;
case ConnectionString::PAIR:
assert( ! "pair not support for sharding" );
break;
case ConnectionString::SYNC:
// TODO: we should check later that we aren't actually sharded on this
conn = &conn_in;
break;
case ConnectionString::SET:
DBClientReplicaSet* set = (DBClientReplicaSet*)&conn_in;
conn = &(set->masterConn());
break;
}
assert(conn);
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManagerIfExists( ns , authoritative );
// It's possible the chunk manager was reset since we checked whether sharded was true,
// so must check this here.
if( manager ) officialSequenceNumber = manager->getSequenceNumber();
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie., last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = 0;
if ( isSharded && manager ) {
version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
}
LOG(2) << " have to set shard version for conn: " << conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
checkShardVersion( *conn , ns , 1 , tryNumber + 1 );
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl;
sleepmillis( 10 * tryNumber );
checkShardVersion( *conn , ns , true , tryNumber + 1 );
return true;
}
string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase& conn , const string& ns , bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( conn );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManager( ns , authoritative );
officialSequenceNumber = manager->getSequenceNumber();
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie, last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(&conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = 0;
if ( isSharded ) {
version = manager->getVersion( Shard::make( conn.getServerAddress() ) );
}
log(2) << " have to set shard version for conn: " << &conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( &conn , ns , officialSequenceNumber );
return true;
}
log(1) << " setShardVersion failed!\n" << result << endl;
if ( result.getBoolField( "need_authoritative" ) )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
checkShardVersion( conn , ns , 1 , tryNumber + 1 );
return true;
}
if ( tryNumber < 4 ) {
log(1) << "going to retry checkShardVersion" << endl;
sleepmillis( 10 );
checkShardVersion( conn , ns , 1 , tryNumber + 1 );
return true;
}
log() << " setShardVersion failed: " << result << endl;
massert( 10429 , (string)"setShardVersion failed! " + result.jsonString() , 0 );
return true;
}
/**
* Updates the remote cached version on the remote shard host (primary, in the case of replica
* sets) if needed with a fully-qualified shard version for the given namespace:
* config server(s) + shard name + shard version
*
* If no remote cached version has ever been set, an initial shard version is sent.
*
* If the namespace is empty and no version has ever been sent, the config server + shard name
* is sent to the remote shard host to initialize the connection as coming from mongos.
* NOTE: This initialization is *best-effort only*. Operations which wish to correctly version
* must send the namespace.
*
* Config servers are special and are not (unless otherwise a shard) kept up to date with this
* protocol. This is safe so long as config servers only contain unversioned collections.
*
* It is an error to call checkShardVersion with an unversionable connection (isVersionableCB).
*
* @return true if we contacted the remote host
*/
bool checkShardVersion(DBClientBase* conn_in,
const string& ns,
ChunkManagerPtr refManager,
bool authoritative,
int tryNumber) {
// TODO: cache, optimize, etc...
// Empty namespaces are special - we require initialization but not versioning
if (ns.size() == 0) {
return initShardVersionEmptyNS(conn_in);
}
auto status = grid.catalogCache()->getDatabase(nsToDatabase(ns));
if (!status.isOK()) {
return false;
}
shared_ptr<DBConfig> conf = status.getValue();
DBClientBase* conn = getVersionable(conn_in);
verify(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ShardPtr primary;
ChunkManagerPtr manager;
if (authoritative)
conf->getChunkManagerIfExists(ns, true);
conf->getChunkManagerOrPrimary(ns, manager, primary);
if (manager) {
officialSequenceNumber = manager->getSequenceNumber();
}
const auto shard = grid.shardRegistry()->getShard(conn->getServerAddress());
uassert(ErrorCodes::ShardNotFound,
str::stream() << conn->getServerAddress() << " is not recognized as a shard",
shard);
// Check this manager against the reference manager
if (manager) {
if (refManager && !refManager->compatibleWith(*manager, shard->getId())) {
const ChunkVersion refVersion(refManager->getVersion(shard->getId()));
const ChunkVersion currentVersion(manager->getVersion(shard->getId()));
string msg(str::stream()
<< "manager (" << currentVersion.toString() << " : "
<< manager->getSequenceNumber() << ") "
<< "not compatible with reference manager (" << refVersion.toString()
<< " : " << refManager->getSequenceNumber() << ") "
<< "on shard " << shard->getId() << " (" << shard->getConnString().toString()
<< ")");
throw SendStaleConfigException(ns, msg, refVersion, currentVersion);
}
} else if (refManager) {
string msg(str::stream() << "not sharded ("
<< ((manager.get() == 0) ? string("<none>") : str::stream()
<< manager->getSequenceNumber())
<< ") but has reference manager ("
<< refManager->getSequenceNumber() << ") "
<< "on conn " << conn->getServerAddress() << " ("
<< conn_in->getServerAddress() << ")");
throw SendStaleConfigException(
ns, msg, refManager->getVersion(shard->getId()), ChunkVersion::UNSHARDED());
}
// Do not send setShardVersion to collections on the config servers - this causes problems
// when config servers are also shards and get SSV with conflicting names.
// TODO: Make config servers regular shards
if (primary && primary->getId() == "config") {
return false;
}
// Has the ChunkManager been reloaded since the last time we updated the shard version over
// this connection? If we've never updated the shard version, do so now.
unsigned long long sequenceNumber = 0;
if (connectionShardStatus.getSequence(conn, ns, &sequenceNumber)) {
if (sequenceNumber == officialSequenceNumber) {
return false;
}
}
ChunkVersion version = ChunkVersion(0, 0, OID());
if (manager) {
version = manager->getVersion(shard->getId());
}
LOG(1) << "setting shard version of " << version << " for " << ns << " on shard "
<< shard->toString();
LOG(3) << "last version sent with chunk manager iteration " << sequenceNumber
<< ", current chunk manager iteration is " << officialSequenceNumber;
BSONObj result;
if (setShardVersion(*conn,
ns,
grid.catalogManager()->connectionString().toString(),
version,
manager.get(),
authoritative,
result)) {
LOG(1) << " setShardVersion success: " << result;
connectionShardStatus.setSequence(conn, ns, officialSequenceNumber);
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if (result["need_authoritative"].trueValue())
massert(10428, "need_authoritative set but in authoritative mode already", !authoritative);
if (!authoritative) {
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
return true;
}
if (result["reloadConfig"].trueValue()) {
if (result["version"].timestampTime() == Date_t()) {
warning() << "reloading full configuration for " << conf->name()
<< ", connection state indicates significant version changes";
// reload db
conf->reload();
} else {
// reload config
conf->getChunkManager(ns, true);
}
}
const int maxNumTries = 7;
if (tryNumber < maxNumTries) {
LOG(tryNumber < (maxNumTries / 2) ? 1 : 0)
<< "going to retry checkShardVersion shard: " << shard->toString() << " " << result;
sleepmillis(10 * tryNumber);
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
return true;
}
string errmsg = str::stream() << "setShardVersion failed shard: " << shard->toString() << " "
<< result;
log() << " " << errmsg << endl;
massert(10429, errmsg, 0);
return true;
}
void WriteBackListener::run() {
int secsToSleep = 0;
scoped_ptr<ChunkVersion> lastNeededVersion;
int lastNeededCount = 0;
bool needsToReloadShardInfo = false;
while ( ! inShutdown() ) {
if ( ! Shard::isAShardNode( _addr ) ) {
LOG(1) << _addr << " is not a shard node" << endl;
sleepsecs( 60 );
continue;
}
try {
if (needsToReloadShardInfo) {
// It's possible this shard was removed
Shard::reloadShardInfo();
needsToReloadShardInfo = false;
}
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getInternalScopedDbConnection( _addr ) );
BSONObj result;
{
BSONObjBuilder cmd;
cmd.appendOID( "writebacklisten" , &serverID ); // Command will block for data
if ( ! conn->get()->runCommand( "admin" , cmd.obj() , result ) ) {
result = result.getOwned();
log() << "writebacklisten command failed! " << result << endl;
conn->done();
continue;
}
}
conn->done();
LOG(1) << "writebacklisten result: " << result << endl;
BSONObj data = result.getObjectField( "data" );
if ( data.getBoolField( "writeBack" ) ) {
string ns = data["ns"].valuestrsafe();
ConnectionIdent cid( "" , 0 );
OID wid;
if ( data["connectionId"].isNumber() && data["id"].type() == jstOID ) {
string s = "";
if ( data["instanceIdent"].type() == String )
s = data["instanceIdent"].String();
cid = ConnectionIdent( s , data["connectionId"].numberLong() );
wid = data["id"].OID();
}
else {
warning() << "mongos/mongod version mismatch (1.7.5 is the split)" << endl;
}
int len; // not used, but needed for next call
Message msg( (void*)data["msg"].binData( len ) , false );
massert( 10427 , "invalid writeback message" , msg.header()->valid() );
DBConfigPtr db = grid.getDBConfig( ns );
ChunkVersion needVersion = ChunkVersion::fromBSON( data, "version" );
//
// TODO: Refactor the sharded strategy to correctly handle all sharding state changes itself,
// we can't rely on WBL to do this for us b/c anything could reset our state in-between.
// We should always reload here for efficiency when possible, but staleness is also caught in the
// loop below.
//
ChunkManagerPtr manager;
ShardPtr primary;
db->getChunkManagerOrPrimary( ns, manager, primary );
ChunkVersion currVersion;
if( manager ) currVersion = manager->getVersion();
LOG(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString()
<< " mine : " << currVersion.toString() << endl;
LOG(1) << msg.toString() << endl;
//
// We should reload only if we need to update our version to be compatible *and* we
// haven't already done so. This avoids lots of reloading when we remove/add a sharded collection
//
bool alreadyReloaded = lastNeededVersion &&
lastNeededVersion->isEquivalentTo( needVersion );
if( alreadyReloaded ){
LOG(1) << "wbl already reloaded config information for version "
<< needVersion << ", at version " << currVersion << endl;
}
else if( lastNeededVersion ) {
log() << "new version change detected to " << needVersion.toString()
<< ", " << lastNeededCount << " writebacks processed at "
<< lastNeededVersion->toString() << endl;
lastNeededCount = 0;
}
//
// Set our lastNeededVersion for next time
//
lastNeededVersion.reset( new ChunkVersion( needVersion ) );
lastNeededCount++;
//
// Determine if we should reload, if so, reload
//
bool shouldReload = ! needVersion.isWriteCompatibleWith( currVersion ) &&
! alreadyReloaded;
if( shouldReload && currVersion.isSet()
&& needVersion.isSet()
&& currVersion.hasCompatibleEpoch( needVersion ) )
{
//
// If we disagree about versions only, reload the chunk manager
//
db->getChunkManagerIfExists( ns, true );
}
else if( shouldReload ){
//
// If we disagree about anything else, reload the full db
//
warning() << "reloading config data for " << db->getName() << ", "
<< "wanted version " << needVersion.toString()
<< " but currently have version " << currVersion.toString() << endl;
db->reload();
}
// do request and then call getLastError
// we have to call getLastError so we can return the right fields to the user if they decide to call getLastError
BSONObj gle;
int attempts = 0;
while ( true ) {
attempts++;
try {
Request r( msg , 0 );
r.init();
r.d().reservedField() |= Reserved_FromWriteback;
ClientInfo * ci = r.getClientInfo();
if (!noauth) {
ci->getAuthorizationManager()->grantInternalAuthorization(
"_writebackListener");
}
ci->noAutoSplit();
r.process( attempts );
ci->newRequest(); // this so we flip prev and cur shards
BSONObjBuilder b;
string errmsg;
if ( ! ci->getLastError( "admin",
BSON( "getLastError" << 1 ),
b,
errmsg,
true ) )
{
b.appendBool( "commandFailed" , true );
if( ! b.hasField( "errmsg" ) ){
b.append( "errmsg", errmsg );
gle = b.obj();
}
else if( errmsg.size() > 0 ){
// Rebuild GLE object with errmsg
// TODO: Make this less clumsy by improving GLE interface
gle = b.obj();
if( gle["errmsg"].type() == String ){
BSONObj gleNoErrmsg =
gle.filterFieldsUndotted( BSON( "errmsg" << 1 ),
false );
BSONObjBuilder bb;
bb.appendElements( gleNoErrmsg );
bb.append( "errmsg", gle["errmsg"].String() +
" ::and:: " +
errmsg );
gle = bb.obj().getOwned();
}
}
}
else{
gle = b.obj();
}
if ( gle["code"].numberInt() == 9517 ) {
log() << "new version change detected, "
<< lastNeededCount << " writebacks processed previously" << endl;
lastNeededVersion.reset();
lastNeededCount = 1;
log() << "writeback failed because of stale config, retrying attempts: " << attempts << endl;
LOG(1) << "writeback error : " << gle << endl;
//
// Bringing this in line with the similar retry logic elsewhere
//
// TODO: Reloading the chunk manager may not help if we dropped a
// collection, but we don't actually have that info in the writeback
// error
//
if( attempts <= 2 ){
db->getChunkManagerIfExists( ns, true );
}
else{
versionManager.forceRemoteCheckShardVersionCB( ns );
sleepsecs( attempts - 1 );
}
uassert( 15884, str::stream()
<< "Could not reload chunk manager after "
<< attempts << " attempts.", attempts <= 4 );
continue;
}
ci->clearSinceLastGetError();
}
catch ( DBException& e ) {
error() << "error processing writeback: " << e << endl;
BSONObjBuilder b;
e.getInfo().append( b, "err", "code" );
gle = b.obj();
}
break;
}
{
scoped_lock lk( _seenWritebacksLock );
WBStatus& s = _seenWritebacks[cid];
s.id = wid;
s.gle = gle;
}
}
else if ( result["noop"].trueValue() ) {
// no-op
}
else {
log() << "unknown writeBack result: " << result << endl;
}
secsToSleep = 0;
continue;
}
catch ( std::exception& e ) {
// Attention! Do not call any method that would throw an exception
// (or assert) in this block.
if ( inShutdown() ) {
// we're shutting down, so just clean up
return;
}
log() << "WriteBackListener exception : " << e.what() << endl;
needsToReloadShardInfo = true;
}
catch ( ... ) {
log() << "WriteBackListener uncaught exception!" << endl;
}
secsToSleep++;
sleepsecs(secsToSleep);
if ( secsToSleep > 10 )
secsToSleep = 0;
}
log() << "WriteBackListener exiting : address no longer in cluster " << _addr;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase& conn_in , const string& ns , bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( conn_in );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = getVersionable( &conn_in );
assert(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManagerIfExists( ns , authoritative );
// It's possible the chunk manager was reset since we checked whether sharded was true,
// so must check this here.
if( manager ) officialSequenceNumber = manager->getSequenceNumber();
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie., last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = 0;
if ( isSharded && manager ) {
version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
}
if( version == 0 ){
LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
( ! isSharded ? "no longer sharded" :
( ! manager ? "no chunk manager found" :
"version is zero" ) ) << endl;
}
LOG(2) << " have to set shard version for conn: " << conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
checkShardVersion( *conn , ns , 1 , tryNumber + 1 );
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl;
sleepmillis( 10 * tryNumber );
checkShardVersion( *conn , ns , true , tryNumber + 1 );
return true;
}
string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
