void checkVersions( const string& ns ) {
vector<Shard> all;
Shard::getAllShards( all );
// Now only check top-level shard connections
for ( unsigned i=0; i<all.size(); i++ ) {
Shard& shard = all[i];
try {
string sconnString = shard.getConnString();
Status* s = _getStatus( sconnString );
if( ! s->avail ) {
s->avail = shardConnectionPool.get( sconnString );
s->created++; // After, so failed creation doesn't get counted
}
versionManager.checkShardVersionCB( s->avail, ns, false, 1 );
}
catch ( const std::exception& e ) {
warning() << "problem while initially checking shard versions on"
<< " " << shard.getName() << causedBy(e) << endl;
throw;
}
}
}
DBClientBase * get( const string& addr , const string& ns ) {
_check( ns );
Status* s = _getStatus( addr );
auto_ptr<DBClientBase> c; // Handles cleanup if there's an exception thrown
if ( s->avail ) {
c.reset( s->avail );
s->avail = 0;
shardConnectionPool.onHandedOut( c.get() ); // May throw an exception
} else {
c.reset( shardConnectionPool.get( addr ) );
s->created++; // After, so failed creation doesn't get counted
}
return c.release();
}
void checkVersions( const string& ns ) {
vector<Shard> all;
Shard::getAllShards( all );
// Don't report exceptions here as errors in GetLastError
LastError::Disabled ignoreForGLE(lastError.get(false));
// Now only check top-level shard connections
for ( unsigned i=0; i<all.size(); i++ ) {
Shard& shard = all[i];
try {
string sconnString = shard.getConnString();
Status* s = _getStatus( sconnString );
if( ! s->avail ) {
s->avail = shardConnectionPool.get( sconnString );
s->created++; // After, so failed creation doesn't get counted
}
versionManager.checkShardVersionCB( s->avail, ns, false, 1 );
}
catch ( const DBException& ex ) {
warning() << "problem while initially checking shard versions on" << " "
<< shard.getName() << causedBy( ex ) << endl;
// NOTE: This is only a heuristic, to avoid multiple stale version retries
// across multiple shards, and does not affect correctness.
}
}
}
virtual bool run( OperationContext* txn, const string&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder& result, bool ) {
// Base pool info
shardConnectionPool.appendInfo( result );
// Thread connection info
activeClientConnections.appendInfo( result );
return true;
}
void checkVersions( const string& ns ) {
vector<Shard> all;
Shard::getAllShards( all );
// Now only check top-level shard connections
for ( unsigned i=0; i<all.size(); i++ ) {
Shard& shard = all[i];
try {
string sconnString = shard.getConnString();
Status* &s = _hosts[sconnString];
if ( ! s ){
s = new Status();
}
if( ! s->avail )
s->avail = shardConnectionPool.get( sconnString );
versionManager.checkShardVersionCB( s->avail, ns, false, 1 );
} catch(...) {
LOGATMOST(2) << "exception in checkAllVersions shard:" << shard.getName() << endl;
throw;
}
}
}
DBClientBase * get( const string& addr , const string& ns, bool ignoreDirect = false ) {
_check( ns );
// Determine if non-shard conn is RS member for warning
// All shards added to _hosts if not present in _check()
if( ( logLevel >= 1 || ! printedShardConnWarning ) && ! ignoreDirect && _hosts.find( addr ) == _hosts.end() ){
vector<Shard> all;
Shard::getAllShards( all );
bool isRSMember = false;
string parentShard;
for ( unsigned i = 0; i < all.size(); i++ ) {
string connString = all[i].getConnString();
if( connString.find( addr ) != string::npos && connString.find( '/' ) != string::npos ){
isRSMember = true;
parentShard = connString;
break;
}
}
if( isRSMember ){
printedShardConnWarning = true;
warning() << "adding shard sub-connection " << addr << " (parent " << parentShard << ") as sharded, this is safe but unexpected" << endl;
printStackTrace();
}
}
Status* &s = _hosts[addr];
if ( ! s )
s = new Status();
if ( s->avail ) {
DBClientBase* c = s->avail;
s->avail = 0;
shardConnectionPool.onHandedOut( c );
return c;
}
s->created++;
return shardConnectionPool.get( addr );
}
DBClientBase * get( const string& addr , const string& ns ) {
_check( ns );
Status* &s = _hosts[addr];
if ( ! s )
s = new Status();
auto_ptr<DBClientBase> c; // Handles cleanup if there's an exception thrown
if ( s->avail ) {
c.reset( s->avail );
s->avail = 0;
shardConnectionPool.onHandedOut( c.get() ); // May throw an exception
} else {
s->created++;
c.reset( shardConnectionPool.get( addr ) );
}
if ( !noauth ) {
c->setAuthenticationTable( ClientBasic::getCurrent()->getAuthenticationInfo()->
getAuthTable() );
}
return c.release();
}
DBClientBase * get( const string& addr , const string& ns ) {
_check( ns );
Status* &s = _hosts[addr];
if ( ! s )
s = new Status();
if ( s->avail ) {
DBClientBase* c = s->avail;
s->avail = 0;
try {
shardConnectionPool.onHandedOut( c );
}
catch ( std::exception& e ) {
delete c;
throw;
}
return c;
}
s->created++;
return shardConnectionPool.get( addr );
}
bool DBConnectionPool::isConnectionGood(const string& hostName, DBClientBase* conn) {
if (conn == NULL) {
return false;
}
if (conn->isFailed()) {
return false;
}
{
scoped_lock sl(_mutex);
PoolForHost& pool = _pools[PoolKey(hostName, conn->getSoTimeout())];
if (pool.isBadSocketCreationTime(conn->getSockCreationMicroSec())) {
return false;
}
}
return true;
}
void done( const string& addr , DBClientBase* conn ) {
Status* s = _hosts[addr];
verify( s );
const bool isConnGood = shardConnectionPool.isConnectionGood(addr, conn);
if (s->avail != NULL) {
warning() << "Detected additional sharded connection in the "
"thread local pool for " << addr << endl;
if (DBException::traceExceptions) {
// There shouldn't be more than one connection checked out to the same
// host on the same thread.
printStackTrace();
}
if (!isConnGood) {
delete s->avail;
s->avail = NULL;
}
// Let the internal pool handle the bad connection, this can also
// update the lower bounds for the known good socket creation time
// for this host.
release(addr, conn);
return;
}
if (!isConnGood) {
// Let the internal pool handle the bad connection.
release(addr, conn);
return;
}
// Note: Although we try our best to clear bad connections as much as possible,
// some of them can still slip through because of how ClientConnections are being
// used - as thread local variables. This means that threads won't be able to
// see the s->avail connection of other threads.
s->avail = conn;
}
void checkVersions( const string& ns ) {
vector<Shard> all;
Shard::getAllShards( all );
// Now only check top-level shard connections
for ( unsigned i=0; i<all.size(); i++ ) {
string sconnString = all[i].getConnString();
Status* &s = _hosts[sconnString];
if ( ! s ){
s = new Status();
}
if( ! s->avail )
s->avail = shardConnectionPool.get( sconnString );
versionManager.checkShardVersionCB( s->avail, ns, false, 1 );
}
}
void release( const string& addr , DBClientBase * conn ) {
shardConnectionPool.release( addr , conn );
}
namespace mongo {
// The code in shardconnection may run not only in mongos context. When elsewhere, chunk shard versioning
// is disabled. To enable chunk shard versioning, provide the check/resetShardVerionCB's below
//
// TODO: better encapsulate this mechanism.
bool defaultIsVersionable( DBClientBase * conn ){
return false;
}
bool defaultInitShardVersion( DBClientBase & conn, BSONObj& result ){
return false;
}
bool defaultForceRemoteCheckShardVersion( const string& ns ){
return true;
}
bool defaultCheckShardVersion( DBClientBase & conn , const string& ns , bool authoritative , int tryNumber ) {
// no-op in mongod
return false;
}
void defaultResetShardVersion( DBClientBase * conn ) {
// no-op in mongod
}
boost::function1<bool, DBClientBase* > isVersionableCB = defaultIsVersionable;
boost::function2<bool, DBClientBase&, BSONObj& > initShardVersionCB = defaultInitShardVersion;
boost::function1<bool, const string& > forceRemoteCheckShardVersionCB = defaultForceRemoteCheckShardVersion;
boost::function4<bool, DBClientBase&, const string&, bool, int> checkShardVersionCB = defaultCheckShardVersion;
boost::function1<void, DBClientBase*> resetShardVersionCB = defaultResetShardVersion;
DBConnectionPool shardConnectionPool;
// Only print the non-top-level-shard-conn warning once if not verbose
volatile bool printedShardConnWarning = false;
/**
* holds all the actual db connections for a client to various servers
* 1 per thread, so doesn't have to be thread safe
*/
class ClientConnections : boost::noncopyable {
public:
struct Status : boost::noncopyable {
Status() : created(0), avail(0) {}
long long created;
DBClientBase* avail;
};
ClientConnections() {}
~ClientConnections() {
for ( HostMap::iterator i=_hosts.begin(); i!=_hosts.end(); ++i ) {
string addr = i->first;
Status* ss = i->second;
assert( ss );
if ( ss->avail ) {
/* if we're shutting down, don't want to initiate release mechanism as it is slow,
and isn't needed since all connections will be closed anyway */
if ( inShutdown() ) {
if( isVersionableCB( ss->avail ) ) resetShardVersionCB( ss->avail );
delete ss->avail;
}
else
release( addr , ss->avail );
ss->avail = 0;
}
delete ss;
}
_hosts.clear();
}
DBClientBase * get( const string& addr , const string& ns, bool ignoreDirect = false ) {
_check( ns );
// Determine if non-shard conn is RS member for warning
// All shards added to _hosts if not present in _check()
if( ( logLevel >= 1 || ! printedShardConnWarning ) && ! ignoreDirect && _hosts.find( addr ) == _hosts.end() ){
vector<Shard> all;
Shard::getAllShards( all );
bool isRSMember = false;
string parentShard;
for ( unsigned i = 0; i < all.size(); i++ ) {
string connString = all[i].getConnString();
if( connString.find( addr ) != string::npos && connString.find( '/' ) != string::npos ){
isRSMember = true;
parentShard = connString;
break;
}
}
if( isRSMember ){
printedShardConnWarning = true;
warning() << "adding shard sub-connection " << addr << " (parent " << parentShard << ") as sharded, this is safe but unexpected" << endl;
printStackTrace();
}
}
Status* &s = _hosts[addr];
if ( ! s )
s = new Status();
if ( s->avail ) {
DBClientBase* c = s->avail;
s->avail = 0;
shardConnectionPool.onHandedOut( c );
return c;
}
s->created++;
return shardConnectionPool.get( addr );
}
void done( const string& addr , DBClientBase* conn ) {
Status* s = _hosts[addr];
assert( s );
if ( s->avail ) {
release( addr , conn );
return;
}
s->avail = conn;
}
void sync() {
for ( HostMap::iterator i=_hosts.begin(); i!=_hosts.end(); ++i ) {
string addr = i->first;
Status* ss = i->second;
if ( ss->avail )
ss->avail->getLastError();
}
}
void checkVersions( const string& ns ) {
vector<Shard> all;
Shard::getAllShards( all );
// Now only check top-level shard connections
for ( unsigned i=0; i<all.size(); i++ ) {
string sconnString = all[i].getConnString();
Status* &s = _hosts[sconnString];
if ( ! s ){
s = new Status();
}
if( ! s->avail )
s->avail = shardConnectionPool.get( sconnString );
checkShardVersionCB( *s->avail, ns, false, 1 );
}
}
void release( const string& addr , DBClientBase * conn ) {
shardConnectionPool.release( addr , conn );
}
void _check( const string& ns ) {
if ( ns.size() == 0 || _seenNS.count( ns ) )
return;
_seenNS.insert( ns );
checkVersions( ns );
}
typedef map<string,Status*,DBConnectionPool::serverNameCompare> HostMap;
HostMap _hosts;
set<string> _seenNS;
// -----
static thread_specific_ptr<ClientConnections> _perThread;
static ClientConnections* threadInstance() {
ClientConnections* cc = _perThread.get();
if ( ! cc ) {
cc = new ClientConnections();
_perThread.reset( cc );
}
return cc;
}
};
thread_specific_ptr<ClientConnections> ClientConnections::_perThread;
ShardConnection::ShardConnection( const Shard * s , const string& ns, bool ignoreDirect )
: _addr( s->getConnString() ) , _ns( ns ) {
_init( ignoreDirect );
}
ShardConnection::ShardConnection( const Shard& s , const string& ns, bool ignoreDirect )
: _addr( s.getConnString() ) , _ns( ns ) {
_init( ignoreDirect );
}
ShardConnection::ShardConnection( const string& addr , const string& ns, bool ignoreDirect )
: _addr( addr ) , _ns( ns ) {
_init( ignoreDirect );
}
void ShardConnection::_init( bool ignoreDirect ) {
assert( _addr.size() );
_conn = ClientConnections::threadInstance()->get( _addr , _ns, ignoreDirect );
_finishedInit = false;
}
void ShardConnection::_finishInit() {
if ( _finishedInit )
return;
_finishedInit = true;
if ( _ns.size() && isVersionableCB( _conn ) ) {
_setVersion = checkShardVersionCB( *_conn , _ns , false , 1 );
}
else {
_setVersion = false;
}
}
void ShardConnection::done() {
if ( _conn ) {
ClientConnections::threadInstance()->done( _addr , _conn );
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::kill() {
if ( _conn ) {
if( isVersionableCB( _conn ) ) resetShardVersionCB( _conn );
delete _conn;
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::sync() {
ClientConnections::threadInstance()->sync();
}
bool ShardConnection::runCommand( const string& db , const BSONObj& cmd , BSONObj& res ) {
assert( _conn );
bool ok = _conn->runCommand( db , cmd , res );
if ( ! ok ) {
if ( res["code"].numberInt() == SendStaleConfigCode ) {
done();
throw RecvStaleConfigException( res["ns"].String() , res["errmsg"].String() );
}
}
return ok;
}
void ShardConnection::checkMyConnectionVersions( const string & ns ) {
ClientConnections::threadInstance()->checkVersions( ns );
}
ShardConnection::~ShardConnection() {
if ( _conn ) {
if ( ! _conn->isFailed() ) {
/* see done() comments above for why we log this line */
log() << "~ScopedDBConnection: _conn != null" << endl;
}
kill();
}
}
}
void ScopedDbConnection::clearPool() {
pool.clear();
}
namespace mongo {
DBConnectionPool shardConnectionPool;
class ClientConnections;
/**
* Class which tracks ClientConnections (the client connection pool) for each incoming
* connection, allowing stats access.
*/
class ActiveClientConnections {
public:
ActiveClientConnections() : _mutex( "ActiveClientConnections" ) {
}
void add( const ClientConnections* cc ) {
scoped_lock lock( _mutex );
_clientConnections.insert( cc );
}
void remove( const ClientConnections* cc ) {
scoped_lock lock( _mutex );
_clientConnections.erase( cc );
}
// Implemented after ClientConnections
void appendInfo( BSONObjBuilder& b );
private:
mongo::mutex _mutex;
set<const ClientConnections*> _clientConnections;
} activeClientConnections;
/**
* Command to allow access to the sharded conn pool information in mongos.
* TODO: Refactor with other connection pooling changes
*/
class ShardedPoolStats : public Command {
public:
ShardedPoolStats() : Command( "shardConnPoolStats" ) {}
virtual void help( stringstream &help ) const { help << "stats about the shard connection pool"; }
virtual LockType locktype() const { return NONE; }
virtual bool slaveOk() const { return true; }
// Same privs as connPoolStats
virtual void addRequiredPrivileges( const std::string& dbname,
const BSONObj& cmdObj,
std::vector<Privilege>* out )
{
ActionSet actions;
actions.addAction( ActionType::connPoolStats );
out->push_back( Privilege( AuthorizationManager::SERVER_RESOURCE_NAME, actions ) );
}
virtual bool run ( const string&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder& result, bool ) {
// Base pool info
shardConnectionPool.appendInfo( result );
// Thread connection info
activeClientConnections.appendInfo( result );
return true;
}
} shardedPoolStatsCmd;
/**
* holds all the actual db connections for a client to various servers
* 1 per thread, so doesn't have to be thread safe
*/
class ClientConnections : boost::noncopyable {
public:
struct Status : boost::noncopyable {
Status() : created(0), avail(0) {}
// May be read concurrently, but only written from
// this thread.
long long created;
DBClientBase* avail;
};
// Gets or creates the status object for the host
Status* _getStatus( const string& addr ) {
scoped_spinlock lock( _lock );
Status* &temp = _hosts[addr];
if ( ! temp )
temp = new Status();
return temp;
}
ClientConnections() {
// Start tracking client connections
activeClientConnections.add( this );
}
~ClientConnections() {
// Stop tracking these client connections
activeClientConnections.remove( this );
releaseAll( true );
}
void releaseAll( bool fromDestructor = false ) {
// Don't need spinlock protection because if not in the destructor, we don't
// modify _hosts, and if in the destructor we are not accessible to external
// threads.
for ( HostMap::iterator i=_hosts.begin(); i!=_hosts.end(); ++i ) {
string addr = i->first;
Status* ss = i->second;
verify( ss );
if ( ss->avail ) {
/* if we're shutting down, don't want to initiate release mechanism as it is slow,
and isn't needed since all connections will be closed anyway */
if ( inShutdown() ) {
if( versionManager.isVersionableCB( ss->avail ) ) versionManager.resetShardVersionCB( ss->avail );
delete ss->avail;
}
else
release( addr , ss->avail );
ss->avail = 0;
}
if ( fromDestructor ) delete ss;
}
if ( fromDestructor ) _hosts.clear();
}
DBClientBase * get( const string& addr , const string& ns ) {
_check( ns );
Status* s = _getStatus( addr );
auto_ptr<DBClientBase> c; // Handles cleanup if there's an exception thrown
if ( s->avail ) {
c.reset( s->avail );
s->avail = 0;
shardConnectionPool.onHandedOut( c.get() ); // May throw an exception
} else {
c.reset( shardConnectionPool.get( addr ) );
s->created++; // After, so failed creation doesn't get counted
}
return c.release();
}
void done( const string& addr , DBClientBase* conn ) {
Status* s = _hosts[addr];
verify( s );
const bool isConnGood = shardConnectionPool.isConnectionGood(addr, conn);
if (s->avail != NULL) {
warning() << "Detected additional sharded connection in the "
"thread local pool for " << addr << endl;
if (DBException::traceExceptions) {
// There shouldn't be more than one connection checked out to the same
// host on the same thread.
printStackTrace();
}
if (!isConnGood) {
delete s->avail;
s->avail = NULL;
}
// Let the internal pool handle the bad connection, this can also
// update the lower bounds for the known good socket creation time
// for this host.
release(addr, conn);
return;
}
if (!isConnGood) {
// Let the internal pool handle the bad connection.
release(addr, conn);
return;
}
// Note: Although we try our best to clear bad connections as much as possible,
// some of them can still slip through because of how ClientConnections are being
// used - as thread local variables. This means that threads won't be able to
// see the s->avail connection of other threads.
s->avail = conn;
}
void sync() {
for ( HostMap::iterator i=_hosts.begin(); i!=_hosts.end(); ++i ) {
string addr = i->first;
Status* ss = i->second;
if ( ss->avail )
ss->avail->getLastError();
}
}
void checkVersions( const string& ns ) {
vector<Shard> all;
Shard::getAllShards( all );
// Now only check top-level shard connections
for ( unsigned i=0; i<all.size(); i++ ) {
Shard& shard = all[i];
try {
string sconnString = shard.getConnString();
Status* s = _getStatus( sconnString );
if( ! s->avail ) {
s->avail = shardConnectionPool.get( sconnString );
s->created++; // After, so failed creation doesn't get counted
}
versionManager.checkShardVersionCB( s->avail, ns, false, 1 );
}
catch ( const std::exception& e ) {
warning() << "problem while initially checking shard versions on"
<< " " << shard.getName() << causedBy(e) << endl;
throw;
}
}
}
void release( const string& addr , DBClientBase * conn ) {
shardConnectionPool.release( addr , conn );
}
void _check( const string& ns ) {
{
// We want to report ns stats too
scoped_spinlock lock( _lock );
if ( ns.size() == 0 || _seenNS.count( ns ) )
return;
_seenNS.insert( ns );
}
checkVersions( ns );
}
/**
* Appends info about the client connection pool to a BOBuilder
* Safe to call with activeClientConnections lock
*/
void appendInfo( BSONObjBuilder& b ) const {
scoped_spinlock lock( _lock );
BSONArrayBuilder hostsArrB( b.subarrayStart( "hosts" ) );
for ( HostMap::const_iterator i = _hosts.begin(); i != _hosts.end(); ++i ) {
BSONObjBuilder bb( hostsArrB.subobjStart() );
bb.append( "host", i->first );
bb.append( "created", i->second->created );
bb.appendBool( "avail", static_cast<bool>( i->second->avail ) );
bb.done();
}
hostsArrB.done();
BSONArrayBuilder nsArrB( b.subarrayStart( "seenNS" ) );
for ( set<string>::const_iterator i = _seenNS.begin(); i != _seenNS.end(); ++i ) {
nsArrB.append(*i);
}
nsArrB.done();
}
// Protects only the creation of new entries in the _hosts and _seenNS map
// from external threads. Reading _hosts / _seenNS in this thread doesn't
// need protection.
mutable SpinLock _lock;
typedef map<string,Status*,DBConnectionPool::serverNameCompare> HostMap;
HostMap _hosts;
set<string> _seenNS;
/**
* Clears the connections kept by this pool (ie, not including the global pool)
*/
void clearPool() {
for(HostMap::iterator iter = _hosts.begin(); iter != _hosts.end(); ++iter) {
if (iter->second->avail != NULL) {
delete iter->second->avail;
}
}
_hosts.clear();
}
void forgetNS( const string& ns ) {
scoped_spinlock lock( _lock );
_seenNS.erase( ns );
}
// -----
static thread_specific_ptr<ClientConnections> _perThread;
static ClientConnections* threadInstance() {
ClientConnections* cc = _perThread.get();
if ( ! cc ) {
cc = new ClientConnections();
_perThread.reset( cc );
}
return cc;
}
};
thread_specific_ptr<ClientConnections> ClientConnections::_perThread;
/**
* Appends info about all active client shard connections to a BOBuilder
*/
void ActiveClientConnections::appendInfo( BSONObjBuilder& b ) {
BSONArrayBuilder arr( 64 * 1024 ); // There may be quite a few threads
{
scoped_lock lock( _mutex );
for ( set<const ClientConnections*>::const_iterator i = _clientConnections.begin();
i != _clientConnections.end(); ++i )
{
BSONObjBuilder bb( arr.subobjStart() );
(*i)->appendInfo( bb );
bb.done();
}
}
b.appendArray( "threads", arr.obj() );
}
ShardConnection::ShardConnection( const Shard * s , const string& ns, ChunkManagerPtr manager )
: _addr( s->getConnString() ) , _ns( ns ), _manager( manager ) {
_init();
}
ShardConnection::ShardConnection( const Shard& s , const string& ns, ChunkManagerPtr manager )
: _addr( s.getConnString() ) , _ns( ns ), _manager( manager ) {
_init();
}
ShardConnection::ShardConnection( const string& addr , const string& ns, ChunkManagerPtr manager )
: _addr( addr ) , _ns( ns ), _manager( manager ) {
_init();
}
void usingAShardConnection( const string& addr );
void ShardConnection::_init() {
verify( _addr.size() );
_conn = ClientConnections::threadInstance()->get( _addr , _ns );
_finishedInit = false;
usingAShardConnection( _addr );
}
void ShardConnection::_finishInit() {
if ( _finishedInit )
return;
_finishedInit = true;
if ( _ns.size() && versionManager.isVersionableCB( _conn ) ) {
// Make sure we specified a manager for the correct namespace
if( _manager ) verify( _manager->getns() == _ns );
_setVersion = versionManager.checkShardVersionCB( this , false , 1 );
}
else {
// Make sure we didn't specify a manager for an empty namespace
verify( ! _manager );
_setVersion = false;
}
}
void ShardConnection::done() {
if ( _conn ) {
ClientConnections::threadInstance()->done( _addr , _conn );
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::kill() {
if ( _conn ) {
if( versionManager.isVersionableCB( _conn ) ) versionManager.resetShardVersionCB( _conn );
if (_conn->isFailed()) {
// Let the pool know about the bad connection and also delegate disposal to it.
ClientConnections::threadInstance()->done(_addr, _conn);
}
else {
delete _conn;
}
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::sync() {
ClientConnections::threadInstance()->sync();
}
bool ShardConnection::runCommand( const string& db , const BSONObj& cmd , BSONObj& res ) {
verify( _conn );
bool ok = _conn->runCommand( db , cmd , res );
if ( ! ok ) {
if ( res["code"].numberInt() == SendStaleConfigCode ) {
done();
throw RecvStaleConfigException( res["errmsg"].String(), res );
}
}
return ok;
}
void ShardConnection::checkMyConnectionVersions( const string & ns ) {
ClientConnections::threadInstance()->checkVersions( ns );
}
ShardConnection::~ShardConnection() {
if ( _conn ) {
if (_conn->isFailed()) {
if (_conn->getSockCreationMicroSec() ==
DBClientBase::INVALID_SOCK_CREATION_TIME) {
kill();
}
else {
// The pool takes care of deleting the failed connection - this
// will also trigger disposal of older connections in the pool
done();
}
}
else {
/* see done() comments above for why we log this line */
log() << "sharded connection to " << _conn->getServerAddress()
<< " not being returned to the pool" << endl;
kill();
}
}
}
bool ShardConnection::releaseConnectionsAfterResponse( false );
ExportedServerParameter<bool> ReleaseConnectionsAfterResponse(
ServerParameterSet::getGlobal(),
"releaseConnectionsAfterResponse",
&ShardConnection::releaseConnectionsAfterResponse,
true,
true
);
void ShardConnection::releaseMyConnections() {
ClientConnections::threadInstance()->releaseAll();
}
void ShardConnection::clearPool() {
shardConnectionPool.clear();
ClientConnections::threadInstance()->clearPool();
}
void ShardConnection::forgetNS( const string& ns ) {
ClientConnections::threadInstance()->forgetNS( ns );
}
}
namespace mongo {
using std::unique_ptr;
using std::map;
using std::set;
using std::string;
using std::stringstream;
using std::vector;
namespace {
class ClientConnections;
/**
* Class which tracks ClientConnections (the client connection pool) for each incoming
* connection, allowing stats access.
*/
class ActiveClientConnections {
public:
void add(const ClientConnections* cc) {
stdx::lock_guard<stdx::mutex> lock(_mutex);
_clientConnections.insert(cc);
}
void remove(const ClientConnections* cc) {
stdx::lock_guard<stdx::mutex> lock(_mutex);
_clientConnections.erase(cc);
}
void appendInfo(BSONObjBuilder& b);
private:
stdx::mutex _mutex;
set<const ClientConnections*> _clientConnections;
} activeClientConnections;
/**
* Command to allow access to the sharded conn pool information in mongos.
*/
class ShardedPoolStats : public Command {
public:
ShardedPoolStats() : Command("shardConnPoolStats") {}
virtual void help(stringstream& help) const {
help << "stats about the shard connection pool";
}
virtual bool isWriteCommandForConfigServer() const {
return false;
}
virtual bool slaveOk() const {
return true;
}
// Same privs as connPoolStats
virtual void addRequiredPrivileges(const std::string& dbname,
const BSONObj& cmdObj,
std::vector<Privilege>* out) {
ActionSet actions;
actions.addAction(ActionType::connPoolStats);
out->push_back(Privilege(ResourcePattern::forClusterResource(), actions));
}
virtual bool run(OperationContext* txn,
const string& dbname,
mongo::BSONObj& cmdObj,
int options,
std::string& errmsg,
mongo::BSONObjBuilder& result) {
// Base pool info
shardConnectionPool.appendInfo(result);
// Thread connection info
activeClientConnections.appendInfo(result);
return true;
}
} shardedPoolStatsCmd;
/**
* holds all the actual db connections for a client to various servers 1 per thread, so
* doesn't have to be thread safe.
*/
class ClientConnections {
MONGO_DISALLOW_COPYING(ClientConnections);
public:
struct Status {
Status() : created(0), avail(0) {}
// May be read concurrently, but only written from
// this thread.
long long created;
DBClientBase* avail;
};
// Gets or creates the status object for the host
Status* _getStatus(const string& addr) {
scoped_spinlock lock(_lock);
Status*& temp = _hosts[addr];
if (!temp) {
temp = new Status();
}
return temp;
}
ClientConnections() {
// Start tracking client connections
activeClientConnections.add(this);
}
~ClientConnections() {
// Stop tracking these client connections
activeClientConnections.remove(this);
releaseAll(true);
}
void releaseAll(bool fromDestructor = false) {
// Don't need spinlock protection because if not in the destructor, we don't modify
// _hosts, and if in the destructor we are not accessible to external threads.
for (HostMap::iterator i = _hosts.begin(); i != _hosts.end(); ++i) {
const string addr = i->first;
Status* ss = i->second;
invariant(ss);
if (ss->avail) {
// If we're shutting down, don't want to initiate release mechanism as it is
// slow, and isn't needed since all connections will be closed anyway.
if (inShutdown()) {
if (versionManager.isVersionableCB(ss->avail)) {
versionManager.resetShardVersionCB(ss->avail);
}
delete ss->avail;
} else {
release(addr, ss->avail);
}
ss->avail = 0;
}
if (fromDestructor) {
delete ss;
}
}
if (fromDestructor) {
_hosts.clear();
}
}
DBClientBase* get(const string& addr, const string& ns) {
{
// We want to report ns stats
scoped_spinlock lock(_lock);
if (ns.size() > 0)
_seenNS.insert(ns);
}
Status* s = _getStatus(addr);
unique_ptr<DBClientBase> c;
if (s->avail) {
c.reset(s->avail);
s->avail = 0;
// May throw an exception
shardConnectionPool.onHandedOut(c.get());
} else {
c.reset(shardConnectionPool.get(addr));
// After, so failed creation doesn't get counted
s->created++;
}
return c.release();
}
void done(const string& addr, DBClientBase* conn) {
Status* s = _hosts[addr];
verify(s);
const bool isConnGood = shardConnectionPool.isConnectionGood(addr, conn);
if (s->avail != NULL) {
warning() << "Detected additional sharded connection in the "
<< "thread local pool for " << addr;
if (DBException::traceExceptions) {
// There shouldn't be more than one connection checked out to the same
// host on the same thread.
printStackTrace();
}
if (!isConnGood) {
delete s->avail;
s->avail = NULL;
}
// Let the internal pool handle the bad connection, this can also
// update the lower bounds for the known good socket creation time
// for this host.
release(addr, conn);
return;
}
if (!isConnGood) {
// Let the internal pool handle the bad connection.
release(addr, conn);
return;
}
// Note: Although we try our best to clear bad connections as much as possible,
// some of them can still slip through because of how ClientConnections are being
// used - as thread local variables. This means that threads won't be able to
// see the s->avail connection of other threads.
s->avail = conn;
}
void sync() {
for (HostMap::iterator i = _hosts.begin(); i != _hosts.end(); ++i) {
string addr = i->first;
Status* ss = i->second;
if (ss->avail)
ss->avail->getLastError();
}
}
void checkVersions(OperationContext* txn, const string& ns) {
vector<ShardId> all;
grid.shardRegistry()->getAllShardIds(&all);
// Don't report exceptions here as errors in GetLastError
LastError::Disabled ignoreForGLE(&LastError::get(cc()));
// Now only check top-level shard connections
for (const ShardId& shardId : all) {
try {
const auto shard = grid.shardRegistry()->getShard(txn, shardId);
if (!shard) {
continue;
}
string sconnString = shard->getConnString().toString();
Status* s = _getStatus(sconnString);
if (!s->avail) {
s->avail = shardConnectionPool.get(sconnString);
s->created++; // After, so failed creation doesn't get counted
}
versionManager.checkShardVersionCB(txn, s->avail, ns, false, 1);
} catch (const DBException& ex) {
warning() << "problem while initially checking shard versions on"
<< " " << shardId << causedBy(ex);
// NOTE: This is only a heuristic, to avoid multiple stale version retries
// across multiple shards, and does not affect correctness.
}
}
}
void release(const string& addr, DBClientBase* conn) {
shardConnectionPool.release(addr, conn);
}
/**
* Appends info about the client connection pool to a BOBuilder
* Safe to call with activeClientConnections lock
*/
void appendInfo(BSONObjBuilder& b) const {
scoped_spinlock lock(_lock);
BSONArrayBuilder hostsArrB(b.subarrayStart("hosts"));
for (HostMap::const_iterator i = _hosts.begin(); i != _hosts.end(); ++i) {
BSONObjBuilder bb(hostsArrB.subobjStart());
bb.append("host", i->first);
bb.append("created", i->second->created);
bb.appendBool("avail", static_cast<bool>(i->second->avail));
bb.done();
}
hostsArrB.done();
BSONArrayBuilder nsArrB(b.subarrayStart("seenNS"));
for (set<string>::const_iterator i = _seenNS.begin(); i != _seenNS.end(); ++i) {
nsArrB.append(*i);
}
nsArrB.done();
}
// Protects only the creation of new entries in the _hosts and _seenNS map
// from external threads. Reading _hosts / _seenNS in this thread doesn't
// need protection.
mutable SpinLock _lock;
typedef map<string, Status*, DBConnectionPool::serverNameCompare> HostMap;
HostMap _hosts;
set<string> _seenNS;
/**
* Clears the connections kept by this pool (ie, not including the global pool)
*/
void clearPool() {
for (HostMap::iterator iter = _hosts.begin(); iter != _hosts.end(); ++iter) {
if (iter->second->avail != NULL) {
delete iter->second->avail;
}
delete iter->second;
}
_hosts.clear();
}
void forgetNS(const string& ns) {
scoped_spinlock lock(_lock);
_seenNS.erase(ns);
}
// -----
static thread_specific_ptr<ClientConnections> _perThread;
static ClientConnections* threadInstance() {
ClientConnections* cc = _perThread.get();
if (!cc) {
cc = new ClientConnections();
_perThread.reset(cc);
}
return cc;
}
};
void ActiveClientConnections::appendInfo(BSONObjBuilder& b) {
BSONArrayBuilder arr(64 * 1024); // There may be quite a few threads
{
stdx::lock_guard<stdx::mutex> lock(_mutex);
for (set<const ClientConnections*>::const_iterator i = _clientConnections.begin();
i != _clientConnections.end();
++i) {
BSONObjBuilder bb(arr.subobjStart());
(*i)->appendInfo(bb);
bb.done();
}
}
b.appendArray("threads", arr.obj());
}
thread_specific_ptr<ClientConnections> ClientConnections::_perThread;
} // namespace
// The global connection pool
DBConnectionPool shardConnectionPool;
// Different between mongos and mongod
void usingAShardConnection(const string& addr);
ShardConnection::ShardConnection(const ConnectionString& connectionString,
const string& ns,
std::shared_ptr<ChunkManager> manager)
: _cs(connectionString), _ns(ns), _manager(manager), _finishedInit(false) {
invariant(_cs.isValid());
// Make sure we specified a manager for the correct namespace
if (_ns.size() && _manager) {
invariant(_manager->getns() == _ns);
}
_conn = ClientConnections::threadInstance()->get(_cs.toString(), _ns);
usingAShardConnection(_cs.toString());
}
ShardConnection::~ShardConnection() {
if (_conn) {
if (_conn->isFailed()) {
if (_conn->getSockCreationMicroSec() == DBClientBase::INVALID_SOCK_CREATION_TIME) {
kill();
} else {
// The pool takes care of deleting the failed connection - this
// will also trigger disposal of older connections in the pool
done();
}
} else {
// see done() comments above for why we log this line
log() << "sharded connection to " << _conn->getServerAddress()
<< " not being returned to the pool";
kill();
}
}
}
void ShardConnection::_finishInit() {
if (_finishedInit)
return;
_finishedInit = true;
if (versionManager.isVersionableCB(_conn)) {
auto& client = cc();
auto txn = client.getOperationContext();
invariant(txn);
_setVersion = versionManager.checkShardVersionCB(txn, this, false, 1);
} else {
// Make sure we didn't specify a manager for a non-versionable connection (i.e. config)
verify(!_manager);
_setVersion = false;
}
}
void ShardConnection::done() {
if (_conn) {
ClientConnections::threadInstance()->done(_cs.toString(), _conn);
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::kill() {
if (_conn) {
if (versionManager.isVersionableCB(_conn)) {
versionManager.resetShardVersionCB(_conn);
}
if (_conn->isFailed()) {
// Let the pool know about the bad connection and also delegate disposal to it.
ClientConnections::threadInstance()->done(_cs.toString(), _conn);
} else {
delete _conn;
}
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::sync() {
ClientConnections::threadInstance()->sync();
}
void ShardConnection::checkMyConnectionVersions(OperationContext* txn, const string& ns) {
ClientConnections::threadInstance()->checkVersions(txn, ns);
}
void ShardConnection::releaseMyConnections() {
ClientConnections::threadInstance()->releaseAll();
}
void ShardConnection::clearPool() {
shardConnectionPool.clear();
ClientConnections::threadInstance()->clearPool();
}
void ShardConnection::forgetNS(const string& ns) {
ClientConnections::threadInstance()->forgetNS(ns);
}
} // namespace mongo
namespace mongo {
DBConnectionPool pool;
DBClientBase* DBConnectionPool::get(const string& host) {
boostlock L(poolMutex);
PoolForHost *&p = pools[host];
if ( p == 0 )
p = new PoolForHost();
if ( p->pool.empty() ) {
string errmsg;
DBClientBase *c;
if( host.find(',') == string::npos ) {
DBClientConnection *cc = new DBClientConnection(true);
if ( !cc->connect(host.c_str(), errmsg) ) {
delete cc;
uassert( (string)"dbconnectionpool: connect failed" + host , false);
return 0;
}
c = cc;
}
else {
DBClientPaired *p = new DBClientPaired();
if( !p->connect(host) ) {
delete p;
uassert( (string)"dbconnectionpool: connect failed [2] " + host , false);
return 0;
}
c = p;
}
return c;
}
DBClientBase *c = p->pool.front();
p->pool.pop();
return c;
}
void DBConnectionPool::flush(){
boostlock L(poolMutex);
for ( map<string,PoolForHost*>::iterator i = pools.begin(); i != pools.end(); i++ ){
PoolForHost* p = i->second;
vector<DBClientBase*> all;
while ( ! p->pool.empty() ){
DBClientBase * c = p->pool.front();
p->pool.pop();
all.push_back( c );
bool res;
c->isMaster( res );
}
for ( vector<DBClientBase*>::iterator i=all.begin(); i != all.end(); i++ ){
p->pool.push( *i );
}
}
}
class PoolFlushCmd : public Command {
public:
PoolFlushCmd() : Command( "connpoolsync" ){}
virtual bool run(const char*, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool){
pool.flush();
result << "ok" << 1;
return true;
}
virtual bool slaveOk(){
return true;
}
} poolFlushCmd;
} // namespace mongo
namespace mongo {
// ------ PoolForHost ------
PoolForHost::~PoolForHost() {
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
delete sc.conn;
_pool.pop();
}
}
void PoolForHost::done( DBConnectionPool * pool, DBClientBase * c ) {
if ( _pool.size() >= _maxPerHost ) {
pool->onDestroy( c );
delete c;
}
else {
_pool.push(c);
}
}
DBClientBase * PoolForHost::get( DBConnectionPool * pool , double socketTimeout ) {
time_t now = time(0);
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
_pool.pop();
if ( ! sc.ok( now ) ) {
pool->onDestroy( sc.conn );
delete sc.conn;
continue;
}
assert( sc.conn->getSoTimeout() == socketTimeout );
return sc.conn;
}
return NULL;
}
void PoolForHost::flush() {
vector<StoredConnection> all;
while ( ! _pool.empty() ) {
StoredConnection c = _pool.top();
_pool.pop();
all.push_back( c );
bool res;
c.conn->isMaster( res );
}
for ( vector<StoredConnection>::iterator i=all.begin(); i != all.end(); ++i ) {
_pool.push( *i );
}
}
void PoolForHost::getStaleConnections( vector<DBClientBase*>& stale ) {
time_t now = time(0);
vector<StoredConnection> all;
while ( ! _pool.empty() ) {
StoredConnection c = _pool.top();
_pool.pop();
if ( c.ok( now ) )
all.push_back( c );
else
stale.push_back( c.conn );
}
for ( size_t i=0; i<all.size(); i++ ) {
_pool.push( all[i] );
}
}
PoolForHost::StoredConnection::StoredConnection( DBClientBase * c ) {
conn = c;
when = time(0);
}
bool PoolForHost::StoredConnection::ok( time_t now ) {
// if connection has been idle for 30 minutes, kill it
return ( now - when ) < 1800;
}
void PoolForHost::createdOne( DBClientBase * base) {
if ( _created == 0 )
_type = base->type();
_created++;
}
unsigned PoolForHost::_maxPerHost = 50;
// ------ DBConnectionPool ------
DBConnectionPool pool;
DBConnectionPool::DBConnectionPool()
: _mutex("DBConnectionPool") ,
_name( "dbconnectionpool" ) ,
_hooks( new list<DBConnectionHook*>() ) {
}
DBClientBase* DBConnectionPool::_get(const string& ident , double socketTimeout ) {
assert( ! inShutdown() );
scoped_lock L(_mutex);
PoolForHost& p = _pools[PoolKey(ident,socketTimeout)];
return p.get( this , socketTimeout );
}
DBClientBase* DBConnectionPool::_finishCreate( const string& host , double socketTimeout , DBClientBase* conn ) {
{
scoped_lock L(_mutex);
PoolForHost& p = _pools[PoolKey(host,socketTimeout)];
p.createdOne( conn );
}
try {
onCreate( conn );
onHandedOut( conn );
}
catch ( std::exception& e ) {
delete conn;
throw;
}
return conn;
}
DBClientBase* DBConnectionPool::get(const ConnectionString& url, double socketTimeout) {
DBClientBase * c = _get( url.toString() , socketTimeout );
if ( c ) {
try {
onHandedOut( c );
}
catch ( std::exception& e ) {
delete c;
throw;
}
return c;
}
string errmsg;
c = url.connect( errmsg, socketTimeout );
uassert( 13328 , _name + ": connect failed " + url.toString() + " : " + errmsg , c );
return _finishCreate( url.toString() , socketTimeout , c );
}
DBClientBase* DBConnectionPool::get(const string& host, double socketTimeout) {
DBClientBase * c = _get( host , socketTimeout );
if ( c ) {
try {
onHandedOut( c );
}
catch ( std::exception& e ) {
delete c;
throw;
}
return c;
}
string errmsg;
ConnectionString cs = ConnectionString::parse( host , errmsg );
uassert( 13071 , (string)"invalid hostname [" + host + "]" + errmsg , cs.isValid() );
c = cs.connect( errmsg, socketTimeout );
if ( ! c )
throw SocketException( SocketException::CONNECT_ERROR , host , 11002 , str::stream() << _name << " error: " << errmsg );
return _finishCreate( host , socketTimeout , c );
}
void DBConnectionPool::release(const string& host, DBClientBase *c) {
if ( c->isFailed() ) {
onDestroy( c );
delete c;
return;
}
scoped_lock L(_mutex);
_pools[PoolKey(host,c->getSoTimeout())].done(this,c);
}
DBConnectionPool::~DBConnectionPool() {
// connection closing is handled by ~PoolForHost
}
void DBConnectionPool::flush() {
scoped_lock L(_mutex);
for ( PoolMap::iterator i = _pools.begin(); i != _pools.end(); i++ ) {
PoolForHost& p = i->second;
p.flush();
}
}
void DBConnectionPool::addHook( DBConnectionHook * hook ) {
_hooks->push_back( hook );
}
void DBConnectionPool::onCreate( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onCreate( conn );
}
}
void DBConnectionPool::onHandedOut( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onHandedOut( conn );
}
}
void DBConnectionPool::onDestroy( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onDestroy( conn );
}
}
void DBConnectionPool::appendInfo( BSONObjBuilder& b ) {
int avail = 0;
long long created = 0;
map<ConnectionString::ConnectionType,long long> createdByType;
set<string> replicaSets;
BSONObjBuilder bb( b.subobjStart( "hosts" ) );
{
scoped_lock lk( _mutex );
for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
if ( i->second.numCreated() == 0 )
continue;
string s = str::stream() << i->first.ident << "::" << i->first.timeout;
BSONObjBuilder temp( bb.subobjStart( s ) );
temp.append( "available" , i->second.numAvailable() );
temp.appendNumber( "created" , i->second.numCreated() );
temp.done();
avail += i->second.numAvailable();
created += i->second.numCreated();
long long& x = createdByType[i->second.type()];
x += i->second.numCreated();
{
string setName = i->first.ident;
if ( setName.find( "/" ) != string::npos ) {
setName = setName.substr( 0 , setName.find( "/" ) );
replicaSets.insert( setName );
}
}
}
}
bb.done();
BSONObjBuilder setBuilder( b.subobjStart( "replicaSets" ) );
for ( set<string>::iterator i=replicaSets.begin(); i!=replicaSets.end(); ++i ) {
string rs = *i;
ReplicaSetMonitorPtr m = ReplicaSetMonitor::get( rs );
if ( ! m ) {
warning() << "no monitor for set: " << rs << endl;
continue;
}
BSONObjBuilder temp( setBuilder.subobjStart( rs ) );
m->appendInfo( temp );
temp.done();
}
setBuilder.done();
{
BSONObjBuilder temp( bb.subobjStart( "createdByType" ) );
for ( map<ConnectionString::ConnectionType,long long>::iterator i=createdByType.begin(); i!=createdByType.end(); ++i ) {
temp.appendNumber( ConnectionString::typeToString( i->first ) , i->second );
}
temp.done();
}
b.append( "totalAvailable" , avail );
b.appendNumber( "totalCreated" , created );
}
bool DBConnectionPool::serverNameCompare::operator()( const string& a , const string& b ) const{
const char* ap = a.c_str();
const char* bp = b.c_str();
while (true){
if (*ap == '\0' || *ap == '/'){
if (*bp == '\0' || *bp == '/')
return false; // equal strings
else
return true; // a is shorter
}
if (*bp == '\0' || *bp == '/')
return false; // b is shorter
if ( *ap < *bp)
return true;
else if (*ap > *bp)
return false;
++ap;
++bp;
}
assert(false);
}
bool DBConnectionPool::poolKeyCompare::operator()( const PoolKey& a , const PoolKey& b ) const {
if (DBConnectionPool::serverNameCompare()( a.ident , b.ident ))
return true;
if (DBConnectionPool::serverNameCompare()( b.ident , a.ident ))
return false;
return a.timeout < b.timeout;
}
void DBConnectionPool::taskDoWork() {
vector<DBClientBase*> toDelete;
{
// we need to get the connections inside the lock
// but we can actually delete them outside
scoped_lock lk( _mutex );
for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
i->second.getStaleConnections( toDelete );
}
}
for ( size_t i=0; i<toDelete.size(); i++ ) {
try {
onDestroy( toDelete[i] );
delete toDelete[i];
}
catch ( ... ) {
// we don't care if there was a socket error
}
}
}
// ------ ScopedDbConnection ------
ScopedDbConnection * ScopedDbConnection::steal() {
assert( _conn );
ScopedDbConnection * n = new ScopedDbConnection( _host , _conn, _socketTimeout );
_conn = 0;
return n;
}
void ScopedDbConnection::_setSocketTimeout(){
if( ! _conn ) return;
if( _conn->type() == ConnectionString::MASTER )
(( DBClientConnection* ) _conn)->setSoTimeout( _socketTimeout );
else if( _conn->type() == ConnectionString::SYNC )
(( SyncClusterConnection* ) _conn)->setAllSoTimeouts( _socketTimeout );
}
ScopedDbConnection::~ScopedDbConnection() {
if ( _conn ) {
if ( ! _conn->isFailed() ) {
/* see done() comments above for why we log this line */
log() << "~ScopedDbConnection: _conn != null" << endl;
}
kill();
}
}
ScopedDbConnection::ScopedDbConnection(const Shard& shard, double socketTimeout )
: _host( shard.getConnString() ) , _conn( pool.get(_host, socketTimeout) ), _socketTimeout( socketTimeout ) {
_setSocketTimeout();
}
ScopedDbConnection::ScopedDbConnection(const Shard* shard, double socketTimeout )
: _host( shard->getConnString() ) , _conn( pool.get(_host, socketTimeout) ), _socketTimeout( socketTimeout ) {
_setSocketTimeout();
}
AtomicUInt AScopedConnection::_numConnections;
} // namespace mongo
ScopedDbConnection::ScopedDbConnection(const Shard* shard, double socketTimeout )
: _host( shard->getConnString() ) , _conn( pool.get(_host, socketTimeout) ), _socketTimeout( socketTimeout ) {
_setSocketTimeout();
}
ScopedDbConnection::ScopedDbConnection(const Shard* shard )
: _host( shard->getConnString() ) , _conn( pool.get(_host) ){
}
virtual bool run(const string&, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool) {
pool.appendInfo( result );
result.append( "numDBClientConnection" , DBClientConnection::getNumConnections() );
result.append( "numAScopedConnection" , AScopedConnection::getNumConnections() );
return true;
}
virtual bool run(const string&, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool){
pool.appendInfo( result );
return true;
}
namespace mongo {
// ------ PoolForHost ------
PoolForHost::~PoolForHost(){
while ( ! _pool.empty() ){
StoredConnection sc = _pool.top();
delete sc.conn;
_pool.pop();
}
}
void PoolForHost::done( DBClientBase * c ) {
_pool.push(c);
}
DBClientBase * PoolForHost::get() {
time_t now = time(0);
while ( ! _pool.empty() ){
StoredConnection sc = _pool.top();
_pool.pop();
if ( sc.ok( now ) )
return sc.conn;
delete sc.conn;
}
return NULL;
}
void PoolForHost::flush() {
vector<StoredConnection> all;
while ( ! _pool.empty() ){
StoredConnection c = _pool.top();
_pool.pop();
all.push_back( c );
bool res;
c.conn->isMaster( res );
}
for ( vector<StoredConnection>::iterator i=all.begin(); i != all.end(); ++i ){
_pool.push( *i );
}
}
PoolForHost::StoredConnection::StoredConnection( DBClientBase * c ){
conn = c;
when = time(0);
}
bool PoolForHost::StoredConnection::ok( time_t now ){
// if connection has been idle for an hour, kill it
return ( now - when ) < 3600;
}
// ------ DBConnectionPool ------
DBConnectionPool pool;
DBClientBase* DBConnectionPool::_get(const string& ident) {
scoped_lock L(_mutex);
PoolForHost& p = _pools[ident];
return p.get();
}
DBClientBase* DBConnectionPool::_finishCreate( const string& host , DBClientBase* conn ){
{
scoped_lock L(_mutex);
PoolForHost& p = _pools[host];
p.createdOne();
}
onCreate( conn );
onHandedOut( conn );
return conn;
}
DBClientBase* DBConnectionPool::get(const ConnectionString& url) {
DBClientBase * c = _get( url.toString() );
if ( c ){
onHandedOut( c );
return c;
}
string errmsg;
c = url.connect( errmsg );
uassert( 13328 , _name + ": connect failed " + url.toString() + " : " + errmsg , c );
return _finishCreate( url.toString() , c );
}
DBClientBase* DBConnectionPool::get(const string& host) {
DBClientBase * c = _get( host );
if ( c ){
onHandedOut( c );
return c;
}
string errmsg;
ConnectionString cs = ConnectionString::parse( host , errmsg );
uassert( 13071 , (string)"invalid hostname [" + host + "]" + errmsg , cs.isValid() );
c = cs.connect( errmsg );
uassert( 11002 , _name + ": connect failed " + host + " : " + errmsg , c );
return _finishCreate( host , c );
}
DBConnectionPool::~DBConnectionPool(){
// connection closing is handled by ~PoolForHost
}
void DBConnectionPool::flush(){
scoped_lock L(_mutex);
for ( map<string,PoolForHost>::iterator i = _pools.begin(); i != _pools.end(); i++ ){
PoolForHost& p = i->second;
p.flush();
}
}
void DBConnectionPool::addHook( DBConnectionHook * hook ){
_hooks.push_back( hook );
}
void DBConnectionPool::onCreate( DBClientBase * conn ){
if ( _hooks.size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks.begin(); i != _hooks.end(); i++ ){
(*i)->onCreate( conn );
}
}
void DBConnectionPool::onHandedOut( DBClientBase * conn ){
if ( _hooks.size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks.begin(); i != _hooks.end(); i++ ){
(*i)->onHandedOut( conn );
}
}
void DBConnectionPool::appendInfo( BSONObjBuilder& b ){
scoped_lock lk( _mutex );
BSONObjBuilder bb( b.subobjStart( "hosts" ) );
for ( map<string,PoolForHost>::iterator i=_pools.begin(); i!=_pools.end(); ++i ){
string s = i->first;
BSONObjBuilder temp( bb.subobjStart( s ) );
temp.append( "available" , i->second.numAvailable() );
temp.appendNumber( "created" , i->second.numCreated() );
temp.done();
}
bb.done();
}
ScopedDbConnection * ScopedDbConnection::steal(){
assert( _conn );
ScopedDbConnection * n = new ScopedDbConnection( _host , _conn );
_conn = 0;
return n;
}
ScopedDbConnection::~ScopedDbConnection() {
if ( _conn ){
if ( ! _conn->isFailed() ) {
/* see done() comments above for why we log this line */
log() << "~ScopedDbConnection: _conn != null" << endl;
}
kill();
}
}
ScopedDbConnection::ScopedDbConnection(const Shard& shard )
: _host( shard.getConnString() ) , _conn( pool.get(_host) ){
}
ScopedDbConnection::ScopedDbConnection(const Shard* shard )
: _host( shard->getConnString() ) , _conn( pool.get(_host) ){
}
class PoolFlushCmd : public Command {
public:
PoolFlushCmd() : Command( "connPoolSync" , false , "connpoolsync" ){}
virtual void help( stringstream &help ) const { help<<"internal"; }
virtual LockType locktype() const { return NONE; }
virtual bool run(const string&, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool){
pool.flush();
return true;
}
virtual bool slaveOk() const {
return true;
}
} poolFlushCmd;
class PoolStats : public Command {
public:
PoolStats() : Command( "connPoolStats" ){}
virtual void help( stringstream &help ) const { help<<"stats about connection pool"; }
virtual LockType locktype() const { return NONE; }
virtual bool run(const string&, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool){
pool.appendInfo( result );
return true;
}
virtual bool slaveOk() const {
return true;
}
} poolStatsCmd;
} // namespace mongo
virtual bool run(const char*, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool){
pool.flush();
result << "ok" << 1;
return true;
}
namespace mongo {
DBConnectionPool shardConnectionPool;
/**
* holds all the actual db connections for a client to various servers
* 1 per thread, so doesn't have to be thread safe
*/
class ClientConnections : boost::noncopyable {
public:
struct Status : boost::noncopyable {
Status() : created(0), avail(0) {}
long long created;
DBClientBase* avail;
};
ClientConnections() {}
~ClientConnections() {
for ( HostMap::iterator i=_hosts.begin(); i!=_hosts.end(); ++i ) {
string addr = i->first;
Status* ss = i->second;
assert( ss );
if ( ss->avail ) {
/* if we're shutting down, don't want to initiate release mechanism as it is slow,
and isn't needed since all connections will be closed anyway */
if ( inShutdown() ) {
if( versionManager.isVersionableCB( ss->avail ) ) versionManager.resetShardVersionCB( ss->avail );
delete ss->avail;
}
else
release( addr , ss->avail );
ss->avail = 0;
}
delete ss;
}
_hosts.clear();
}
DBClientBase * get( const string& addr , const string& ns ) {
_check( ns );
Status* &s = _hosts[addr];
if ( ! s )
s = new Status();
if ( s->avail ) {
DBClientBase* c = s->avail;
s->avail = 0;
try {
shardConnectionPool.onHandedOut( c );
}
catch ( std::exception& e ) {
delete c;
throw;
}
return c;
}
s->created++;
return shardConnectionPool.get( addr );
}
void done( const string& addr , DBClientBase* conn ) {
Status* s = _hosts[addr];
assert( s );
if ( s->avail ) {
release( addr , conn );
return;
}
s->avail = conn;
}
void sync() {
for ( HostMap::iterator i=_hosts.begin(); i!=_hosts.end(); ++i ) {
string addr = i->first;
Status* ss = i->second;
if ( ss->avail )
ss->avail->getLastError();
}
}
void checkVersions( const string& ns ) {
vector<Shard> all;
Shard::getAllShards( all );
// Now only check top-level shard connections
for ( unsigned i=0; i<all.size(); i++ ) {
string sconnString = all[i].getConnString();
Status* &s = _hosts[sconnString];
if ( ! s ){
s = new Status();
}
if( ! s->avail )
s->avail = shardConnectionPool.get( sconnString );
versionManager.checkShardVersionCB( s->avail, ns, false, 1 );
}
}
void release( const string& addr , DBClientBase * conn ) {
shardConnectionPool.release( addr , conn );
}
void _check( const string& ns ) {
if ( ns.size() == 0 || _seenNS.count( ns ) )
return;
_seenNS.insert( ns );
checkVersions( ns );
}
typedef map<string,Status*,DBConnectionPool::serverNameCompare> HostMap;
HostMap _hosts;
set<string> _seenNS;
// -----
static thread_specific_ptr<ClientConnections> _perThread;
static ClientConnections* threadInstance() {
ClientConnections* cc = _perThread.get();
if ( ! cc ) {
cc = new ClientConnections();
_perThread.reset( cc );
}
return cc;
}
};
thread_specific_ptr<ClientConnections> ClientConnections::_perThread;
ShardConnection::ShardConnection( const Shard * s , const string& ns, ChunkManagerPtr manager )
: _addr( s->getConnString() ) , _ns( ns ), _manager( manager ) {
_init();
}
ShardConnection::ShardConnection( const Shard& s , const string& ns, ChunkManagerPtr manager )
: _addr( s.getConnString() ) , _ns( ns ), _manager( manager ) {
_init();
}
ShardConnection::ShardConnection( const string& addr , const string& ns, ChunkManagerPtr manager )
: _addr( addr ) , _ns( ns ), _manager( manager ) {
_init();
}
void ShardConnection::_init() {
assert( _addr.size() );
_conn = ClientConnections::threadInstance()->get( _addr , _ns );
_finishedInit = false;
}
void ShardConnection::_finishInit() {
if ( _finishedInit )
return;
_finishedInit = true;
if ( _ns.size() && versionManager.isVersionableCB( _conn ) ) {
// Make sure we specified a manager for the correct namespace
if( _manager ) assert( _manager->getns() == _ns );
_setVersion = versionManager.checkShardVersionCB( this , false , 1 );
}
else {
// Make sure we didn't specify a manager for an empty namespace
assert( ! _manager );
_setVersion = false;
}
}
void ShardConnection::done() {
if ( _conn ) {
ClientConnections::threadInstance()->done( _addr , _conn );
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::kill() {
if ( _conn ) {
if( versionManager.isVersionableCB( _conn ) ) versionManager.resetShardVersionCB( _conn );
delete _conn;
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::sync() {
ClientConnections::threadInstance()->sync();
}
bool ShardConnection::runCommand( const string& db , const BSONObj& cmd , BSONObj& res ) {
assert( _conn );
bool ok = _conn->runCommand( db , cmd , res );
if ( ! ok ) {
if ( res["code"].numberInt() == SendStaleConfigCode ) {
done();
throw RecvStaleConfigException( res["ns"].String() , res["errmsg"].String() );
}
}
return ok;
}
void ShardConnection::checkMyConnectionVersions( const string & ns ) {
ClientConnections::threadInstance()->checkVersions( ns );
}
ShardConnection::~ShardConnection() {
if ( _conn ) {
if ( ! _conn->isFailed() ) {
/* see done() comments above for why we log this line */
log() << "~ScopedDBConnection: _conn != null" << endl;
}
kill();
}
}
}
namespace mongo {
// ------ PoolForHost ------
PoolForHost::~PoolForHost() {
clear();
}
void PoolForHost::clear() {
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
delete sc.conn;
_pool.pop();
}
}
void PoolForHost::done( DBConnectionPool * pool, DBClientBase * c ) {
if (c->isFailed()) {
reportBadConnectionAt(c->getSockCreationMicroSec());
pool->onDestroy(c);
delete c;
}
else if (_pool.size() >= _maxPerHost ||
c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) {
pool->onDestroy(c);
delete c;
}
else {
_pool.push(c);
}
}
void PoolForHost::reportBadConnectionAt(uint64_t microSec) {
if (microSec != DBClientBase::INVALID_SOCK_CREATION_TIME &&
microSec > _minValidCreationTimeMicroSec) {
log() << "Detecting bad connection created at " << _minValidCreationTimeMicroSec
<< " microSec, clearing pool for " << _hostName << endl;
_minValidCreationTimeMicroSec = microSec;
clear();
}
}
bool PoolForHost::isBadSocketCreationTime(uint64_t microSec) {
return microSec != DBClientBase::INVALID_SOCK_CREATION_TIME &&
microSec <= _minValidCreationTimeMicroSec;
}
DBClientBase * PoolForHost::get( DBConnectionPool * pool , double socketTimeout ) {
time_t now = time(0);
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
_pool.pop();
if ( ! sc.ok( now ) ) {
pool->onDestroy( sc.conn );
delete sc.conn;
continue;
}
verify( sc.conn->getSoTimeout() == socketTimeout );
return sc.conn;
}
return NULL;
}
void PoolForHost::flush() {
vector<StoredConnection> all;
while ( ! _pool.empty() ) {
StoredConnection c = _pool.top();
_pool.pop();
bool res;
bool alive = false;
// When a connection is in the pool it doesn't have an AuthenticationTable set.
// Set the table temporarily for the isMaster command.
c.conn->setAuthenticationTable(
AuthenticationTable::getInternalSecurityAuthenticationTable() );
try {
c.conn->isMaster( res );
alive = true;
} catch ( const DBException e ) {
// There's something wrong with this connection, swallow the exception and do not
// put the connection back in the pool.
LOG(1) << "Exception thrown when checking pooled connection to " <<
c.conn->getServerAddress() << ": " << causedBy(e) << endl;
delete c.conn;
c.conn = NULL;
}
if ( alive ) {
c.conn->clearAuthenticationTable();
all.push_back( c );
}
}
for ( vector<StoredConnection>::iterator i=all.begin(); i != all.end(); ++i ) {
_pool.push( *i );
}
}
void PoolForHost::getStaleConnections( vector<DBClientBase*>& stale ) {
time_t now = time(0);
vector<StoredConnection> all;
while ( ! _pool.empty() ) {
StoredConnection c = _pool.top();
_pool.pop();
if ( c.ok( now ) )
all.push_back( c );
else
stale.push_back( c.conn );
}
for ( size_t i=0; i<all.size(); i++ ) {
_pool.push( all[i] );
}
}
PoolForHost::StoredConnection::StoredConnection( DBClientBase * c ) {
conn = c;
when = time(0);
}
bool PoolForHost::StoredConnection::ok( time_t now ) {
// if connection has been idle for 30 minutes, kill it
return ( now - when ) < 1800;
}
void PoolForHost::createdOne( DBClientBase * base) {
if ( _created == 0 )
_type = base->type();
_created++;
}
void PoolForHost::initializeHostName(const std::string& hostName) {
if (_hostName.empty()) {
_hostName = hostName;
}
}
unsigned PoolForHost::_maxPerHost = 50;
// ------ DBConnectionPool ------
DBConnectionPool pool;
DBConnectionPool::DBConnectionPool()
: _mutex("DBConnectionPool") ,
_name( "dbconnectionpool" ) ,
_hooks( new list<DBConnectionHook*>() ) {
}
DBClientBase* DBConnectionPool::_get(const string& ident , double socketTimeout ) {
verify( ! inShutdown() );
scoped_lock L(_mutex);
PoolForHost& p = _pools[PoolKey(ident,socketTimeout)];
p.initializeHostName(ident);
return p.get( this , socketTimeout );
}
DBClientBase* DBConnectionPool::_finishCreate( const string& host , double socketTimeout , DBClientBase* conn ) {
{
scoped_lock L(_mutex);
PoolForHost& p = _pools[PoolKey(host,socketTimeout)];
p.initializeHostName(host);
p.createdOne( conn );
}
try {
onCreate( conn );
onHandedOut( conn );
}
catch ( std::exception & ) {
delete conn;
throw;
}
return conn;
}
DBClientBase* DBConnectionPool::get(const ConnectionString& url, double socketTimeout) {
DBClientBase * c = _get( url.toString() , socketTimeout );
if ( c ) {
try {
onHandedOut( c );
}
catch ( std::exception& ) {
delete c;
throw;
}
return c;
}
string errmsg;
c = url.connect( errmsg, socketTimeout );
uassert( 13328 , _name + ": connect failed " + url.toString() + " : " + errmsg , c );
return _finishCreate( url.toString() , socketTimeout , c );
}
DBClientBase* DBConnectionPool::get(const string& host, double socketTimeout) {
DBClientBase * c = _get( host , socketTimeout );
if ( c ) {
try {
onHandedOut( c );
}
catch ( std::exception& ) {
delete c;
throw;
}
return c;
}
string errmsg;
ConnectionString cs = ConnectionString::parse( host , errmsg );
uassert( 13071 , (string)"invalid hostname [" + host + "]" + errmsg , cs.isValid() );
c = cs.connect( errmsg, socketTimeout );
if ( ! c )
throw SocketException( SocketException::CONNECT_ERROR , host , 11002 , str::stream() << _name << " error: " << errmsg );
return _finishCreate( host , socketTimeout , c );
}
void DBConnectionPool::release(const string& host, DBClientBase *c) {
scoped_lock L(_mutex);
_pools[PoolKey(host,c->getSoTimeout())].done(this,c);
}
DBConnectionPool::~DBConnectionPool() {
// connection closing is handled by ~PoolForHost
}
void DBConnectionPool::flush() {
scoped_lock L(_mutex);
for ( PoolMap::iterator i = _pools.begin(); i != _pools.end(); i++ ) {
PoolForHost& p = i->second;
p.flush();
}
}
void DBConnectionPool::clear() {
scoped_lock L(_mutex);
LOG(2) << "Removing connections on all pools owned by " << _name << endl;
for (PoolMap::iterator iter = _pools.begin(); iter != _pools.end(); ++iter) {
iter->second.clear();
}
}
void DBConnectionPool::removeHost( const string& host ) {
scoped_lock L(_mutex);
LOG(2) << "Removing connections from all pools for host: " << host << endl;
for ( PoolMap::iterator i = _pools.begin(); i != _pools.end(); ++i ) {
const string& poolHost = i->first.ident;
if ( !serverNameCompare()(host, poolHost) && !serverNameCompare()(poolHost, host) ) {
// hosts are the same
i->second.clear();
}
}
}
void DBConnectionPool::addHook( DBConnectionHook * hook ) {
_hooks->push_back( hook );
}
void DBConnectionPool::onCreate( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onCreate( conn );
}
}
void DBConnectionPool::onHandedOut( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onHandedOut( conn );
}
}
void DBConnectionPool::onDestroy( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onDestroy( conn );
}
}
void DBConnectionPool::appendInfo( BSONObjBuilder& b ) {
int avail = 0;
long long created = 0;
map<ConnectionString::ConnectionType,long long> createdByType;
set<string> replicaSets;
BSONObjBuilder bb( b.subobjStart( "hosts" ) );
{
scoped_lock lk( _mutex );
for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
if ( i->second.numCreated() == 0 )
continue;
string s = str::stream() << i->first.ident << "::" << i->first.timeout;
BSONObjBuilder temp( bb.subobjStart( s ) );
temp.append( "available" , i->second.numAvailable() );
temp.appendNumber( "created" , i->second.numCreated() );
temp.done();
avail += i->second.numAvailable();
created += i->second.numCreated();
long long& x = createdByType[i->second.type()];
x += i->second.numCreated();
{
string setName = i->first.ident;
if ( setName.find( "/" ) != string::npos ) {
setName = setName.substr( 0 , setName.find( "/" ) );
replicaSets.insert( setName );
}
}
}
}
bb.done();
BSONObjBuilder setBuilder( b.subobjStart( "replicaSets" ) );
for ( set<string>::iterator i=replicaSets.begin(); i!=replicaSets.end(); ++i ) {
string rs = *i;
ReplicaSetMonitorPtr m = ReplicaSetMonitor::get( rs );
if ( ! m ) {
warning() << "no monitor for set: " << rs << endl;
continue;
}
BSONObjBuilder temp( setBuilder.subobjStart( rs ) );
m->appendInfo( temp );
temp.done();
}
setBuilder.done();
{
BSONObjBuilder temp( bb.subobjStart( "createdByType" ) );
for ( map<ConnectionString::ConnectionType,long long>::iterator i=createdByType.begin(); i!=createdByType.end(); ++i ) {
temp.appendNumber( ConnectionString::typeToString( i->first ) , i->second );
}
temp.done();
}
b.append( "totalAvailable" , avail );
b.appendNumber( "totalCreated" , created );
}
bool DBConnectionPool::serverNameCompare::operator()( const string& a , const string& b ) const{
const char* ap = a.c_str();
const char* bp = b.c_str();
while (true){
if (*ap == '\0' || *ap == '/'){
if (*bp == '\0' || *bp == '/')
return false; // equal strings
else
return true; // a is shorter
}
if (*bp == '\0' || *bp == '/')
return false; // b is shorter
if ( *ap < *bp)
return true;
else if (*ap > *bp)
return false;
++ap;
++bp;
}
verify(false);
}
bool DBConnectionPool::poolKeyCompare::operator()( const PoolKey& a , const PoolKey& b ) const {
if (DBConnectionPool::serverNameCompare()( a.ident , b.ident ))
return true;
if (DBConnectionPool::serverNameCompare()( b.ident , a.ident ))
return false;
return a.timeout < b.timeout;
}
bool DBConnectionPool::isConnectionGood(const string& hostName, DBClientBase* conn) {
if (conn == NULL) {
return false;
}
if (conn->isFailed()) {
return false;
}
{
scoped_lock sl(_mutex);
PoolForHost& pool = _pools[PoolKey(hostName, conn->getSoTimeout())];
if (pool.isBadSocketCreationTime(conn->getSockCreationMicroSec())) {
return false;
}
}
return true;
}
void DBConnectionPool::taskDoWork() {
vector<DBClientBase*> toDelete;
{
// we need to get the connections inside the lock
// but we can actually delete them outside
scoped_lock lk( _mutex );
for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
i->second.getStaleConnections( toDelete );
}
}
for ( size_t i=0; i<toDelete.size(); i++ ) {
try {
onDestroy( toDelete[i] );
delete toDelete[i];
}
catch ( ... ) {
// we don't care if there was a socket error
}
}
}
// ------ ScopedDbConnection ------
void ScopedDbConnection::_setSocketTimeout(){
if( ! _conn ) return;
if( _conn->type() == ConnectionString::MASTER )
(( DBClientConnection* ) _conn)->setSoTimeout( _socketTimeout );
else if( _conn->type() == ConnectionString::SYNC )
(( SyncClusterConnection* ) _conn)->setAllSoTimeouts( _socketTimeout );
}
ScopedDbConnection::~ScopedDbConnection() {
if ( _conn ) {
if (_conn->isFailed()) {
if (_conn->getSockCreationMicroSec() ==
DBClientBase::INVALID_SOCK_CREATION_TIME) {
kill();
}
else {
// The pool takes care of deleting the failed connection - this
// will also trigger disposal of older connections in the pool
done();
}
}
else {
/* see done() comments above for why we log this line */
log() << "scoped connection to " << _conn->getServerAddress()
<< " not being returned to the pool" << endl;
kill();
}
}
}
void ScopedDbConnection::clearPool() {
pool.clear();
}
AtomicUInt AScopedConnection::_numConnections;
} // namespace mongo
namespace mongo {
namespace {
class ClientConnections;
/**
* Class which tracks ClientConnections (the client connection pool) for each incoming connection,
* allowing stats access.
*/
class ActiveClientConnections {
public:
void add(const ClientConnections* cc) {
stdx::lock_guard<stdx::mutex> lg(_mutex);
_clientConnections.insert(cc);
}
void remove(const ClientConnections* cc) {
stdx::lock_guard<stdx::mutex> lg(_mutex);
_clientConnections.erase(cc);
}
void appendInfo(BSONObjBuilder* b) const;
private:
mutable stdx::mutex _mutex;
std::set<const ClientConnections*> _clientConnections;
} activeClientConnections;
/**
* Holds all the actual db connections for a client to various servers 1 per thread, so doesn't have
* to be thread safe.
*/
class ClientConnections {
MONGO_DISALLOW_COPYING(ClientConnections);
public:
struct Status {
// May be read concurrently, but only written from this thread
long long created = 0;
DBClientBase* avail = nullptr;
};
ClientConnections() {
// Start tracking client connections
activeClientConnections.add(this);
}
~ClientConnections() {
// Stop tracking these client connections
activeClientConnections.remove(this);
releaseAll(true);
}
static ClientConnections* threadInstance() {
if (!_perThread) {
_perThread = stdx::make_unique<ClientConnections>();
}
return _perThread.get();
}
DBClientBase* get(const std::string& addr, const std::string& ns) {
{
// We want to report ns stats
scoped_spinlock lock(_lock);
if (ns.size() > 0)
_seenNS.insert(ns);
}
Status* s = _getStatus(addr);
std::unique_ptr<DBClientBase> c;
if (s->avail) {
c.reset(s->avail);
s->avail = 0;
// May throw an exception
shardConnectionPool.onHandedOut(c.get());
} else {
c.reset(shardConnectionPool.get(addr));
// After, so failed creation doesn't get counted
s->created++;
}
return c.release();
}
void releaseAll(bool fromDestructor = false) {
// Don't need spinlock protection because if not in the destructor, we don't modify
// _hosts, and if in the destructor we are not accessible to external threads.
for (HostMap::iterator i = _hosts.begin(); i != _hosts.end(); ++i) {
const auto addr = i->first;
Status* ss = i->second;
invariant(ss);
if (ss->avail) {
// If we're shutting down, don't want to initiate release mechanism as it is
// slow, and isn't needed since all connections will be closed anyway.
if (globalInShutdownDeprecated()) {
if (versionManager.isVersionableCB(ss->avail)) {
versionManager.resetShardVersionCB(ss->avail);
}
delete ss->avail;
} else {
release(addr, ss->avail);
}
ss->avail = 0;
}
if (fromDestructor) {
delete ss;
}
}
if (fromDestructor) {
_hosts.clear();
}
}
void done(const std::string& addr, DBClientBase* conn) {
Status* s = _hosts[addr];
verify(s);
const bool isConnGood = shardConnectionPool.isConnectionGood(addr, conn);
if (s->avail != NULL) {
warning() << "Detected additional sharded connection in the "
<< "thread local pool for " << addr;
if (DBException::traceExceptions.load()) {
// There shouldn't be more than one connection checked out to the same
// host on the same thread.
printStackTrace();
}
if (!isConnGood) {
delete s->avail;
s->avail = NULL;
}
// Let the internal pool handle the bad connection, this can also
// update the lower bounds for the known good socket creation time
// for this host.
release(addr, conn);
return;
}
if (!isConnGood) {
// Let the internal pool handle the bad connection.
release(addr, conn);
return;
}
// Note: Although we try our best to clear bad connections as much as possible, some of them
// can still slip through because of how ClientConnections are being used - as thread local
// variables. This means that threads won't be able to see the s->avail connection of other
// threads.
s->avail = conn;
}
void checkVersions(OperationContext* opCtx, const std::string& ns) {
auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
std::vector<ShardId> all;
shardRegistry->getAllShardIdsNoReload(&all);
// Don't report exceptions here as errors in GetLastError
LastError::Disabled ignoreForGLE(&LastError::get(cc()));
// Now only check top-level shard connections
for (const ShardId& shardId : all) {
try {
auto shardStatus = shardRegistry->getShard(opCtx, shardId);
if (!shardStatus.isOK()) {
invariant(shardStatus == ErrorCodes::ShardNotFound);
continue;
}
const auto shard = shardStatus.getValue();
const auto sconnString = shard->getConnString().toString();
Status* s = _getStatus(sconnString);
if (!s->avail) {
s->avail = shardConnectionPool.get(sconnString);
s->created++; // After, so failed creation doesn't get counted
}
versionManager.checkShardVersionCB(opCtx, s->avail, ns, false, 1);
} catch (const DBException& ex) {
warning() << "Problem while initially checking shard versions on"
<< " " << shardId << causedBy(redact(ex));
// NOTE: This is only a heuristic, to avoid multiple stale version retries across
// multiple shards, and does not affect correctness.
}
}
}
void release(const std::string& addr, DBClientBase* conn) {
shardConnectionPool.release(addr, conn);
}
void forgetNS(const std::string& ns) {
scoped_spinlock lock(_lock);
_seenNS.erase(ns);
}
/**
* Clears the connections kept by this pool (ie, not including the global pool)
*/
void clearPool() {
for (HostMap::iterator iter = _hosts.begin(); iter != _hosts.end(); ++iter) {
if (iter->second->avail != NULL) {
delete iter->second->avail;
}
delete iter->second;
}
_hosts.clear();
}
/**
* Appends info about the client connection pool to a BSONObjBuilder. Safe to call with
* activeClientConnections lock.
*/
void appendInfo(BSONObjBuilder& b) const {
scoped_spinlock lock(_lock);
BSONArrayBuilder hostsArrB(b.subarrayStart("hosts"));
for (HostMap::const_iterator i = _hosts.begin(); i != _hosts.end(); ++i) {
BSONObjBuilder bb(hostsArrB.subobjStart());
bb.append("host", i->first);
bb.append("created", i->second->created);
bb.appendBool("avail", static_cast<bool>(i->second->avail));
bb.done();
}
hostsArrB.done();
BSONArrayBuilder nsArrB(b.subarrayStart("seenNS"));
for (const auto& ns : _seenNS) {
nsArrB.append(ns);
}
nsArrB.done();
}
private:
/**
* Gets or creates the status object for the host.
*/
Status* _getStatus(const std::string& addr) {
scoped_spinlock lock(_lock);
Status*& temp = _hosts[addr];
if (!temp) {
temp = new Status();
}
return temp;
}
static thread_local std::unique_ptr<ClientConnections> _perThread;
// Protects only the creation of new entries in the _hosts and _seenNS map from external
// threads. Reading _hosts/_seenNS in this thread doesn't need protection.
mutable SpinLock _lock;
using HostMap = std::map<std::string, Status*, DBConnectionPool::serverNameCompare>;
HostMap _hosts;
std::set<std::string> _seenNS;
};
void ActiveClientConnections::appendInfo(BSONObjBuilder* b) const {
// Preallocate the buffer because there may be quite a few threads to report
BSONArrayBuilder arr(64 * 1024);
{
stdx::lock_guard<stdx::mutex> lock(_mutex);
for (const auto* conn : _clientConnections) {
BSONObjBuilder bb(arr.subobjStart());
conn->appendInfo(bb);
bb.doneFast();
}
}
b->appendArray("threads", arr.obj());
}
thread_local std::unique_ptr<ClientConnections> ClientConnections::_perThread;
// Maximum connections per host the sharded conn pool should store
int maxShardedConnsPerHost(200);
ExportedServerParameter<int, ServerParameterType::kStartupOnly> //
maxShardedConnsPerHostParameter(ServerParameterSet::getGlobal(),
"connPoolMaxShardedConnsPerHost",
&maxShardedConnsPerHost);
// Maximum in-use connections per host in the sharded connection pool
int maxShardedInUseConnsPerHost(std::numeric_limits<int>::max());
ExportedServerParameter<int, ServerParameterType::kStartupOnly> //
maxShardedInUseConnsPerHostParameter(ServerParameterSet::getGlobal(),
"connPoolMaxShardedInUseConnsPerHost",
&maxShardedInUseConnsPerHost);
// Amount of time, in minutes, to keep idle connections in the sharded connection pool
int shardedConnPoolIdleTimeout(std::numeric_limits<int>::max());
ExportedServerParameter<int, ServerParameterType::kStartupOnly> //
shardedConnPoolIdleTimeoutParameter(ServerParameterSet::getGlobal(),
"shardedConnPoolIdleTimeoutMinutes",
&shardedConnPoolIdleTimeout);
MONGO_INITIALIZER(InitializeShardedConnectionPool)(InitializerContext* context) {
shardConnectionPool.setName("sharded connection pool");
shardConnectionPool.setMaxPoolSize(maxShardedConnsPerHost);
shardConnectionPool.setMaxInUse(maxShardedInUseConnsPerHost);
shardConnectionPool.setIdleTimeout(shardedConnPoolIdleTimeout);
return Status::OK();
}
} // namespace
DBConnectionPool shardConnectionPool;
ShardConnection::ShardConnection(OperationContext* opCtx,
const ConnectionString& connectionString,
const std::string& ns,
std::shared_ptr<ChunkManager> manager)
: _cs(connectionString), _ns(ns), _manager(manager) {
invariant(_cs.isValid());
// This code should never run under a cross-shard transaction
invariant(!TransactionRouter::get(opCtx));
// Make sure we specified a manager for the correct namespace
if (_ns.size() && _manager) {
invariant(_manager->getns().ns() == _ns);
}
auto csString = _cs.toString();
_conn = ClientConnections::threadInstance()->get(csString, _ns);
if (isMongos()) {
// In mongos, we record this connection as having been used for useful work to provide
// useful information in getLastError.
ClusterLastErrorInfo::get(opCtx->getClient())->addShardHost(csString);
}
}
ShardConnection::~ShardConnection() {
if (_conn) {
if (_conn->isFailed()) {
if (_conn->getSockCreationMicroSec() == DBClientBase::INVALID_SOCK_CREATION_TIME) {
kill();
} else {
// The pool takes care of deleting the failed connection - this
// will also trigger disposal of older connections in the pool
done();
}
} else {
// see done() comments above for why we log this line
log() << "sharded connection to " << _conn->getServerAddress()
<< " not being returned to the pool";
kill();
}
}
}
void ShardConnection::_finishInit() {
if (_finishedInit)
return;
_finishedInit = true;
if (versionManager.isVersionableCB(_conn)) {
auto& client = cc();
auto opCtx = client.getOperationContext();
invariant(opCtx);
_setVersion = versionManager.checkShardVersionCB(opCtx, this, false, 1);
} else {
// Make sure we didn't specify a manager for a non-versionable connection (i.e. config)
invariant(!_manager);
_setVersion = false;
}
}
void ShardConnection::done() {
if (_conn) {
ClientConnections::threadInstance()->done(_cs.toString(), _conn);
_conn = nullptr;
_finishedInit = true;
}
}
void ShardConnection::kill() {
if (_conn) {
if (versionManager.isVersionableCB(_conn)) {
versionManager.resetShardVersionCB(_conn);
}
if (_conn->isFailed()) {
// Let the pool know about the bad connection and also delegate disposal to it.
ClientConnections::threadInstance()->done(_cs.toString(), _conn);
} else {
delete _conn;
}
_conn = 0;
_finishedInit = true;
}
}
void ShardConnection::reportActiveClientConnections(BSONObjBuilder* builder) {
activeClientConnections.appendInfo(builder);
}
void ShardConnection::checkMyConnectionVersions(OperationContext* opCtx, const std::string& ns) {
ClientConnections::threadInstance()->checkVersions(opCtx, ns);
}
void ShardConnection::releaseMyConnections() {
ClientConnections::threadInstance()->releaseAll();
}
void ShardConnection::clearPool() {
shardConnectionPool.clear();
ClientConnections::threadInstance()->clearPool();
}
void ShardConnection::forgetNS(const std::string& ns) {
ClientConnections::threadInstance()->forgetNS(ns);
}
} // namespace mongo
namespace mongo {
// ------ PoolForHost ------
PoolForHost::~PoolForHost() {
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
delete sc.conn;
_pool.pop();
}
}
void PoolForHost::done( DBClientBase * c ) {
if ( _pool.size() >= _maxPerHost ) {
delete c;
}
else {
_pool.push(c);
}
}
DBClientBase * PoolForHost::get() {
time_t now = time(0);
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
_pool.pop();
if ( sc.ok( now ) )
return sc.conn;
delete sc.conn;
}
return NULL;
}
void PoolForHost::flush() {
vector<StoredConnection> all;
while ( ! _pool.empty() ) {
StoredConnection c = _pool.top();
_pool.pop();
all.push_back( c );
bool res;
c.conn->isMaster( res );
}
for ( vector<StoredConnection>::iterator i=all.begin(); i != all.end(); ++i ) {
_pool.push( *i );
}
}
PoolForHost::StoredConnection::StoredConnection( DBClientBase * c ) {
conn = c;
when = time(0);
}
bool PoolForHost::StoredConnection::ok( time_t now ) {
// if connection has been idle for an hour, kill it
return ( now - when ) < 3600;
}
void PoolForHost::createdOne( DBClientBase * base) {
if ( _created == 0 )
_type = base->type();
_created++;
}
unsigned PoolForHost::_maxPerHost = 50;
// ------ DBConnectionPool ------
DBConnectionPool pool;
DBClientBase* DBConnectionPool::_get(const string& ident) {
scoped_lock L(_mutex);
PoolForHost& p = _pools[ident];
return p.get();
}
DBClientBase* DBConnectionPool::_finishCreate( const string& host , DBClientBase* conn ) {
{
scoped_lock L(_mutex);
PoolForHost& p = _pools[host];
p.createdOne( conn );
}
onCreate( conn );
onHandedOut( conn );
return conn;
}
DBClientBase* DBConnectionPool::get(const ConnectionString& url) {
DBClientBase * c = _get( url.toString() );
if ( c ) {
onHandedOut( c );
return c;
}
string errmsg;
c = url.connect( errmsg );
uassert( 13328 , _name + ": connect failed " + url.toString() + " : " + errmsg , c );
return _finishCreate( url.toString() , c );
}
DBClientBase* DBConnectionPool::get(const string& host) {
DBClientBase * c = _get( host );
if ( c ) {
onHandedOut( c );
return c;
}
string errmsg;
ConnectionString cs = ConnectionString::parse( host , errmsg );
uassert( 13071 , (string)"invalid hostname [" + host + "]" + errmsg , cs.isValid() );
c = cs.connect( errmsg );
if ( ! c )
throw SocketException( SocketException::CONNECT_ERROR , host , 11002 , str::stream() << _name << " error: " << errmsg );
return _finishCreate( host , c );
}
DBConnectionPool::~DBConnectionPool() {
// connection closing is handled by ~PoolForHost
}
void DBConnectionPool::flush() {
scoped_lock L(_mutex);
for ( PoolMap::iterator i = _pools.begin(); i != _pools.end(); i++ ) {
PoolForHost& p = i->second;
p.flush();
}
}
void DBConnectionPool::addHook( DBConnectionHook * hook ) {
_hooks.push_back( hook );
}
void DBConnectionPool::onCreate( DBClientBase * conn ) {
if ( _hooks.size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks.begin(); i != _hooks.end(); i++ ) {
(*i)->onCreate( conn );
}
}
void DBConnectionPool::onHandedOut( DBClientBase * conn ) {
if ( _hooks.size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks.begin(); i != _hooks.end(); i++ ) {
(*i)->onHandedOut( conn );
}
}
void DBConnectionPool::appendInfo( BSONObjBuilder& b ) {
BSONObjBuilder bb( b.subobjStart( "hosts" ) );
int avail = 0;
long long created = 0;
map<ConnectionString::ConnectionType,long long> createdByType;
{
scoped_lock lk( _mutex );
for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
if ( i->second.numCreated() == 0 )
continue;
string s = i->first;
BSONObjBuilder temp( bb.subobjStart( s ) );
temp.append( "available" , i->second.numAvailable() );
temp.appendNumber( "created" , i->second.numCreated() );
temp.done();
avail += i->second.numAvailable();
created += i->second.numCreated();
long long& x = createdByType[i->second.type()];
x += i->second.numCreated();
}
}
bb.done();
{
BSONObjBuilder temp( bb.subobjStart( "createdByType" ) );
for ( map<ConnectionString::ConnectionType,long long>::iterator i=createdByType.begin(); i!=createdByType.end(); ++i ) {
temp.appendNumber( ConnectionString::typeToString( i->first ) , i->second );
}
temp.done();
}
b.append( "totalAvailable" , avail );
b.appendNumber( "totalCreated" , created );
}
bool DBConnectionPool::serverNameCompare::operator()( const string& a , const string& b ) const{
string ap = str::before( a , "/" );
string bp = str::before( b , "/" );
return ap < bp;
}
// ------ ScopedDbConnection ------
ScopedDbConnection * ScopedDbConnection::steal() {
assert( _conn );
ScopedDbConnection * n = new ScopedDbConnection( _host , _conn );
_conn = 0;
return n;
}
ScopedDbConnection::~ScopedDbConnection() {
if ( _conn ) {
if ( ! _conn->isFailed() ) {
/* see done() comments above for why we log this line */
log() << "~ScopedDbConnection: _conn != null" << endl;
}
kill();
}
}
ScopedDbConnection::ScopedDbConnection(const Shard& shard )
: _host( shard.getConnString() ) , _conn( pool.get(_host) ) {
}
ScopedDbConnection::ScopedDbConnection(const Shard* shard )
: _host( shard->getConnString() ) , _conn( pool.get(_host) ) {
}
class PoolFlushCmd : public Command {
public:
PoolFlushCmd() : Command( "connPoolSync" , false , "connpoolsync" ) {}
virtual void help( stringstream &help ) const { help<<"internal"; }
virtual LockType locktype() const { return NONE; }
virtual bool run(const string&, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool) {
pool.flush();
return true;
}
virtual bool slaveOk() const {
return true;
}
} poolFlushCmd;
class PoolStats : public Command {
public:
PoolStats() : Command( "connPoolStats" ) {}
virtual void help( stringstream &help ) const { help<<"stats about connection pool"; }
virtual LockType locktype() const { return NONE; }
virtual bool run(const string&, mongo::BSONObj&, std::string&, mongo::BSONObjBuilder& result, bool) {
pool.appendInfo( result );
result.append( "numDBClientConnection" , DBClientConnection::getNumConnections() );
result.append( "numAScopedConnection" , AScopedConnection::getNumConnections() );
return true;
}
virtual bool slaveOk() const {
return true;
}
} poolStatsCmd;
AtomicUInt AScopedConnection::_numConnections;
} // namespace mongo
void ShardConnection::clearPool() {
shardConnectionPool.clear();
ClientConnections::threadInstance()->clearPool();
}
namespace mongo {
using std::endl;
using std::list;
using std::map;
using std::set;
using std::string;
using std::vector;
// ------ PoolForHost ------
PoolForHost::~PoolForHost() {
clear();
}
void PoolForHost::clear() {
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
delete sc.conn;
_pool.pop();
}
}
void PoolForHost::done(DBConnectionPool* pool, DBClientBase* c) {
bool isFailed = c->isFailed();
// Remember that this host had a broken connection for later
if (isFailed) reportBadConnectionAt(c->getSockCreationMicroSec());
if (isFailed ||
// Another (later) connection was reported as broken to this host
(c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) ||
// We have a pool size that we need to enforce
(_maxPoolSize >= 0 && static_cast<int>(_pool.size()) >= _maxPoolSize)) {
pool->onDestroy(c);
delete c;
}
else {
// The connection is probably fine, save for later
_pool.push(c);
}
}
void PoolForHost::reportBadConnectionAt(uint64_t microSec) {
if (microSec != DBClientBase::INVALID_SOCK_CREATION_TIME &&
microSec > _minValidCreationTimeMicroSec) {
_minValidCreationTimeMicroSec = microSec;
log() << "Detected bad connection created at " << _minValidCreationTimeMicroSec
<< " microSec, clearing pool for " << _hostName
<< " of " << _pool.size() << " connections" << endl;
clear();
}
}
bool PoolForHost::isBadSocketCreationTime(uint64_t microSec) {
return microSec != DBClientBase::INVALID_SOCK_CREATION_TIME &&
microSec <= _minValidCreationTimeMicroSec;
}
DBClientBase * PoolForHost::get( DBConnectionPool * pool , double socketTimeout ) {
time_t now = time(0);
while ( ! _pool.empty() ) {
StoredConnection sc = _pool.top();
_pool.pop();
if ( ! sc.ok( now ) ) {
pool->onDestroy( sc.conn );
delete sc.conn;
continue;
}
verify( sc.conn->getSoTimeout() == socketTimeout );
return sc.conn;
}
return NULL;
}
void PoolForHost::flush() {
while (!_pool.empty()) {
StoredConnection c = _pool.top();
_pool.pop();
delete c.conn;
}
}
void PoolForHost::getStaleConnections( vector<DBClientBase*>& stale ) {
time_t now = time(0);
vector<StoredConnection> all;
while ( ! _pool.empty() ) {
StoredConnection c = _pool.top();
_pool.pop();
if ( c.ok( now ) )
all.push_back( c );
else
stale.push_back( c.conn );
}
for ( size_t i=0; i<all.size(); i++ ) {
_pool.push( all[i] );
}
}
PoolForHost::StoredConnection::StoredConnection( DBClientBase * c ) {
conn = c;
when = time(0);
}
bool PoolForHost::StoredConnection::ok( time_t now ) {
// Poke the connection to see if we're still ok
return conn->isStillConnected();
}
void PoolForHost::createdOne( DBClientBase * base) {
if ( _created == 0 )
_type = base->type();
_created++;
}
void PoolForHost::initializeHostName(const std::string& hostName) {
if (_hostName.empty()) {
_hostName = hostName;
}
}
// ------ DBConnectionPool ------
DBConnectionPool pool;
const int PoolForHost::kPoolSizeUnlimited(-1);
DBConnectionPool::DBConnectionPool()
: _name( "dbconnectionpool" ) ,
_maxPoolSize(PoolForHost::kPoolSizeUnlimited) ,
_hooks( new list<DBConnectionHook*>() ) {
}
DBClientBase* DBConnectionPool::_get(const string& ident , double socketTimeout ) {
uassert(17382, "Can't use connection pool during shutdown",
!inShutdown());
boost::lock_guard<boost::mutex> L(_mutex);
PoolForHost& p = _pools[PoolKey(ident,socketTimeout)];
p.setMaxPoolSize(_maxPoolSize);
p.initializeHostName(ident);
return p.get( this , socketTimeout );
}
DBClientBase* DBConnectionPool::_finishCreate( const string& host , double socketTimeout , DBClientBase* conn ) {
{
boost::lock_guard<boost::mutex> L(_mutex);
PoolForHost& p = _pools[PoolKey(host,socketTimeout)];
p.setMaxPoolSize(_maxPoolSize);
p.initializeHostName(host);
p.createdOne( conn );
}
try {
onCreate( conn );
onHandedOut( conn );
}
catch ( std::exception & ) {
delete conn;
throw;
}
return conn;
}
DBClientBase* DBConnectionPool::get(const ConnectionString& url, double socketTimeout) {
DBClientBase * c = _get( url.toString() , socketTimeout );
if ( c ) {
try {
onHandedOut( c );
}
catch ( std::exception& ) {
delete c;
throw;
}
return c;
}
string errmsg;
c = url.connect( errmsg, socketTimeout );
uassert( 13328 , _name + ": connect failed " + url.toString() + " : " + errmsg , c );
return _finishCreate( url.toString() , socketTimeout , c );
}
DBClientBase* DBConnectionPool::get(const string& host, double socketTimeout) {
DBClientBase * c = _get( host , socketTimeout );
if ( c ) {
try {
onHandedOut( c );
}
catch ( std::exception& ) {
delete c;
throw;
}
return c;
}
string errmsg;
ConnectionString cs = ConnectionString::parse( host , errmsg );
uassert( 13071 , (string)"invalid hostname [" + host + "]" + errmsg , cs.isValid() );
c = cs.connect( errmsg, socketTimeout );
if ( ! c )
throw SocketException( SocketException::CONNECT_ERROR , host , 11002 , str::stream() << _name << " error: " << errmsg );
return _finishCreate( host , socketTimeout , c );
}
void DBConnectionPool::onRelease(DBClientBase* conn) {
if (_hooks->empty()) {
return;
}
for (list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++) {
(*i)->onRelease( conn );
}
}
void DBConnectionPool::release(const string& host, DBClientBase *c) {
onRelease(c);
boost::lock_guard<boost::mutex> L(_mutex);
_pools[PoolKey(host,c->getSoTimeout())].done(this,c);
}
DBConnectionPool::~DBConnectionPool() {
// connection closing is handled by ~PoolForHost
}
void DBConnectionPool::flush() {
boost::lock_guard<boost::mutex> L(_mutex);
for ( PoolMap::iterator i = _pools.begin(); i != _pools.end(); i++ ) {
PoolForHost& p = i->second;
p.flush();
}
}
void DBConnectionPool::clear() {
boost::lock_guard<boost::mutex> L(_mutex);
LOG(2) << "Removing connections on all pools owned by " << _name << endl;
for (PoolMap::iterator iter = _pools.begin(); iter != _pools.end(); ++iter) {
iter->second.clear();
}
}
void DBConnectionPool::removeHost( const string& host ) {
boost::lock_guard<boost::mutex> L(_mutex);
LOG(2) << "Removing connections from all pools for host: " << host << endl;
for ( PoolMap::iterator i = _pools.begin(); i != _pools.end(); ++i ) {
const string& poolHost = i->first.ident;
if ( !serverNameCompare()(host, poolHost) && !serverNameCompare()(poolHost, host) ) {
// hosts are the same
i->second.clear();
}
}
}
void DBConnectionPool::addHook( DBConnectionHook * hook ) {
_hooks->push_back( hook );
}
void DBConnectionPool::onCreate( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onCreate( conn );
}
}
void DBConnectionPool::onHandedOut( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onHandedOut( conn );
}
}
void DBConnectionPool::onDestroy( DBClientBase * conn ) {
if ( _hooks->size() == 0 )
return;
for ( list<DBConnectionHook*>::iterator i = _hooks->begin(); i != _hooks->end(); i++ ) {
(*i)->onDestroy( conn );
}
}
void DBConnectionPool::appendInfo( BSONObjBuilder& b ) {
int avail = 0;
long long created = 0;
map<ConnectionString::ConnectionType,long long> createdByType;
BSONObjBuilder bb( b.subobjStart( "hosts" ) );
{
boost::lock_guard<boost::mutex> lk( _mutex );
for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
if ( i->second.numCreated() == 0 )
continue;
string s = str::stream() << i->first.ident << "::" << i->first.timeout;
BSONObjBuilder temp( bb.subobjStart( s ) );
temp.append( "available" , i->second.numAvailable() );
temp.appendNumber( "created" , i->second.numCreated() );
temp.done();
avail += i->second.numAvailable();
created += i->second.numCreated();
long long& x = createdByType[i->second.type()];
x += i->second.numCreated();
}
}
bb.done();
// Always report all replica sets being tracked
set<string> replicaSets = ReplicaSetMonitor::getAllTrackedSets();
BSONObjBuilder setBuilder( b.subobjStart( "replicaSets" ) );
for ( set<string>::iterator i=replicaSets.begin(); i!=replicaSets.end(); ++i ) {
string rs = *i;
ReplicaSetMonitorPtr m = ReplicaSetMonitor::get( rs );
if ( ! m ) {
warning() << "no monitor for set: " << rs << endl;
continue;
}
BSONObjBuilder temp( setBuilder.subobjStart( rs ) );
m->appendInfo( temp );
temp.done();
}
setBuilder.done();
{
BSONObjBuilder temp( bb.subobjStart( "createdByType" ) );
for ( map<ConnectionString::ConnectionType,long long>::iterator i=createdByType.begin(); i!=createdByType.end(); ++i ) {
temp.appendNumber( ConnectionString::typeToString( i->first ) , i->second );
}
temp.done();
}
b.append( "totalAvailable" , avail );
b.appendNumber( "totalCreated" , created );
}
bool DBConnectionPool::serverNameCompare::operator()( const string& a , const string& b ) const{
const char* ap = a.c_str();
const char* bp = b.c_str();
while (true){
if (*ap == '\0' || *ap == '/'){
if (*bp == '\0' || *bp == '/')
return false; // equal strings
else
return true; // a is shorter
}
if (*bp == '\0' || *bp == '/')
return false; // b is shorter
if ( *ap < *bp)
return true;
else if (*ap > *bp)
return false;
++ap;
++bp;
}
verify(false);
}
bool DBConnectionPool::poolKeyCompare::operator()( const PoolKey& a , const PoolKey& b ) const {
if (DBConnectionPool::serverNameCompare()( a.ident , b.ident ))
return true;
if (DBConnectionPool::serverNameCompare()( b.ident , a.ident ))
return false;
return a.timeout < b.timeout;
}
bool DBConnectionPool::isConnectionGood(const string& hostName, DBClientBase* conn) {
if (conn == NULL) {
return false;
}
if (conn->isFailed()) {
return false;
}
{
boost::lock_guard<boost::mutex> sl(_mutex);
PoolForHost& pool = _pools[PoolKey(hostName, conn->getSoTimeout())];
if (pool.isBadSocketCreationTime(conn->getSockCreationMicroSec())) {
return false;
}
}
return true;
}
void DBConnectionPool::taskDoWork() {
vector<DBClientBase*> toDelete;
{
// we need to get the connections inside the lock
// but we can actually delete them outside
boost::lock_guard<boost::mutex> lk( _mutex );
for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
i->second.getStaleConnections( toDelete );
}
}
for ( size_t i=0; i<toDelete.size(); i++ ) {
try {
onDestroy( toDelete[i] );
delete toDelete[i];
}
catch ( ... ) {
// we don't care if there was a socket error
}
}
}
// ------ ScopedDbConnection ------
void ScopedDbConnection::_setSocketTimeout(){
if( ! _conn ) return;
if( _conn->type() == ConnectionString::MASTER )
(( DBClientConnection* ) _conn)->setSoTimeout( _socketTimeout );
else if( _conn->type() == ConnectionString::SYNC )
(( SyncClusterConnection* ) _conn)->setAllSoTimeouts( _socketTimeout );
}
ScopedDbConnection::~ScopedDbConnection() {
if ( _conn ) {
if (_conn->isFailed()) {
if (_conn->getSockCreationMicroSec() ==
DBClientBase::INVALID_SOCK_CREATION_TIME) {
kill();
}
else {
// The pool takes care of deleting the failed connection - this
// will also trigger disposal of older connections in the pool
done();
}
}
else {
/* see done() comments above for why we log this line */
log() << "scoped connection to " << _conn->getServerAddress()
<< " not being returned to the pool" << endl;
kill();
}
}
}
void ScopedDbConnection::clearPool() {
pool.clear();
}
AtomicInt32 AScopedConnection::_numConnections;
} // namespace mongo
