bool DBClientReplicaSet::call( Message &toSend, Message &response, bool assertOk , string * actualServer ) {
if ( toSend.operation() == dbQuery ) {
// TODO: might be possible to do this faster by changing api
DbMessage dm( toSend );
QueryMessage qm( dm );
if ( qm.queryOptions & QueryOption_SlaveOk ) {
for ( int i=0; i<3; i++ ) {
try {
DBClientConnection* s = checkSlave();
if ( actualServer )
*actualServer = s->getServerAddress();
return s->call( toSend , response , assertOk );
}
catch ( DBException &e ) {
LOG(1) << "can't call replica set slave " << i << " : " << _slaveHost << causedBy( e ) << endl;
if ( actualServer )
*actualServer = "";
}
}
}
}
DBClientConnection* m = checkMaster();
if ( actualServer )
*actualServer = m->getServerAddress();
return m->call( toSend , response , assertOk );
}
DBClientConnection* DBClientReplicaSet::selectNodeUsingTags(
shared_ptr<ReadPreferenceSetting> readPref) {
if (!shouldReevaluate() && checkLastHost(readPref.get())) {
LOG( 3 ) << "dbclient_rs selecting compatible last used node " << _lastSlaveOkHost
<< endl;
return _lastSlaveOkConn.get();
}
ReplicaSetMonitorPtr monitor = _getMonitor();
_lastSlaveOkHost = monitor->getHostOrRefresh(*readPref);
if ( _lastSlaveOkHost.empty() ){
LOG( 3 ) << "dbclient_rs no compatible node found" << endl;
return NULL;
}
_lastReadPref = readPref;
// Primary connection is special because it is the only connection that is
// versioned in mongos. Therefore, we have to make sure that this object
// maintains only one connection to the primary and use that connection
// every time we need to talk to the primary.
if (monitor->isPrimary(_lastSlaveOkHost)) {
checkMaster();
_lastSlaveOkConn = _master;
_lastSlaveOkHost = _masterHost; // implied, but still assign just to be safe
LOG( 3 ) << "dbclient_rs selecting primary node " << _lastSlaveOkHost << endl;
return _master.get();
}
string errmsg;
ConnectionString connStr(_lastSlaveOkHost);
// Needs to perform a dynamic_cast because we need to set the replSet
// callback. We should eventually not need this after we remove the
// callback.
DBClientConnection* newConn = dynamic_cast<DBClientConnection*>(
connStr.connect(errmsg, _so_timeout));
// Assert here instead of returning NULL since the contract of this method is such
// that returning NULL means none of the nodes were good, which is not the case here.
uassert(16532, str::stream() << "Failed to connect to " << _lastSlaveOkHost.toString(),
newConn != NULL);
_lastSlaveOkConn.reset(newConn);
_lastSlaveOkConn->setReplSetClientCallback(this);
_lastSlaveOkConn->setRunCommandHook(_runCommandHook);
_lastSlaveOkConn->setPostRunCommandHook(_postRunCommandHook);
_auth(_lastSlaveOkConn.get());
LOG( 3 ) << "dbclient_rs selecting node " << _lastSlaveOkHost << endl;
return _lastSlaveOkConn.get();
}
bool DBClientReplicaSet::connect() {
try {
checkMaster();
}
catch (AssertionException&) {
return false;
}
return true;
}
BSONObj DBClientReplicaSet::findOne(const string &ns,
const Query& query,
const BSONObj *fieldsToReturn,
int queryOptions) {
shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( query.obj, queryOptions ) );
if ( _isSecondaryQuery( ns, query.obj, *readPref ) ) {
LOG( 3 ) << "dbclient_rs findOne using secondary or tagged node selection in "
<< _getMonitor()->getName() << ", read pref is "
<< readPref->toBSON() << " (primary : "
<< ( _master.get() != NULL ?
_master->getServerAddress() : "[not cached]" )
<< ", lastTagged : "
<< ( _lastSlaveOkConn.get() != NULL ?
_lastSlaveOkConn->getServerAddress() : "[not cached]" )
<< ")" << endl;
string lastNodeErrMsg;
for (size_t retry = 0; retry < MAX_RETRY; retry++) {
try {
DBClientConnection* conn = selectNodeUsingTags(readPref);
if (conn == NULL) {
break;
}
return conn->findOne(ns,query,fieldsToReturn,queryOptions);
}
catch ( const DBException &dbExcep ) {
StringBuilder errMsgBuilder;
errMsgBuilder << "can't findone replica set node "
<< _lastSlaveOkHost.toString() << ": " << causedBy( dbExcep );
lastNodeErrMsg = errMsgBuilder.str();
LOG(1) << lastNodeErrMsg << endl;
invalidateLastSlaveOkCache();
}
}
StringBuilder assertMsg;
assertMsg << "Failed to call findOne, no good nodes in " << _getMonitor()->getName();
if ( !lastNodeErrMsg.empty() ) {
assertMsg << ", last error: " << lastNodeErrMsg;
}
uasserted(16379, assertMsg.str());
}
LOG( 3 ) << "dbclient_rs findOne to primary node in " << _getMonitor()->getName()
<< endl;
return checkMaster()->findOne(ns,query,fieldsToReturn,queryOptions);
}
bool Manager::stepRun()
{
handleEvent();
bool ret = true;
switch (mStatus) {
case Idle:
start();
break;
case DealWait:
mStatus = Deal;
deal();
break;
case DealFinish:
checkMaster();
break;
case ChangeDealer:
checkMaster();
break;
case FixedMaster:
mStatus = PickBottom;
pickBottom();
break;
case BottomFinish:
mStatus = DiscardNotify;
mPerformer = mSeats[mDealer];
break;
case DiscardNotify:
discard();
break;
case DiscardWait:
//discard();
break;
case ThrowFail:
throwFailForceDiscard();
break;
default:
ret = false;
break;
}
handleEvent();
return ret;
}
bool DBClientReplicaSet::auth(const string &dbname, const string &username, const string &pwd, string& errmsg, bool digestPassword ) {
DBClientConnection * m = checkMaster();
// first make sure it actually works
if( ! m->auth(dbname, username, pwd, errmsg, digestPassword ) )
return false;
// now that it does, we should save so that for a new node we can auth
_auths.push_back( AuthInfo( dbname , username , pwd , digestPassword ) );
return true;
}
bool DBClientReplicaSet::connect() {
try {
checkMaster();
}
catch (AssertionException&) {
if (_master && _monitor) {
_monitor->notifyFailure(_masterHost);
}
return false;
}
return true;
}
void DBClientReplicaSet::checkResponse( const char* data, int nReturned, bool* retry, string* targetHost ){
// For now, do exactly as we did before, so as not to break things. In general though, we
// should fix this so checkResponse has a more consistent contract.
if( ! retry ){
if( _lazyState._lastClient )
return _lazyState._lastClient->checkResponse( data, nReturned );
else
return checkMaster()->checkResponse( data, nReturned );
}
*retry = false;
if( targetHost && _lazyState._lastClient ) *targetHost = _lazyState._lastClient->getServerAddress();
else *targetHost = "";
if( ! _lazyState._lastClient ) return;
if( nReturned != 1 && nReturned != -1 ) return;
BSONObj dataObj;
if( nReturned == 1 ) dataObj = BSONObj( data );
// Check if we should retry here
if( _lazyState._lastOp == dbQuery && _lazyState._slaveOk ){
// Check the error code for a slave not secondary error
if( nReturned == -1 ||
( hasErrField( dataObj ) && ! dataObj["code"].eoo() && dataObj["code"].Int() == 13436 ) ){
bool wasMaster = false;
if( _lazyState._lastClient == _slave.get() ){
isntSecondary();
}
else if( _lazyState._lastClient == _master.get() ){
wasMaster = true;
isntMaster();
}
else
warning() << "passed " << dataObj << " but last rs client " << _lazyState._lastClient->toString() << " is not master or secondary" << endl;
if( _lazyState._retries < 3 ){
_lazyState._retries++;
*retry = true;
}
else{
// assert( wasMaster );
// printStackTrace();
log() << "too many retries (" << _lazyState._retries << "), could not get data from replica set" << endl;
}
}
}
}
bool DBClientPaired::connect(const string &serverHostname1, const string &serverHostname2) {
string errmsg;
bool l = left.connect(serverHostname1, errmsg);
bool r = right.connect(serverHostname2, errmsg);
master = l ? NotSetL : NotSetR;
if ( !l && !r ) // it would be ok to fall through, but checkMaster will then try an immediate reconnect which is slow
return false;
try {
checkMaster();
}
catch (AssertionException&) {
return false;
}
return true;
}
bool DBClientPaired::auth(const string &dbname, const string &username, const string &pwd, string& errmsg) {
DBClientConnection& m = checkMaster();
if( !m.auth(dbname, username, pwd, errmsg) )
return false;
/* we try to authentiate with the other half of the pair -- even if down, that way the authInfo is cached. */
string e;
try {
if( &m == &left )
right.auth(dbname, username, pwd, e);
else
left.auth(dbname, username, pwd, e);
}
catch( AssertionException&) {
}
return true;
}
BSONObj DBClientReplicaSet::findOne(const string &ns, const Query& query, const BSONObj *fieldsToReturn, int queryOptions) {
if ( queryOptions & QueryOption_SlaveOk ) {
// we're ok sending to a slave
// we'll try 2 slaves before just using master
// checkSlave will try a different slave automatically after a failure
for ( int i=0; i<2; i++ ) {
try {
return checkSlave()->findOne(ns,query,fieldsToReturn,queryOptions);
}
catch ( DBException & ) {
LOG(1) << "can't query replica set slave: " << _slaveHost << endl;
}
}
}
return checkMaster()->findOne(ns,query,fieldsToReturn,queryOptions);
}
bool DBClientReplicaSet::call( Message &toSend, Message &response, bool assertOk ) {
if ( toSend.operation() == dbQuery ) {
// TODO: might be possible to do this faster by changing api
DbMessage dm( toSend );
QueryMessage qm( dm );
if ( qm.queryOptions & QueryOption_SlaveOk ) {
for ( int i=0; i<2; i++ ) {
try {
return checkSlave()->call( toSend , response , assertOk );
}
catch ( DBException & ) {
log(1) << "can't query replica set slave: " << _slaveHost << endl;
}
}
}
}
return checkMaster()->call( toSend , response , assertOk );
}
DBClientBase* DBClientReplicaSet::callLazy( Message& toSend ) {
if ( toSend.operation() == dbQuery ) {
// TODO: might be possible to do this faster by changing api
DbMessage dm( toSend );
QueryMessage qm( dm );
if ( qm.queryOptions & QueryOption_SlaveOk ) {
for ( int i=0; i<3; i++ ) {
try {
return checkSlave()->callLazy( toSend );
}
catch ( DBException &e ) {
LOG(1) << "can't callLazy replica set slave " << i << " : " << _slaveHost << causedBy( e ) << endl;
}
}
}
}
return checkMaster()->callLazy( toSend );
}
auto_ptr<DBClientCursor> DBClientReplicaSet::query(const string &ns, Query query, int nToReturn, int nToSkip,
const BSONObj *fieldsToReturn, int queryOptions, int batchSize) {
if ( queryOptions & QueryOption_SlaveOk ) {
// we're ok sending to a slave
// we'll try 2 slaves before just using master
// checkSlave will try a different slave automatically after a failure
for ( int i=0; i<2; i++ ) {
try {
return checkSlave()->query(ns,query,nToReturn,nToSkip,fieldsToReturn,queryOptions,batchSize);
}
catch ( DBException & ) {
LOG(1) << "can't query replica set slave: " << _slaveHost << endl;
}
}
}
return checkMaster()->query(ns,query,nToReturn,nToSkip,fieldsToReturn,queryOptions,batchSize);
}
void DBClientReplicaSet::say( Message& toSend, bool isRetry ) {
if( ! isRetry )
_lazyState = LazyState();
int lastOp = -1;
bool slaveOk = false;
if ( ( lastOp = toSend.operation() ) == dbQuery ) {
// TODO: might be possible to do this faster by changing api
DbMessage dm( toSend );
QueryMessage qm( dm );
if ( ( slaveOk = ( qm.queryOptions & QueryOption_SlaveOk ) ) ) {
for ( int i = _lazyState._retries; i < 3; i++ ) {
try {
DBClientConnection* slave = checkSlave();
slave->say( toSend );
_lazyState._lastOp = lastOp;
_lazyState._slaveOk = slaveOk;
_lazyState._retries = i;
_lazyState._lastClient = slave;
return;
}
catch ( DBException &e ) {
LOG(1) << "can't callLazy replica set slave " << i << " : " << _slaveHost << causedBy( e ) << endl;
}
}
}
}
DBClientConnection* master = checkMaster();
master->say( toSend );
_lazyState._lastOp = lastOp;
_lazyState._slaveOk = slaveOk;
_lazyState._retries = 3;
_lazyState._lastClient = master;
return;
}
void DBClientReplicaSet::logout(const string &dbname, BSONObj& info) {
DBClientConnection* priConn = checkMaster();
priConn->logout(dbname, info);
_auths.erase(dbname);
/* Also logout the cached secondary connection. Note that this is only
* needed when we actually have something cached and is last known to be
* working.
*/
if (_lastSlaveOkConn.get() != NULL && !_lastSlaveOkConn->isFailed()) {
try {
BSONObj dummy;
_lastSlaveOkConn->logout(dbname, dummy);
}
catch (const DBException&) {
// Make sure we can't use this connection again.
verify(_lastSlaveOkConn->isFailed());
}
}
}
void DBClientReplicaSet::killCursor( long long cursorID ) {
checkMaster()->killCursor( cursorID );
}
void DBClientReplicaSet::update( const string &ns , Query query , BSONObj obj , bool upsert , bool multi ) {
return checkMaster()->update(ns, query, obj, upsert,multi);
}
void DBClientReplicaSet::remove( const string &ns , Query obj , bool justOne ) {
checkMaster()->remove(ns, obj, justOne);
}
void DBClientReplicaSet::checkResponse( const char* data, int nReturned, bool* retry, string* targetHost ){
// For now, do exactly as we did before, so as not to break things. In general though, we
// should fix this so checkResponse has a more consistent contract.
if( ! retry ){
if( _lazyState._lastClient )
return _lazyState._lastClient->checkResponse( data, nReturned );
else
return checkMaster()->checkResponse( data, nReturned );
}
*retry = false;
if( targetHost && _lazyState._lastClient ) *targetHost = _lazyState._lastClient->getServerAddress();
else if (targetHost) *targetHost = "";
if( ! _lazyState._lastClient ) return;
// nReturned == 1 means that we got one result back, which might be an error
// nReturned == -1 is a sentinel value for "no data returned" aka (usually) network problem
// If neither, this must be a query result so our response is ok wrt the replica set
if( nReturned != 1 && nReturned != -1 ) return;
BSONObj dataObj;
if( nReturned == 1 ) dataObj = BSONObj( data );
// Check if we should retry here
if( _lazyState._lastOp == dbQuery && _lazyState._secondaryQueryOk ){
// query could potentially go to a secondary, so see if this is an error (or empty) and
// retry if we're not past our retry limit.
if( nReturned == -1 /* no result, maybe network problem */ ||
( hasErrField( dataObj ) && ! dataObj["code"].eoo()
&& dataObj["code"].Int() == NotMasterOrSecondaryCode ) ){
if( _lazyState._lastClient == _lastSlaveOkConn.get() ){
isntSecondary();
}
else if( _lazyState._lastClient == _master.get() ){
isntMaster();
}
else {
warning() << "passed " << dataObj << " but last rs client "
<< _lazyState._lastClient->toString() << " is not master or secondary"
<< endl;
}
if ( _lazyState._retries < static_cast<int>( MAX_RETRY ) ) {
_lazyState._retries++;
*retry = true;
}
else{
log() << "too many retries (" << _lazyState._retries
<< "), could not get data from replica set" << endl;
}
}
}
else if( _lazyState._lastOp == dbQuery ){
// if query could not potentially go to a secondary, just mark the master as bad
if( nReturned == -1 /* no result, maybe network problem */ ||
( hasErrField( dataObj ) && ! dataObj["code"].eoo()
&& dataObj["code"].Int() == NotMasterNoSlaveOkCode ) )
{
if( _lazyState._lastClient == _master.get() ){
isntMaster();
}
}
}
}
void DBClientReplicaSet::insert( const string &ns , BSONObj obj ) {
checkMaster()->insert(ns, obj);
}
void DBClientReplicaSet::remove( const string &ns , Query obj , int flags ) {
checkMaster()->remove(ns, obj, flags);
}
void DBClientReplicaSet::update( const string &ns , Query query , BSONObj obj , int flags ) {
return checkMaster()->update( ns, query, obj, flags );
}
auto_ptr<DBClientCursor> DBClientPaired::query(const string &a, Query b, int c, int d,
BSONObj *e, int f)
{
return checkMaster().query(a,b,c,d,e,f);
}
DBClientConnection& DBClientPaired::slaveConn(){
DBClientConnection& m = checkMaster();
assert( ! m.isFailed() );
return master == Left ? right : left;
}
bool DBClientReplicaSet::call(Message &toSend,
Message &response,
bool assertOk,
string * actualServer) {
const char * ns = 0;
if (toSend.operation() == dbQuery) {
// TODO: might be possible to do this faster by changing api
DbMessage dm(toSend);
QueryMessage qm(dm);
ns = qm.ns;
shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( qm.query,
qm.queryOptions ) );
if ( _isSecondaryQuery( ns, qm.query, *readPref ) ) {
LOG( 3 ) << "dbclient_rs call using secondary or tagged node selection in "
<< _getMonitor()->getName() << ", read pref is "
<< readPref->toBSON() << " (primary : "
<< ( _master.get() != NULL ?
_master->getServerAddress() : "[not cached]" )
<< ", lastTagged : "
<< ( _lastSlaveOkConn.get() != NULL ?
_lastSlaveOkConn->getServerAddress() : "[not cached]" )
<< ")" << endl;
for (size_t retry = 0; retry < MAX_RETRY; retry++) {
try {
DBClientConnection* conn = selectNodeUsingTags(readPref);
if (conn == NULL) {
return false;
}
if (actualServer != NULL) {
*actualServer = conn->getServerAddress();
}
return conn->call(toSend, response, assertOk);
}
catch ( const DBException& dbExcep ) {
LOG(1) << "can't call replica set node " << _lastSlaveOkHost << ": "
<< causedBy( dbExcep ) << endl;
if ( actualServer ) *actualServer = "";
invalidateLastSlaveOkCache();
}
}
// Was not able to successfully send after max retries
return false;
}
}
LOG( 3 ) << "dbclient_rs call to primary node in " << _getMonitor()->getName() << endl;
DBClientConnection* m = checkMaster();
if ( actualServer )
*actualServer = m->getServerAddress();
if ( ! m->call( toSend , response , assertOk ) )
return false;
if ( ns ) {
QueryResult * res = (QueryResult*)response.singleData();
if ( res->nReturned == 1 ) {
BSONObj x(res->data() );
if ( str::contains( ns , "$cmd" ) ) {
if ( isNotMasterErrorString( x["errmsg"] ) )
isntMaster();
}
else {
if ( isNotMasterErrorString( getErrField( x ) ) )
isntMaster();
}
}
}
return true;
}
void DBClientReplicaSet::insert( const string &ns, const vector< BSONObj >& v ) {
checkMaster()->insert(ns, v);
}
void DBClientReplicaSet::say(Message& toSend, bool isRetry, string* actualServer) {
if (!isRetry)
_lazyState = LazyState();
const int lastOp = toSend.operation();
if (lastOp == dbQuery) {
// TODO: might be possible to do this faster by changing api
DbMessage dm(toSend);
QueryMessage qm(dm);
shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( qm.query,
qm.queryOptions ) );
if ( _isSecondaryQuery( qm.ns, qm.query, *readPref ) ) {
LOG( 3 ) << "dbclient_rs say using secondary or tagged node selection in "
<< _getMonitor()->getName() << ", read pref is "
<< readPref->toBSON() << " (primary : "
<< ( _master.get() != NULL ?
_master->getServerAddress() : "[not cached]" )
<< ", lastTagged : "
<< ( _lastSlaveOkConn.get() != NULL ?
_lastSlaveOkConn->getServerAddress() : "[not cached]" )
<< ")" << endl;
string lastNodeErrMsg;
for (size_t retry = 0; retry < MAX_RETRY; retry++) {
_lazyState._retries = retry;
try {
DBClientConnection* conn = selectNodeUsingTags(readPref);
if (conn == NULL) {
break;
}
if (actualServer != NULL) {
*actualServer = conn->getServerAddress();
}
conn->say(toSend);
_lazyState._lastOp = lastOp;
_lazyState._secondaryQueryOk = true;
_lazyState._lastClient = conn;
}
catch ( const DBException& DBExcep ) {
StringBuilder errMsgBuilder;
errMsgBuilder << "can't callLazy replica set node "
<< _lastSlaveOkHost.toString() << ": " << causedBy( DBExcep );
lastNodeErrMsg = errMsgBuilder.str();
LOG(1) << lastNodeErrMsg << endl;
invalidateLastSlaveOkCache();
continue;
}
return;
}
StringBuilder assertMsg;
assertMsg << "Failed to call say, no good nodes in " << _getMonitor()->getName();
if ( !lastNodeErrMsg.empty() ) {
assertMsg << ", last error: " << lastNodeErrMsg;
}
uasserted(16380, assertMsg.str());
}
}
LOG( 3 ) << "dbclient_rs say to primary node in " << _getMonitor()->getName()
<< endl;
DBClientConnection* master = checkMaster();
if (actualServer)
*actualServer = master->getServerAddress();
_lazyState._lastOp = lastOp;
_lazyState._secondaryQueryOk = false;
// Don't retry requests to primary since there is only one host to try
_lazyState._retries = MAX_RETRY;
_lazyState._lastClient = master;
master->say(toSend);
return;
}
BSONObj DBClientPaired::findOne(const string &a, Query b, BSONObj *c, int d) {
return checkMaster().findOne(a,b,c,d);
}
DBClientConnection& DBClientReplicaSet::masterConn() {
return *checkMaster();
}
