void Strategy::doQuery( Request& r , const Shard& shard ) {
r.checkAuth( Auth::READ );
ShardConnection dbcon( shard , r.getns() );
DBClientBase &c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call( r.m(), response, true , &actualServer );
uassert( 10200 , "mongos: error calling db", ok );
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ) {
dbcon.done();
// Version is zero b/c this is deprecated codepath
throw RecvStaleConfigException( r.getns() , "Strategy::doQuery", ShardChunkVersion( 0 ), ShardChunkVersion( 0 ) );
}
}
r.reply( response , actualServer.size() ? actualServer : c.getServerAddress() );
dbcon.done();
}
void Strategy::doQuery( Request& r , const Shard& shard ){
try{
ShardConnection dbcon( shard , r.getns() );
DBClientBase &c = dbcon.conn();
Message response;
bool ok = c.call( r.m(), response);
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ){
dbcon.done();
throw StaleConfigException( r.getns() , "Strategy::doQuery" );
}
}
uassert( 10200 , "mongos: error calling db", ok);
r.reply( response , c.getServerAddress() );
dbcon.done();
}
catch ( AssertionException& e ) {
BSONObjBuilder err;
e.getInfo().append( err );
BSONObj errObj = err.done();
replyToQuery(ResultFlag_ErrSet, r.p() , r.m() , errObj);
}
}
/**
* Returns true if request is a query for sharded indexes.
*/
static bool doShardedIndexQuery(OperationContext* txn, Request& r, const QuerySpec& qSpec) {
// Extract the ns field from the query, which may be embedded within the "query" or
// "$query" field.
auto nsField = qSpec.filter()["ns"];
if (nsField.eoo()) {
return false;
}
const NamespaceString indexNSSQuery(nsField.str());
auto status = grid.catalogCache()->getDatabase(txn, indexNSSQuery.db().toString());
if (!status.isOK()) {
return false;
}
shared_ptr<DBConfig> config = status.getValue();
if (!config->isSharded(indexNSSQuery.ns())) {
return false;
}
// if you are querying on system.indexes, we need to make sure we go to a shard
// that actually has chunks. This is not a perfect solution (what if you just
// look at all indexes), but better than doing nothing.
ShardPtr shard;
ChunkManagerPtr cm;
config->getChunkManagerOrPrimary(indexNSSQuery.ns(), cm, shard);
if (cm) {
set<ShardId> shardIds;
cm->getAllShardIds(&shardIds);
verify(shardIds.size() > 0);
shard = grid.shardRegistry()->getShard(*shardIds.begin());
}
ShardConnection dbcon(shard->getConnString(), r.getns());
DBClientBase& c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call(r.m(), response, true, &actualServer);
uassert(10200, "mongos: error calling db", ok);
{
QueryResult::View qr = response.singleData().view2ptr();
if (qr.getResultFlags() & ResultFlag_ShardConfigStale) {
dbcon.done();
// Version is zero b/c this is deprecated codepath
throw RecvStaleConfigException(r.getns(),
"Strategy::doQuery",
ChunkVersion(0, 0, OID()),
ChunkVersion(0, 0, OID()));
}
}
r.reply(response, actualServer.size() ? actualServer : c.getServerAddress());
dbcon.done();
return true;
}
void Strategy::insert( const Shard& shard , const char * ns , const BSONObj& obj ) {
ShardConnection dbcon( shard , ns );
if ( dbcon.setVersion() ) {
dbcon.done();
throw StaleConfigException( ns , "for insert" );
}
dbcon->insert( ns , obj );
dbcon.done();
}
void Strategy::doWrite( int op , Request& r , const Shard& shard ){
ShardConnection dbcon( shard , r.getns() );
DBClientBase &_c = dbcon.conn();
/* TODO FIX - do not case and call DBClientBase::say() */
DBClientConnection&c = dynamic_cast<DBClientConnection&>(_c);
c.port().say( r.m() );
dbcon.done();
}
void Strategy::insert( const Shard& shard , const char * ns , const BSONObj& obj , int flags, bool safe ) {
ShardConnection dbcon( shard , ns );
if ( dbcon.setVersion() ) {
dbcon.done();
throw StaleConfigException( ns , "for insert" );
}
dbcon->insert( ns , obj , flags);
if (safe)
dbcon->getLastError();
dbcon.done();
}
void Strategy::insert( const Shard& shard , const char * ns , const vector<BSONObj>& v , int flags, bool safe ) {
ShardConnection dbcon( shard , ns );
if ( dbcon.setVersion() ) {
dbcon.done();
// Version is zero b/c we don't yet have a way to get the local version conflict
throw RecvStaleConfigException( ns , "for insert", ShardChunkVersion( 0 ), ShardChunkVersion( 0 ) );
}
dbcon->insert( ns , v , flags);
if (safe)
dbcon->getLastError();
dbcon.done();
}
void Strategy::update( const Shard& shard , const char * ns , const BSONObj& query , const BSONObj& toupdate , int flags ) {
bool upsert = flags & UpdateOption_Upsert;
bool multi = flags & UpdateOption_Multi;
ShardConnection dbcon( shard , ns );
if ( dbcon.setVersion() ) {
dbcon.done();
throw StaleConfigException( ns , "for insert" );
}
dbcon->update( ns , query , toupdate, upsert, multi);
dbcon.done();
}
/**
* Returns true if request is a query for sharded indexes.
*/
static bool doShardedIndexQuery( Request& r, const QuerySpec& qSpec ) {
// Extract the ns field from the query, which may be embedded within the "query" or
// "$query" field.
string indexNSQuery(qSpec.filter()["ns"].str());
DBConfigPtr config = grid.getDBConfig( r.getns() );
if ( !config->isSharded( indexNSQuery )) {
return false;
}
// if you are querying on system.indexes, we need to make sure we go to a shard
// that actually has chunks. This is not a perfect solution (what if you just
// look at all indexes), but better than doing nothing.
ShardPtr shard;
ChunkManagerPtr cm;
config->getChunkManagerOrPrimary( indexNSQuery, cm, shard );
if ( cm ) {
set<Shard> shards;
cm->getAllShards( shards );
verify( shards.size() > 0 );
shard.reset( new Shard( *shards.begin() ) );
}
ShardConnection dbcon( *shard , r.getns() );
DBClientBase &c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call( r.m(), response, true , &actualServer );
uassert( 10200 , "mongos: error calling db", ok );
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ) {
dbcon.done();
// Version is zero b/c this is deprecated codepath
throw RecvStaleConfigException( r.getns(),
"Strategy::doQuery",
ChunkVersion( 0, 0, OID() ),
ChunkVersion( 0, 0, OID() ));
}
}
r.reply( response , actualServer.size() ? actualServer : c.getServerAddress() );
dbcon.done();
return true;
}
void Strategy::update( const Shard& shard , const char * ns , const BSONObj& query , const BSONObj& toupdate , int flags, bool safe ) {
bool upsert = flags & UpdateOption_Upsert;
bool multi = flags & UpdateOption_Multi;
ShardConnection dbcon( shard , ns );
if ( dbcon.setVersion() ) {
dbcon.done();
// Version is zero b/c we don't yet have a way to get the local version conflict
throw RecvStaleConfigException( ns , "for insert", ShardChunkVersion( 0 ), ShardChunkVersion( 0 ) );
}
dbcon->update( ns , query , toupdate, upsert, multi);
if (safe)
dbcon->getLastError();
dbcon.done();
}
virtual void getMore( Request& r ) {
const char *ns = r.getns();
log(3) << "single getmore: " << ns << endl;
ShardConnection dbcon( r.primaryShard() );
DBClientBase& _c = dbcon.conn();
// TODO
DBClientConnection &c = dynamic_cast<DBClientConnection&>(_c);
Message response;
bool ok = c.port().call( r.m() , response);
uassert( 10204 , "dbgrid: getmore: error calling db", ok);
r.reply( response );
dbcon.done();
}
void Strategy::doQuery( Request& r , const Shard& shard ) {
ShardConnection dbcon( shard , r.getns() );
DBClientBase &c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call( r.m(), response, true , &actualServer );
uassert( 10200 , "mongos: error calling db", ok );
{
QueryResult *qr = (QueryResult *) response.singleData();
if ( qr->resultFlags() & ResultFlag_ShardConfigStale ) {
dbcon.done();
throw StaleConfigException( r.getns() , "Strategy::doQuery" );
}
}
r.reply( response , actualServer.size() ? actualServer : c.getServerAddress() );
dbcon.done();
}
///
/// \brief init
/// \param config
/// \return
///
int MysqlProxy::init(const Json::Value & config){
this->dbname = config["dbname"].asString();
this->dbnum = config["dbnum"].asInt();
this->user = config["user"].asString();
this->passwd = config["passwd"].asString();
const Json::Value & addrs = config["addrs"];
servers.resize(this->dbnum);
serverperdb.resize(this->dbnum);
for(unsigned int i=0;i<addrs.size();++i)
{
const Json::Value & dbsplit = addrs[i];
unsigned int first = dbsplit["first"].asInt();
unsigned int last = dbsplit["last"].asInt();
unsigned int moduleid = dbsplit["moduleid"].asInt();
std::string host;
unsigned int port;
if(moduleid == 0)
{
host = dbsplit["host"].asString();
port = dbsplit["port"].asInt();
}
/*
else if(0!= getIpAndPort(moduleid,0,host,port))
{
WARN("Get Host or Port fail,moduleid:"<<moduleid);
return -1;
}*/
std::string curuser = this->user;
std::string curpass = this->passwd;
if(dbsplit.isMember("user"))
curuser = dbsplit["user"].asString();
if(dbsplit.isMember("passwd"))
curpass = dbsplit["passwd"].asString();
for( unsigned int dbidx = first;dbidx <= last; ++dbidx)
{
servers[dbidx].moduleid = moduleid;
servers[dbidx].user = curuser;
servers[dbidx].passwd = curpass;
servers[dbidx].db = this->dbname;
servers[dbidx].host = host;
servers[dbidx].port = port;
}
}
const Json::Value & tables = config["tables"];
for(unsigned int tidx=0; tidx < tables.size();++tidx)
{
std::string table = tables[tidx]["table"].asString();
int num = tables[tidx]["tablenum"].asInt();
tablesplit[table] = num;
}
std::stringstream logstr;
//check connection,return if connection failed
logstr<<"Init DbProxy,db:"<<dbname<<",num:"<<dbnum<<",user:"******",password:"******",dbidx:"<<i<<",host:"<<servers[i].host<<",port:"<<servers[i].port<<",moduleid:"<<servers[i].moduleid;
ServerContext & sc = servers[i];
INFO("TEST connect to "<<",dbidx:"<<i<<",host:"<<servers[i].host<<",port:"<<servers[i].port<<",moduleid:"<<servers[i].moduleid);
SqlConnect dbcon(sc.host,sc.user,sc.passwd,sc.db,sc.port);
if(dbcon.check() !=0)
{
WARN("TEST fail:"<<",dbidx:"<<i<<",host:"<<servers[i].host<<",port:"<<servers[i].port<<",moduleid:"<<servers[i].moduleid);
return -1;
}
}
std::map<std::string ,int >::iterator it = tablesplit.begin();
for(; it != tablesplit.end();++it)
{
logstr<<",tables:"<<it->first<<",num:"<<it->second;
};
INFO(logstr.str());
return 0;
}
void _insert( Request& r , DbMessage& d, ChunkManagerPtr manager, vector<BSONObj>& insertsRemaining, map<ChunkPtr, vector<BSONObj> > insertsForChunks, int retries = 0 ) {
uassert( 16055, str::stream() << "too many retries during bulk insert, " << insertsRemaining.size() << " inserts remaining", retries < 30 );
uassert( 16056, str::stream() << "shutting down server during bulk insert, " << insertsRemaining.size() << " inserts remaining", ! inShutdown() );
const int flags = d.reservedField() | InsertOption_ContinueOnError; // ContinueOnError is always on when using sharding.
_groupInserts( manager, insertsRemaining, insertsForChunks );
while( ! insertsForChunks.empty() ){
ChunkPtr c = insertsForChunks.begin()->first;
vector<BSONObj>& objs = insertsForChunks.begin()->second;
const Shard& shard = c->getShard();
const string& ns = r.getns();
ShardConnection dbcon( shard, ns, manager );
try {
LOG(4) << " server:" << c->getShard().toString() << " bulk insert " << objs.size() << " documents" << endl;
// Taken from single-shard bulk insert, should not need multiple methods in future
// insert( c->getShard() , r.getns() , objs , flags);
// It's okay if the version is set here, an exception will be thrown if the version is incompatible
dbcon.setVersion();
dbcon->insert( ns , objs , flags);
// TODO: Option for safe inserts here - can then use this for all inserts
dbcon.done();
int bytesWritten = 0;
for (vector<BSONObj>::iterator vecIt = objs.begin(); vecIt != objs.end(); ++vecIt) {
r.gotInsert(); // Record the correct number of individual inserts
bytesWritten += (*vecIt).objsize();
}
// TODO: The only reason we're grouping by chunks here is for auto-split, more efficient
// to track this separately and bulk insert to shards
if ( r.getClientInfo()->autoSplitOk() )
c->splitIfShould( bytesWritten );
}
catch ( StaleConfigException& e ) {
// Cleanup the connection
dbcon.done();
// Assume the inserts did *not* succeed, so we don't want to erase them
int logLevel = retries < 2;
LOG( logLevel ) << "retrying bulk insert of " << objs.size() << " documents to chunk " << c << " because of StaleConfigException: " << e << endl;
if( retries > 2 ){
versionManager.forceRemoteCheckShardVersionCB( e.getns() );
}
// TODO: Replace with actual chunk handling code, simplify request
r.reset();
manager = r.getChunkManager();
if( ! manager ) {
// TODO : We can probably handle this better?
uasserted( 14804, "collection no longer sharded" );
}
// End TODO
// We may need to regroup at least some of our inserts since our chunk manager may have changed
_insert( r, d, manager, insertsRemaining, insertsForChunks, retries + 1 );
return;
}
catch( UserException& ){
// Unexpected exception, so don't clean up the conn
dbcon.kill();
// These inserts won't be retried, as something weird happened here
insertsForChunks.erase( insertsForChunks.begin() );
// Throw if this is the last chunk bulk-inserted to
if( insertsForChunks.empty() ){
throw;
}
}
insertsForChunks.erase( insertsForChunks.begin() );
}
}
void Strategy::insert( const Shard& shard , const char * ns , const BSONObj& obj ){
ShardConnection dbcon( shard , ns );
dbcon->insert( ns , obj );
dbcon.done();
}