bool AuthenticationInfo::_isAuthorizedSpecialChecks( const string& dbname ) const {
if ( !_isLocalHost ) {
return false;
}
string adminNs = "admin.system.users";
DBConfigPtr config = grid.getDBConfig( adminNs );
Shard s = config->getShard( adminNs );
ShardConnection conn( s, adminNs );
BSONObj result = conn->findOne("admin.system.users", Query());
if( result.isEmpty() ) {
if( ! _warned ) {
// you could get a few of these in a race, but that's ok
_warned = true;
log() << "note: no users configured in admin.system.users, allowing localhost access" << endl;
}
// Must return conn to pool
// TODO: Check for errors during findOne(), or just let the conn die?
conn.done();
return true;
}
// Must return conn to pool
conn.done();
return false;
}
bool CmdAuthenticate::getUserObj(const string& dbname, const string& user, BSONObj& userObj, string& pwd) {
if (user == internalSecurity.user) {
pwd = internalSecurity.pwd;
}
else {
string systemUsers = dbname + ".system.users";
DBConfigPtr config = grid.getDBConfig( systemUsers );
Shard s = config->getShard( systemUsers );
static BSONObj userPattern = BSON("user" << 1);
ShardConnection conn( s, systemUsers );
OCCASIONALLY conn->ensureIndex(systemUsers, userPattern, false, "user_1");
{
BSONObjBuilder b;
b << "user" << user;
BSONObj query = b.done();
userObj = conn->findOne(systemUsers, query);
if( userObj.isEmpty() ) {
log() << "auth: couldn't find user " << user << ", " << systemUsers << endl;
return false;
}
}
pwd = userObj.getStringField("pwd");
}
return true;
}
bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
ShardConnection::sync();
string ns = cmdObj.firstElement().valuestrsafe();
if ( ns.size() == 0 ){
errmsg = "no ns";
return false;
}
DBConfigPtr config = grid.getDBConfig( ns );
if ( ! config->isSharded( ns ) ){
errmsg = "ns not sharded. have to shard before can split";
return false;
}
BSONObj find = cmdObj.getObjectField( "find" );
if ( find.isEmpty() ){
find = cmdObj.getObjectField( "middle" );
if ( find.isEmpty() ){
errmsg = "need to specify find or middle";
return false;
}
}
ChunkManagerPtr info = config->getChunkManager( ns );
ChunkPtr old = info->findChunk( find );
return _split( result , errmsg , ns , info , old , cmdObj.getObjectField( "middle" ) );
}
bool CmdAuthenticate::getUserObj(const string& dbname, const string& user, BSONObj& userObj, string& pwd) {
if (user == internalSecurity.user) {
uassert(15890, "key file must be used to log in with internal user",
!cmdLine.keyFile.empty());
pwd = internalSecurity.pwd;
}
else {
string systemUsers = dbname + ".system.users";
DBConfigPtr config = grid.getDBConfig( systemUsers );
Shard s = config->getShard( systemUsers );
static BSONObj userPattern = BSON("user" << 1);
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getInternalScopedDbConnection( s.getConnString(), 30.0 ) );
OCCASIONALLY conn->get()->ensureIndex(systemUsers, userPattern, false, "user_1");
{
BSONObjBuilder b;
b << "user" << user;
BSONObj query = b.done();
userObj = conn->get()->findOne(systemUsers, query, 0, QueryOption_SlaveOk);
if( userObj.isEmpty() ) {
log() << "auth: couldn't find user " << user << ", " << systemUsers << endl;
conn->done(); // return to pool
return false;
}
}
pwd = userObj.getStringField("pwd");
conn->done(); // return to pool
}
return true;
}
int Balancer::_moveChunks( const vector<CandidateChunkPtr>* candidateChunks , bool secondaryThrottle ) {
int movedCount = 0;
for ( vector<CandidateChunkPtr>::const_iterator it = candidateChunks->begin(); it != candidateChunks->end(); ++it ) {
const CandidateChunk& chunkInfo = *it->get();
DBConfigPtr cfg = grid.getDBConfig( chunkInfo.ns );
verify( cfg );
ChunkManagerPtr cm = cfg->getChunkManager( chunkInfo.ns );
verify( cm );
ChunkPtr c = cm->findChunk( chunkInfo.chunk.min );
if ( c->getMin().woCompare( chunkInfo.chunk.min ) || c->getMax().woCompare( chunkInfo.chunk.max ) ) {
// likely a split happened somewhere
cm = cfg->getChunkManager( chunkInfo.ns , true /* reload */);
verify( cm );
c = cm->findChunk( chunkInfo.chunk.min );
if ( c->getMin().woCompare( chunkInfo.chunk.min ) || c->getMax().woCompare( chunkInfo.chunk.max ) ) {
log() << "chunk mismatch after reload, ignoring will retry issue " << chunkInfo.chunk.toString() << endl;
continue;
}
}
BSONObj res;
if ( c->moveAndCommit( Shard::make( chunkInfo.to ) , Chunk::MaxChunkSize , secondaryThrottle , res ) ) {
movedCount++;
continue;
}
// the move requires acquiring the collection metadata's lock, which can fail
log() << "balancer move failed: " << res << " from: " << chunkInfo.from << " to: " << chunkInfo.to
<< " chunk: " << chunkInfo.chunk << endl;
if ( res["chunkTooBig"].trueValue() ) {
// reload just to be safe
cm = cfg->getChunkManager( chunkInfo.ns );
verify( cm );
c = cm->findChunk( chunkInfo.chunk.min );
log() << "forcing a split because migrate failed for size reasons" << endl;
res = BSONObj();
c->singleSplit( true , res );
log() << "forced split results: " << res << endl;
if ( ! res["ok"].trueValue() ) {
log() << "marking chunk as jumbo: " << c->toString() << endl;
c->markAsJumbo();
// we increment moveCount so we do another round right away
movedCount++;
}
}
}
return movedCount;
}
// TODO: This refresh logic should be consolidated
void refreshChunkCache( const NamespaceString& nss ) {
DBConfigPtr config = grid.getDBConfig( nss.ns() );
if ( !config->isSharded( nss ) ) return;
// Refreshes chunks as a side-effect
config->getChunkManagerIfExists( nss, true );
}
bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
ShardConnection::sync();
string ns = cmdObj.firstElement().valuestrsafe();
if ( ns.size() == 0 ) {
errmsg = "no ns";
return false;
}
DBConfigPtr config = grid.getDBConfig( ns );
if ( ! config->isSharded( ns ) ) {
errmsg = "ns not sharded. have to shard before can split";
return false;
}
BSONObj find = cmdObj.getObjectField( "find" );
if ( find.isEmpty() ) {
find = cmdObj.getObjectField( "middle" );
if ( find.isEmpty() ) {
errmsg = "need to specify find or middle";
return false;
}
}
ChunkManagerPtr info = config->getChunkManager( ns );
ChunkPtr chunk = info->findChunk( find );
BSONObj middle = cmdObj.getObjectField( "middle" );
assert( chunk.get() );
log() << "splitting: " << ns << " shard: " << chunk << endl;
BSONObj res;
ChunkPtr p;
if ( middle.isEmpty() ) {
p = chunk->singleSplit( true /* force a split even if not enough data */ , res );
}
else {
// sanity check if the key provided is a valid split point
if ( ( middle == chunk->getMin() ) || ( middle == chunk->getMax() ) ) {
errmsg = "cannot split on initial or final chunk's key";
return false;
}
vector<BSONObj> splitPoints;
splitPoints.push_back( middle );
p = chunk->multiSplit( splitPoints , res );
}
if ( p.get() == NULL ) {
errmsg = "split failed";
result.append( "cause" , res );
return false;
}
return true;
}
bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
ShardConnection::sync();
Timer t;
string ns = cmdObj.firstElement().valuestrsafe();
if ( ns.size() == 0 ){
errmsg = "no ns";
return false;
}
DBConfigPtr config = grid.getDBConfig( ns );
if ( ! config->isSharded( ns ) ){
errmsg = "ns not sharded. have to shard before can move a chunk";
return false;
}
BSONObj find = cmdObj.getObjectField( "find" );
if ( find.isEmpty() ){
errmsg = "need to specify find. see help";
return false;
}
string toString = cmdObj["to"].valuestrsafe();
if ( ! toString.size() ){
errmsg = "you have to specify where you want to move the chunk";
return false;
}
Shard to = Shard::make( toString );
// so far, chunk size serves test purposes; it may or may not become a supported parameter
long long maxChunkSizeBytes = cmdObj["maxChunkSizeBytes"].numberLong();
if ( maxChunkSizeBytes == 0 ) {
maxChunkSizeBytes = Chunk::MaxChunkSize;
}
tlog() << "CMD: movechunk: " << cmdObj << endl;
ChunkManagerPtr info = config->getChunkManager( ns );
ChunkPtr c = info->findChunk( find );
const Shard& from = c->getShard();
if ( from == to ){
errmsg = "that chunk is already on that shard";
return false;
}
BSONObj res;
if ( ! c->moveAndCommit( to , maxChunkSizeBytes , res ) ){
errmsg = "move failed";
result.append( "cause" , res );
return false;
}
result.append( "millis" , t.millis() );
return true;
}
int Balancer::_moveChunks( const vector<CandidateChunkPtr>* candidateChunks ) {
int movedCount = 0;
for ( vector<CandidateChunkPtr>::const_iterator it = candidateChunks->begin(); it != candidateChunks->end(); ++it ) {
const CandidateChunk& chunkInfo = *it->get();
DBConfigPtr cfg = grid.getDBConfig( chunkInfo.ns );
assert( cfg );
ChunkManagerPtr cm = cfg->getChunkManager( chunkInfo.ns );
assert( cm );
const BSONObj& chunkToMove = chunkInfo.chunk;
ChunkPtr c = cm->findChunk( chunkToMove["min"].Obj() );
if ( c->getMin().woCompare( chunkToMove["min"].Obj() ) || c->getMax().woCompare( chunkToMove["max"].Obj() ) ) {
// likely a split happened somewhere
cm = cfg->getChunkManager( chunkInfo.ns , true /* reload */);
assert( cm );
c = cm->findChunk( chunkToMove["min"].Obj() );
if ( c->getMin().woCompare( chunkToMove["min"].Obj() ) || c->getMax().woCompare( chunkToMove["max"].Obj() ) ) {
log() << "chunk mismatch after reload, ignoring will retry issue cm: "
<< c->getMin() << " min: " << chunkToMove["min"].Obj() << endl;
continue;
}
}
BSONObj res;
if ( c->moveAndCommit( Shard::make( chunkInfo.to ) , Chunk::MaxChunkSize , res ) ) {
movedCount++;
continue;
}
// the move requires acquiring the collection metadata's lock, which can fail
log() << "balacer move failed: " << res << " from: " << chunkInfo.from << " to: " << chunkInfo.to
<< " chunk: " << chunkToMove << endl;
if ( res["chunkTooBig"].trueValue() ) {
// reload just to be safe
cm = cfg->getChunkManager( chunkInfo.ns );
assert( cm );
c = cm->findChunk( chunkToMove["min"].Obj() );
log() << "forcing a split because migrate failed for size reasons" << endl;
res = BSONObj();
c->singleSplit( true , res );
log() << "forced split results: " << res << endl;
// TODO: if the split fails, mark as jumbo SERVER-2571
}
}
return movedCount;
}
void insertSharded( DBConfigPtr conf, const char* ns, BSONObj& o, int flags ) {
ChunkManagerPtr manager = conf->getChunkManager(ns);
if ( ! manager->hasShardKey( o ) ) {
bool bad = true;
if ( manager->getShardKey().partOfShardKey( "_id" ) ) {
BSONObjBuilder b;
b.appendOID( "_id" , 0 , true );
b.appendElements( o );
o = b.obj();
bad = ! manager->hasShardKey( o );
}
if ( bad ) {
log() << "tried to insert object without shard key: " << ns << " " << o << endl;
uasserted( 14842 , "tried to insert object without shard key" );
}
}
// Many operations benefit from having the shard key early in the object
o = manager->getShardKey().moveToFront(o);
const int maxTries = 30;
for ( int i=0; i<maxTries; i++ ) {
try {
ChunkPtr c = manager->findChunk( o );
log(4) << " server:" << c->getShard().toString() << " " << o << endl;
insert( c->getShard() , ns , o , flags);
// r.gotInsert();
// if ( r.getClientInfo()->autoSplitOk() )
c->splitIfShould( o.objsize() );
break;
}
catch ( StaleConfigException& e ) {
int logLevel = i < ( maxTries / 2 );
LOG( logLevel ) << "retrying insert because of StaleConfigException: " << e << " object: " << o << endl;
// r.reset();
unsigned long long old = manager->getSequenceNumber();
manager = conf->getChunkManager(ns);
LOG( logLevel ) << " sequenece number - old: " << old << " new: " << manager->getSequenceNumber() << endl;
if (!manager) {
uasserted(14843, "collection no longer sharded");
}
}
sleepmillis( i * 20 );
}
}
void checkShardVersion( DBClientBase& conn , const string& ns , bool authoritative ){
// TODO: cache, optimize, etc...
WriteBackListener::init( conn );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return;
ShardChunkVersion version = 0;
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ){
manager = conf->getChunkManager( ns , authoritative );
officialSequenceNumber = manager->getSequenceNumber();
}
unsigned long long & sequenceNumber = checkShardVersionLastSequence[ make_pair(&conn,ns) ];
if ( sequenceNumber == officialSequenceNumber )
return;
if ( isSharded ){
version = manager->getVersion( Shard::make( conn.getServerAddress() ) );
}
log(2) << " have to set shard version for conn: " << &conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( conn , ns , version , authoritative , result ) ){
// success!
log(1) << " setShardVersion success!" << endl;
sequenceNumber = officialSequenceNumber;
dassert( sequenceNumber == checkShardVersionLastSequence[ make_pair(&conn,ns) ] );
return;
}
log(1) << " setShardVersion failed!\n" << result << endl;
if ( result.getBoolField( "need_authoritative" ) )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ){
checkShardVersion( conn , ns , 1 );
return;
}
log() << " setShardVersion failed: " << result << endl;
massert( 10429 , (string)"setShardVersion failed! " + result.jsonString() , 0 );
}
bool forceRemoteCheckShardVersion( const string& ns ){
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf ) return false;
conf->reload();
ChunkManagerPtr manager = conf->getChunkManagerIfExists( ns, true, true );
if( ! manager ) return false;
return true;
}
void ClientInfo::_setupAuth() {
std::string adminNs = "admin";
DBConfigPtr config = grid.getDBConfig(adminNs);
Shard shard = config->getShard(adminNs);
ShardConnection conn(shard, adminNs);
AuthorizationManager* authManager = new AuthorizationManager(new AuthExternalStateImpl());
Status status = authManager->initialize(conn.get());
massert(16479,
mongoutils::str::stream() << "Error initializing AuthorizationManager: "
<< status.reason(),
status == Status::OK());
setAuthorizationManager(authManager);
}
/**
* Returns true if request is a query for sharded indexes.
*/
static bool doShardedIndexQuery( Request& r, const QuerySpec& qSpec ) {
// Extract the ns field from the query, which may be embedded within the "query" or
// "$query" field.
string indexNSQuery(qSpec.filter()["ns"].str());
DBConfigPtr config = grid.getDBConfig( r.getns() );
if ( !config->isSharded( indexNSQuery )) {
return false;
}
// if you are querying on system.indexes, we need to make sure we go to a shard
// that actually has chunks. This is not a perfect solution (what if you just
// look at all indexes), but better than doing nothing.
ShardPtr shard;
ChunkManagerPtr cm;
config->getChunkManagerOrPrimary( indexNSQuery, cm, shard );
if ( cm ) {
set<Shard> shards;
cm->getAllShards( shards );
verify( shards.size() > 0 );
shard.reset( new Shard( *shards.begin() ) );
}
ShardConnection dbcon( *shard , r.getns() );
DBClientBase &c = dbcon.conn();
string actualServer;
Message response;
bool ok = c.call( r.m(), response, true , &actualServer );
uassert( 10200 , "mongos: error calling db", ok );
{
QueryResult::View qr = response.singleData().view2ptr();
if ( qr.getResultFlags() & ResultFlag_ShardConfigStale ) {
dbcon.done();
// Version is zero b/c this is deprecated codepath
throw RecvStaleConfigException( r.getns(),
"Strategy::doQuery",
ChunkVersion( 0, 0, OID() ),
ChunkVersion( 0, 0, OID() ));
}
}
r.reply( response , actualServer.size() ? actualServer : c.getServerAddress() );
dbcon.done();
return true;
}
bool VersionManager::forceRemoteCheckShardVersionCB( const string& ns ){
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf ) return false;
conf->reload();
// If we don't have a collection, don't refresh the chunk manager
if( nsGetCollection( ns ).size() == 0 ) return false;
ChunkManagerPtr manager = conf->getChunkManagerIfExists( ns, true, true );
if( ! manager ) return false;
return true;
}
// TODO: Same limitations as other mongos metadata commands, sometimes we'll be stale here
// and fail. Need to better integrate targeting with commands.
ShardPtr guessMergeShard( const NamespaceString& nss, const BSONObj& minKey ) {
DBConfigPtr config = grid.getDBConfig( nss.ns() );
if ( !config->isSharded( nss ) ) {
config->reload();
if ( !config->isSharded( nss ) ) {
return ShardPtr();
}
}
ChunkManagerPtr manager = config->getChunkManager( nss );
if ( !manager ) return ShardPtr();
ChunkPtr chunk = manager->findChunkForDoc( minKey );
if ( !chunk ) return ShardPtr();
return ShardPtr( new Shard( chunk->getShard() ) );
}
Status ChunkManagerTargeter::init( const NamespaceString& nss ) {
_nss = nss;
//
// Get the latest metadata information from the cache
//
DBConfigPtr config;
string errMsg;
if ( !getDBConfigSafe( _nss.db(), config, &errMsg ) ) {
return Status( ErrorCodes::DatabaseNotFound, errMsg );
}
// Get either the chunk manager or primary shard
config->getChunkManagerOrPrimary( _nss.ns(), _manager, _primary );
return Status::OK();
}
/**
* Splits the chunks touched based from the targeter stats if needed.
*/
static void splitIfNeeded( const string& ns, const TargeterStats& stats ) {
if ( !Chunk::ShouldAutoSplit ) {
return;
}
DBConfigPtr config;
try {
config = grid.getDBConfig( ns );
}
catch ( const DBException& ex ) {
warning() << "failed to get database config for " << ns
<< " while checking for auto-split: " << causedBy( ex ) << endl;
return;
}
ChunkManagerPtr chunkManager;
ShardPtr dummyShard;
config->getChunkManagerOrPrimary( ns, chunkManager, dummyShard );
if ( !chunkManager ) {
return;
}
for ( map<BSONObj, int>::const_iterator it = stats.chunkSizeDelta.begin();
it != stats.chunkSizeDelta.end(); ++it ) {
ChunkPtr chunk;
try {
chunk = chunkManager->findIntersectingChunk( it->first );
}
catch ( const AssertionException& ex ) {
warning() << "could not find chunk while checking for auto-split: "
<< causedBy( ex ) << endl;
return;
}
chunk->splitIfShould( it->second );
}
}
void ClientInfo::_setupAuth() {
std::string adminNs = "admin";
DBConfigPtr config = grid.getDBConfig(adminNs);
Shard shard = config->getShard(adminNs);
scoped_ptr<ScopedDbConnection> connPtr(
ScopedDbConnection::getInternalScopedDbConnection(shard.getConnString(), 30.0));
ScopedDbConnection& conn = *connPtr;
//
// Note: The connection mechanism here is *not* ideal, and should not be used elsewhere.
// It is safe in this particular case because the admin database is always on the config
// server and does not move.
//
AuthorizationManager* authManager = new AuthorizationManager(new AuthExternalStateImpl());
Status status = authManager->initialize(conn.get());
massert(16479,
mongoutils::str::stream() << "Error initializing AuthorizationManager: "
<< status.reason(),
status == Status::OK());
setAuthorizationManager(authManager);
}
int Balancer::_moveChunks( const vector<CandidateChunkPtr>* candidateChunks ) {
int movedCount = 0;
for ( vector<CandidateChunkPtr>::const_iterator it = candidateChunks->begin(); it != candidateChunks->end(); ++it ){
const CandidateChunk& chunkInfo = *it->get();
DBConfigPtr cfg = grid.getDBConfig( chunkInfo.ns );
assert( cfg );
ChunkManagerPtr cm = cfg->getChunkManager( chunkInfo.ns );
assert( cm );
const BSONObj& chunkToMove = chunkInfo.chunk;
ChunkPtr c = cm->findChunk( chunkToMove["min"].Obj() );
if ( c->getMin().woCompare( chunkToMove["min"].Obj() ) ){
// likely a split happened somewhere
cm = cfg->getChunkManager( chunkInfo.ns , true );
assert( cm );
c = cm->findChunk( chunkToMove["min"].Obj() );
if ( c->getMin().woCompare( chunkToMove["min"].Obj() ) ){
log() << "balancer: chunk mismatch after reload, ignoring will retry issue cm: "
<< c->getMin() << " min: " << chunkToMove["min"].Obj() << endl;
continue;
}
}
string errmsg;
if ( c->moveAndCommit( Shard::make( chunkInfo.to ) , errmsg ) ){
movedCount++;
continue;
}
log() << "balancer: MOVE FAILED **** " << errmsg << "\n"
<< " from: " << chunkInfo.from << " to: " << " chunk: " << chunkToMove << endl;
}
return movedCount;
}
bool run(const string& , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) {
string ns = cmdObj.firstElement().valuestrsafe();
if ( ns.size() == 0 ) {
errmsg = "need to specify fully namespace";
return false;
}
DBConfigPtr config = grid.getDBConfig( ns );
if ( ! config->isSharded( ns ) ) {
errmsg = "ns not sharded.";
return false;
}
ChunkManagerPtr cm = config->getChunkManagerIfExists( ns );
if ( ! cm ) {
errmsg = "no chunk manager?";
return false;
}
cm->_printChunks();
result.appendTimestamp( "version" , cm->getVersion().toLong() );
return 1;
}
void AuthenticationInfo::_checkLocalHostSpecialAdmin() {
if (noauth || !_isLocalHost || !_isLocalHostAndLocalHostIsAuthorizedForAll) {
return;
}
string adminNs = "admin.system.users";
DBConfigPtr config = grid.getDBConfig( adminNs );
Shard s = config->getShard( adminNs );
//
// Note: The connection mechanism here is *not* ideal, and should not be used elsewhere.
// It is safe in this particular case because the admin database is always on the config
// server and does not move.
//
scoped_ptr<ScopedDbConnection> conn(
ScopedDbConnection::getInternalScopedDbConnection(s.getConnString(), 30.0));
BSONObj result = (*conn)->findOne("admin.system.users", Query());
if( result.isEmpty() ) {
if( ! _warned ) {
// you could get a few of these in a race, but that's ok
_warned = true;
log() << "note: no users configured in admin.system.users, allowing localhost access" << endl;
}
// Must return conn to pool
// TODO: Check for errors during findOne(), or just let the conn die?
conn->done();
_isLocalHostAndLocalHostIsAuthorizedForAll = true;
return;
}
// Must return conn to pool
conn->done();
_isLocalHostAndLocalHostIsAuthorizedForAll = false;
}
Status ChunkManagerTargeter::refreshNow( RefreshType refreshType ) {
DBConfigPtr config;
string errMsg;
if ( !getDBConfigSafe( _nss.db(), config, &errMsg ) ) {
return Status( ErrorCodes::DatabaseNotFound, errMsg );
}
// Try not to spam the configs
refreshBackoff();
// TODO: Improve synchronization and make more explicit
if ( refreshType == RefreshType_RefreshChunkManager ) {
try {
// Forces a remote check of the collection info, synchronization between threads
// happens internally.
config->getChunkManagerIfExists( _nss.ns(), true );
}
catch ( const DBException& ex ) {
return Status( ErrorCodes::UnknownError, ex.toString() );
}
config->getChunkManagerOrPrimary( _nss.ns(), _manager, _primary );
}
else if ( refreshType == RefreshType_ReloadDatabase ) {
try {
// Dumps the db info, reloads it all, synchronization between threads happens
// internally.
config->reload();
config->getChunkManagerIfExists( _nss.ns(), true, true );
}
catch ( const DBException& ex ) {
return Status( ErrorCodes::UnknownError, ex.toString() );
}
config->getChunkManagerOrPrimary( _nss.ns(), _manager, _primary );
}
return Status::OK();
}
void Balancer::_doBalanceRound( DBClientBase& conn, vector<CandidateChunkPtr>* candidateChunks ) {
verify( candidateChunks );
//
// 1. Check whether there is any sharded collection to be balanced by querying
// the ShardsNS::collections collection
//
auto_ptr<DBClientCursor> cursor = conn.query(CollectionType::ConfigNS, BSONObj());
if ( NULL == cursor.get() ) {
warning() << "could not query " << CollectionType::ConfigNS
<< " while trying to balance" << endl;
return;
}
vector< string > collections;
while ( cursor->more() ) {
BSONObj col = cursor->nextSafe();
// sharded collections will have a shard "key".
if ( ! col[CollectionType::keyPattern()].eoo() &&
! col[CollectionType::noBalance()].trueValue() ){
collections.push_back( col[CollectionType::ns()].String() );
}
else if( col[CollectionType::noBalance()].trueValue() ){
LOG(1) << "not balancing collection " << col[CollectionType::ns()].String()
<< ", explicitly disabled" << endl;
}
}
cursor.reset();
if ( collections.empty() ) {
LOG(1) << "no collections to balance" << endl;
return;
}
//
// 2. Get a list of all the shards that are participating in this balance round
// along with any maximum allowed quotas and current utilization. We get the
// latter by issuing db.serverStatus() (mem.mapped) to all shards.
//
// TODO: skip unresponsive shards and mark information as stale.
//
ShardInfoMap shardInfo;
Status loadStatus = DistributionStatus::populateShardInfoMap(&shardInfo);
if (!loadStatus.isOK()) {
warning() << "failed to load shard metadata" << causedBy(loadStatus) << endl;
return;
}
if (shardInfo.size() < 2) {
LOG(1) << "can't balance without more active shards" << endl;
return;
}
OCCASIONALLY warnOnMultiVersion( shardInfo );
//
// 3. For each collection, check if the balancing policy recommends moving anything around.
//
for (vector<string>::const_iterator it = collections.begin(); it != collections.end(); ++it ) {
const string& ns = *it;
OwnedPointerMap<string, OwnedPointerVector<ChunkType> > shardToChunksMap;
cursor = conn.query(ChunkType::ConfigNS,
QUERY(ChunkType::ns(ns)).sort(ChunkType::min()));
set<BSONObj> allChunkMinimums;
while ( cursor->more() ) {
BSONObj chunkDoc = cursor->nextSafe().getOwned();
auto_ptr<ChunkType> chunk(new ChunkType());
string errmsg;
if (!chunk->parseBSON(chunkDoc, &errmsg)) {
error() << "bad chunk format for " << chunkDoc
<< ": " << errmsg << endl;
return;
}
allChunkMinimums.insert(chunk->getMin().getOwned());
OwnedPointerVector<ChunkType>*& chunkList =
shardToChunksMap.mutableMap()[chunk->getShard()];
if (chunkList == NULL) {
chunkList = new OwnedPointerVector<ChunkType>();
}
chunkList->mutableVector().push_back(chunk.release());
}
cursor.reset();
if (shardToChunksMap.map().empty()) {
LOG(1) << "skipping empty collection (" << ns << ")";
continue;
}
for (ShardInfoMap::const_iterator i = shardInfo.begin(); i != shardInfo.end(); ++i) {
// this just makes sure there is an entry in shardToChunksMap for every shard
OwnedPointerVector<ChunkType>*& chunkList =
shardToChunksMap.mutableMap()[i->first];
if (chunkList == NULL) {
chunkList = new OwnedPointerVector<ChunkType>();
}
}
DistributionStatus status(shardInfo, shardToChunksMap.map());
// load tags
Status result = clusterCreateIndex(TagsType::ConfigNS,
BSON(TagsType::ns() << 1 << TagsType::min() << 1),
true, // unique
WriteConcernOptions::AllConfigs,
NULL);
if ( !result.isOK() ) {
warning() << "could not create index tags_1_min_1: " << result.reason() << endl;
continue;
}
cursor = conn.query(TagsType::ConfigNS,
QUERY(TagsType::ns(ns)).sort(TagsType::min()));
vector<TagRange> ranges;
while ( cursor->more() ) {
BSONObj tag = cursor->nextSafe();
TagRange tr(tag[TagsType::min()].Obj().getOwned(),
tag[TagsType::max()].Obj().getOwned(),
tag[TagsType::tag()].String());
ranges.push_back(tr);
uassert(16356,
str::stream() << "tag ranges not valid for: " << ns,
status.addTagRange(tr) );
}
cursor.reset();
DBConfigPtr cfg = grid.getDBConfig( ns );
if ( !cfg ) {
warning() << "could not load db config to balance " << ns << " collection" << endl;
continue;
}
// This line reloads the chunk manager once if this process doesn't know the collection
// is sharded yet.
ChunkManagerPtr cm = cfg->getChunkManagerIfExists( ns, true );
if ( !cm ) {
warning() << "could not load chunks to balance " << ns << " collection" << endl;
continue;
}
// loop through tags to make sure no chunk spans tags; splits on tag min. for all chunks
bool didAnySplits = false;
for ( unsigned i = 0; i < ranges.size(); i++ ) {
BSONObj min = ranges[i].min;
min = cm->getShardKey().extendRangeBound( min, false );
if ( allChunkMinimums.count( min ) > 0 )
continue;
didAnySplits = true;
log() << "ns: " << ns << " need to split on "
<< min << " because there is a range there" << endl;
ChunkPtr c = cm->findIntersectingChunk( min );
vector<BSONObj> splitPoints;
splitPoints.push_back( min );
BSONObj res;
if ( !c->multiSplit( splitPoints, res ) ) {
error() << "split failed: " << res << endl;
}
else {
LOG(1) << "split worked: " << res << endl;
}
break;
}
if ( didAnySplits ) {
// state change, just wait till next round
continue;
}
CandidateChunk* p = _policy->balance( ns, status, _balancedLastTime );
if ( p ) candidateChunks->push_back( CandidateChunkPtr( p ) );
}
}
bool Grid::addShard( string* name , const ConnectionString& servers , long long maxSize , string& errMsg ) {
// name can be NULL, so provide a dummy one here to avoid testing it elsewhere
string nameInternal;
if ( ! name ) {
name = &nameInternal;
}
ReplicaSetMonitorPtr rsMonitor;
// Check whether the host (or set) exists and run several sanity checks on this request.
// There are two set of sanity checks: making sure adding this particular shard is consistent
// with the replica set state (if it exists) and making sure this shards databases can be
// brought into the grid without conflict.
vector<string> dbNames;
try {
ScopedDbConnection newShardConn(servers.toString());
newShardConn->getLastError();
if ( newShardConn->type() == ConnectionString::SYNC ) {
newShardConn.done();
errMsg = "can't use sync cluster as a shard. for replica set, have to use <setname>/<server1>,<server2>,...";
return false;
}
BSONObj resIsMongos;
bool ok = newShardConn->runCommand( "admin" , BSON( "isdbgrid" << 1 ) , resIsMongos );
// should return ok=0, cmd not found if it's a normal mongod
if ( ok ) {
errMsg = "can't add a mongos process as a shard";
newShardConn.done();
return false;
}
BSONObj resIsMaster;
ok = newShardConn->runCommand( "admin" , BSON( "isMaster" << 1 ) , resIsMaster );
if ( !ok ) {
ostringstream ss;
ss << "failed running isMaster: " << resIsMaster;
errMsg = ss.str();
newShardConn.done();
return false;
}
// if the shard has only one host, make sure it is not part of a replica set
string setName = resIsMaster["setName"].str();
string commandSetName = servers.getSetName();
if ( commandSetName.empty() && ! setName.empty() ) {
ostringstream ss;
ss << "host is part of set " << setName << ", use replica set url format <setname>/<server1>,<server2>,....";
errMsg = ss.str();
newShardConn.done();
return false;
}
if ( !commandSetName.empty() && setName.empty() ) {
ostringstream ss;
ss << "host did not return a set name, is the replica set still initializing? " << resIsMaster;
errMsg = ss.str();
newShardConn.done();
return false;
}
// if the shard is part of replica set, make sure it is the right one
if ( ! commandSetName.empty() && ( commandSetName != setName ) ) {
ostringstream ss;
ss << "host is part of a different set: " << setName;
errMsg = ss.str();
newShardConn.done();
return false;
}
if( setName.empty() ) {
// check this isn't a --configsvr
BSONObj res;
bool ok = newShardConn->runCommand("admin",BSON("replSetGetStatus"<<1),res);
ostringstream ss;
if( !ok && res["info"].type() == String && res["info"].String() == "configsvr" ) {
errMsg = "the specified mongod is a --configsvr and should thus not be a shard server";
newShardConn.done();
return false;
}
}
// if the shard is part of a replica set, make sure all the hosts mentioned in 'servers' are part of
// the set. It is fine if not all members of the set are present in 'servers'.
bool foundAll = true;
string offendingHost;
if ( ! commandSetName.empty() ) {
set<string> hostSet;
BSONObjIterator iter( resIsMaster["hosts"].Obj() );
while ( iter.more() ) {
hostSet.insert( iter.next().String() ); // host:port
}
if ( resIsMaster["passives"].isABSONObj() ) {
BSONObjIterator piter( resIsMaster["passives"].Obj() );
while ( piter.more() ) {
hostSet.insert( piter.next().String() ); // host:port
}
}
if ( resIsMaster["arbiters"].isABSONObj() ) {
BSONObjIterator piter( resIsMaster["arbiters"].Obj() );
while ( piter.more() ) {
hostSet.insert( piter.next().String() ); // host:port
}
}
vector<HostAndPort> hosts = servers.getServers();
for ( size_t i = 0 ; i < hosts.size() ; i++ ) {
if (!hosts[i].hasPort()) {
hosts[i].setPort(ServerGlobalParams::DefaultDBPort);
}
string host = hosts[i].toString(); // host:port
if ( hostSet.find( host ) == hostSet.end() ) {
offendingHost = host;
foundAll = false;
break;
}
}
}
if ( ! foundAll ) {
ostringstream ss;
ss << "in seed list " << servers.toString() << ", host " << offendingHost
<< " does not belong to replica set " << setName;
errMsg = ss.str();
newShardConn.done();
return false;
}
// shard name defaults to the name of the replica set
if ( name->empty() && ! setName.empty() )
*name = setName;
// In order to be accepted as a new shard, that mongod must not have any database name that exists already
// in any other shards. If that test passes, the new shard's databases are going to be entered as
// non-sharded db's whose primary is the newly added shard.
BSONObj resListDB;
ok = newShardConn->runCommand( "admin" , BSON( "listDatabases" << 1 ) , resListDB );
if ( !ok ) {
ostringstream ss;
ss << "failed listing " << servers.toString() << "'s databases:" << resListDB;
errMsg = ss.str();
newShardConn.done();
return false;
}
BSONObjIterator i( resListDB["databases"].Obj() );
while ( i.more() ) {
BSONObj dbEntry = i.next().Obj();
const string& dbName = dbEntry["name"].String();
if ( _isSpecialLocalDB( dbName ) ) {
// 'local', 'admin', and 'config' are system DBs and should be excluded here
continue;
}
else {
dbNames.push_back( dbName );
}
}
if ( newShardConn->type() == ConnectionString::SET )
rsMonitor = ReplicaSetMonitor::get( setName );
newShardConn.done();
}
catch ( DBException& e ) {
if ( servers.type() == ConnectionString::SET ) {
ReplicaSetMonitor::remove( servers.getSetName() );
}
ostringstream ss;
ss << "couldn't connect to new shard ";
ss << e.what();
errMsg = ss.str();
return false;
}
// check that none of the existing shard candidate's db's exist elsewhere
for ( vector<string>::const_iterator it = dbNames.begin(); it != dbNames.end(); ++it ) {
DBConfigPtr config = getDBConfig( *it , false );
if ( config.get() != NULL ) {
ostringstream ss;
ss << "can't add shard " << servers.toString() << " because a local database '" << *it;
ss << "' exists in another " << config->getPrimary().toString();
errMsg = ss.str();
return false;
}
}
// if a name for a shard wasn't provided, pick one.
if ( name->empty() && ! _getNewShardName( name ) ) {
errMsg = "error generating new shard name";
return false;
}
// build the ConfigDB shard document
BSONObjBuilder b;
b.append(ShardType::name(), *name);
b.append(ShardType::host(),
rsMonitor ? rsMonitor->getServerAddress() : servers.toString());
if (maxSize > 0) {
b.append(ShardType::maxSize(), maxSize);
}
BSONObj shardDoc = b.obj();
{
ScopedDbConnection conn(configServer.getPrimary().getConnString(), 30);
// check whether the set of hosts (or single host) is not an already a known shard
BSONObj old = conn->findOne(ShardType::ConfigNS,
BSON(ShardType::host(servers.toString())));
if ( ! old.isEmpty() ) {
errMsg = "host already used";
conn.done();
return false;
}
conn.done();
}
log() << "going to add shard: " << shardDoc << endl;
Status result = clusterInsert( ShardType::ConfigNS,
shardDoc,
WriteConcernOptions::AllConfigs,
NULL );
if ( !result.isOK() ) {
errMsg = result.reason();
log() << "error adding shard: " << shardDoc << " err: " << errMsg << endl;
return false;
}
Shard::reloadShardInfo();
// add all databases of the new shard
for ( vector<string>::const_iterator it = dbNames.begin(); it != dbNames.end(); ++it ) {
DBConfigPtr config = getDBConfig( *it , true , *name );
if ( ! config ) {
log() << "adding shard " << servers << " even though could not add database " << *it << endl;
}
}
// Record in changelog
BSONObjBuilder shardDetails;
shardDetails.append("name", *name);
shardDetails.append("host", servers.toString());
configServer.logChange("addShard", "", shardDetails.obj());
return true;
}
/**
* Updates the remote cached version on the remote shard host (primary, in the case of replica
* sets) if needed with a fully-qualified shard version for the given namespace:
* config server(s) + shard name + shard version
*
* If no remote cached version has ever been set, an initial shard version is sent.
*
* If the namespace is empty and no version has ever been sent, the config server + shard name
* is sent to the remote shard host to initialize the connection as coming from mongos.
* NOTE: This initialization is *best-effort only*. Operations which wish to correctly version
* must send the namespace.
*
* Config servers are special and are not (unless otherwise a shard) kept up to date with this
* protocol. This is safe so long as config servers only contain unversioned collections.
*
* It is an error to call checkShardVersion with an unversionable connection (isVersionableCB).
*
* @return true if we contacted the remote host
*/
bool checkShardVersion( DBClientBase * conn_in , const string& ns , ChunkManagerPtr refManager, bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
// Empty namespaces are special - we require initialization but not versioning
if (ns.size() == 0) {
return initShardVersionEmptyNS(conn_in);
}
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = getVersionable( conn_in );
verify(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ShardPtr primary;
ChunkManagerPtr manager;
if (authoritative)
conf->getChunkManagerIfExists(ns, true);
conf->getChunkManagerOrPrimary(ns, manager, primary);
if (manager)
officialSequenceNumber = manager->getSequenceNumber();
// Check this manager against the reference manager
if( manager ){
Shard shard = Shard::make( conn->getServerAddress() );
if (refManager && !refManager->compatibleWith(*manager, shard.getName())) {
const ChunkVersion refVersion(refManager->getVersion(shard.getName()));
const ChunkVersion currentVersion(manager->getVersion(shard.getName()));
string msg(str::stream() << "manager ("
<< currentVersion.toString()
<< " : " << manager->getSequenceNumber() << ") "
<< "not compatible with reference manager ("
<< refVersion.toString()
<< " : " << refManager->getSequenceNumber() << ") "
<< "on shard " << shard.getName()
<< " (" << shard.getAddress().toString() << ")");
throw SendStaleConfigException(ns,
msg,
refVersion,
currentVersion);
}
}
else if( refManager ){
Shard shard = Shard::make(conn->getServerAddress());
string msg( str::stream() << "not sharded ("
<< ( (manager.get() == 0) ? string( "<none>" ) :
str::stream() << manager->getSequenceNumber() )
<< ") but has reference manager ("
<< refManager->getSequenceNumber() << ") "
<< "on conn " << conn->getServerAddress() << " ("
<< conn_in->getServerAddress() << ")" );
throw SendStaleConfigException(ns,
msg,
refManager->getVersion(shard.getName()),
ChunkVersion::UNSHARDED());
}
// Do not send setShardVersion to collections on the config servers - this causes problems
// when config servers are also shards and get SSV with conflicting names.
// TODO: Make config servers regular shards
if (primary && primary->getName() == "config") {
return false;
}
// Has the ChunkManager been reloaded since the last time we updated the shard version over
// this connection? If we've never updated the shard version, do so now.
unsigned long long sequenceNumber = 0;
if (connectionShardStatus.getSequence(conn, ns, &sequenceNumber)) {
if (sequenceNumber == officialSequenceNumber) {
return false;
}
}
// Now that we're sure we're sending SSV and not to a single config server, get the shard
Shard shard = Shard::make(conn->getServerAddress());
ChunkVersion version = ChunkVersion(0, 0, OID());
if (manager)
version = manager->getVersion(shard.getName());
LOG(1) << "setting shard version of " << version << " for " << ns << " on shard "
<< shard.toString();
LOG(3) << "last version sent with chunk manager iteration " << sequenceNumber
<< ", current chunk manager iteration is " << officialSequenceNumber;
BSONObj result;
if (setShardVersion(*conn,
ns,
configServer.modelServer(),
version,
manager.get(),
authoritative,
result)) {
LOG(1) << " setShardVersion success: " << result;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
warning() << "reloading full configuration for " << conf->name()
<< ", connection state indicates significant version changes";
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion shard: " << shard.toString() << " " << result;
sleepmillis( 10 * tryNumber );
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
return true;
}
string errmsg = str::stream() << "setShardVersion failed shard: " << shard.toString()
<< " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase * conn_in , const string& ns , ChunkManagerPtr refManager, bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( *conn_in );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = getVersionable( conn_in );
verify(conn); // errors thrown above
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManagerIfExists( ns , authoritative );
// It's possible the chunk manager was reset since we checked whether sharded was true,
// so must check this here.
if( manager ) officialSequenceNumber = manager->getSequenceNumber();
}
// Check this manager against the reference manager
if( isSharded && manager ){
Shard shard = Shard::make( conn->getServerAddress() );
if( refManager && ! refManager->compatibleWith( manager, shard ) ){
throw SendStaleConfigException( ns, str::stream() << "manager (" << manager->getVersion( shard ).toString() << " : " << manager->getSequenceNumber() << ") "
<< "not compatible with reference manager (" << refManager->getVersion( shard ).toString() << " : " << refManager->getSequenceNumber() << ") "
<< "on shard " << shard.getName() << " (" << shard.getAddress().toString() << ")",
refManager->getVersion( shard ), manager->getVersion( shard ) );
}
}
else if( refManager ){
Shard shard = Shard::make( conn->getServerAddress() );
string msg( str::stream() << "not sharded ("
<< ( (manager.get() == 0) ? string( "<none>" ) :
str::stream() << manager->getSequenceNumber() )
<< ") but has reference manager ("
<< refManager->getSequenceNumber() << ") "
<< "on conn " << conn->getServerAddress() << " ("
<< conn_in->getServerAddress() << ")" );
throw SendStaleConfigException( ns, msg,
refManager->getVersion( shard ), ShardChunkVersion( 0, OID() ));
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie., last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = ShardChunkVersion( 0, OID() );
if ( isSharded && manager ) {
version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
}
if( ! version.isSet() ){
LOG(0) << "resetting shard version of " << ns << " on " << conn->getServerAddress() << ", " <<
( ! isSharded ? "no longer sharded" :
( ! manager ? "no chunk manager found" :
"version is zero" ) ) << endl;
}
LOG(2) << " have to set shard version for conn: " << conn->getServerAddress() << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
const string versionableServerAddress(conn->getServerAddress());
BSONObj result;
if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1);
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
warning() << "reloading full configuration for " << conf->getName()
<< ", connection state indicates significant version changes" << endl;
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion host: " << versionableServerAddress << " " << result << endl;
sleepmillis( 10 * tryNumber );
// use the original connection and get a fresh versionable connection
// since conn can be invalidated (or worse, freed) after the failure
checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1);
return true;
}
string errmsg = str::stream() << "setShardVersion failed host: " << versionableServerAddress << " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
int Balancer::_moveChunks(const vector<CandidateChunkPtr>* candidateChunks,
bool secondaryThrottle,
bool waitForDelete)
{
int movedCount = 0;
for ( vector<CandidateChunkPtr>::const_iterator it = candidateChunks->begin(); it != candidateChunks->end(); ++it ) {
const CandidateChunk& chunkInfo = *it->get();
// Changes to metadata, borked metadata, and connectivity problems should cause us to
// abort this chunk move, but shouldn't cause us to abort the entire round of chunks.
// TODO: Handle all these things more cleanly, since they're expected problems
try {
DBConfigPtr cfg = grid.getDBConfig( chunkInfo.ns );
verify( cfg );
// NOTE: We purposely do not reload metadata here, since _doBalanceRound already
// tried to do so once.
ChunkManagerPtr cm = cfg->getChunkManager( chunkInfo.ns );
verify( cm );
ChunkPtr c = cm->findIntersectingChunk( chunkInfo.chunk.min );
if ( c->getMin().woCompare( chunkInfo.chunk.min ) || c->getMax().woCompare( chunkInfo.chunk.max ) ) {
// likely a split happened somewhere
cm = cfg->getChunkManager( chunkInfo.ns , true /* reload */);
verify( cm );
c = cm->findIntersectingChunk( chunkInfo.chunk.min );
if ( c->getMin().woCompare( chunkInfo.chunk.min ) || c->getMax().woCompare( chunkInfo.chunk.max ) ) {
log() << "chunk mismatch after reload, ignoring will retry issue " << chunkInfo.chunk.toString() << endl;
continue;
}
}
BSONObj res;
if (c->moveAndCommit(Shard::make(chunkInfo.to),
Chunk::MaxChunkSize,
secondaryThrottle,
waitForDelete,
0, /* maxTimeMS */
res)) {
movedCount++;
continue;
}
// the move requires acquiring the collection metadata's lock, which can fail
log() << "balancer move failed: " << res << " from: " << chunkInfo.from << " to: " << chunkInfo.to
<< " chunk: " << chunkInfo.chunk << endl;
if ( res["chunkTooBig"].trueValue() ) {
// reload just to be safe
cm = cfg->getChunkManager( chunkInfo.ns );
verify( cm );
c = cm->findIntersectingChunk( chunkInfo.chunk.min );
log() << "forcing a split because migrate failed for size reasons" << endl;
res = BSONObj();
c->singleSplit( true , res );
log() << "forced split results: " << res << endl;
if ( ! res["ok"].trueValue() ) {
log() << "marking chunk as jumbo: " << c->toString() << endl;
c->markAsJumbo();
// we increment moveCount so we do another round right away
movedCount++;
}
}
}
catch( const DBException& ex ) {
warning() << "could not move chunk " << chunkInfo.chunk.toString()
<< ", continuing balancing round" << causedBy( ex ) << endl;
}
}
return movedCount;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase& conn , const string& ns , bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( conn );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManager( ns , authoritative );
officialSequenceNumber = manager->getSequenceNumber();
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie, last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(&conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = 0;
if ( isSharded ) {
version = manager->getVersion( Shard::make( conn.getServerAddress() ) );
}
log(2) << " have to set shard version for conn: " << &conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( &conn , ns , officialSequenceNumber );
return true;
}
log(1) << " setShardVersion failed!\n" << result << endl;
if ( result.getBoolField( "need_authoritative" ) )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
checkShardVersion( conn , ns , 1 , tryNumber + 1 );
return true;
}
if ( tryNumber < 4 ) {
log(1) << "going to retry checkShardVersion" << endl;
sleepmillis( 10 );
checkShardVersion( conn , ns , 1 , tryNumber + 1 );
return true;
}
log() << " setShardVersion failed: " << result << endl;
massert( 10429 , (string)"setShardVersion failed! " + result.jsonString() , 0 );
return true;
}
/**
* @return true if had to do something
*/
bool checkShardVersion( DBClientBase& conn_in , const string& ns , bool authoritative , int tryNumber ) {
// TODO: cache, optimize, etc...
WriteBackListener::init( conn_in );
DBConfigPtr conf = grid.getDBConfig( ns );
if ( ! conf )
return false;
DBClientBase* conn = 0;
switch ( conn_in.type() ) {
case ConnectionString::INVALID:
assert(0);
break;
case ConnectionString::MASTER:
// great
conn = &conn_in;
break;
case ConnectionString::PAIR:
assert( ! "pair not support for sharding" );
break;
case ConnectionString::SYNC:
// TODO: we should check later that we aren't actually sharded on this
conn = &conn_in;
break;
case ConnectionString::SET:
DBClientReplicaSet* set = (DBClientReplicaSet*)&conn_in;
conn = &(set->masterConn());
break;
}
assert(conn);
unsigned long long officialSequenceNumber = 0;
ChunkManagerPtr manager;
const bool isSharded = conf->isSharded( ns );
if ( isSharded ) {
manager = conf->getChunkManagerIfExists( ns , authoritative );
// It's possible the chunk manager was reset since we checked whether sharded was true,
// so must check this here.
if( manager ) officialSequenceNumber = manager->getSequenceNumber();
}
// has the ChunkManager been reloaded since the last time we updated the connection-level version?
// (ie., last time we issued the setShardVersions below)
unsigned long long sequenceNumber = connectionShardStatus.getSequence(conn,ns);
if ( sequenceNumber == officialSequenceNumber ) {
return false;
}
ShardChunkVersion version = 0;
if ( isSharded && manager ) {
version = manager->getVersion( Shard::make( conn->getServerAddress() ) );
}
LOG(2) << " have to set shard version for conn: " << conn << " ns:" << ns
<< " my last seq: " << sequenceNumber << " current: " << officialSequenceNumber
<< " version: " << version << " manager: " << manager.get()
<< endl;
BSONObj result;
if ( setShardVersion( *conn , ns , version , authoritative , result ) ) {
// success!
LOG(1) << " setShardVersion success: " << result << endl;
connectionShardStatus.setSequence( conn , ns , officialSequenceNumber );
return true;
}
LOG(1) << " setShardVersion failed!\n" << result << endl;
if ( result["need_authoritative"].trueValue() )
massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative );
if ( ! authoritative ) {
checkShardVersion( *conn , ns , 1 , tryNumber + 1 );
return true;
}
if ( result["reloadConfig"].trueValue() ) {
if( result["version"].timestampTime() == 0 ){
// reload db
conf->reload();
}
else {
// reload config
conf->getChunkManager( ns , true );
}
}
const int maxNumTries = 7;
if ( tryNumber < maxNumTries ) {
LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 )
<< "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl;
sleepmillis( 10 * tryNumber );
checkShardVersion( *conn , ns , true , tryNumber + 1 );
return true;
}
string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result;
log() << " " << errmsg << endl;
massert( 10429 , errmsg , 0 );
return true;
}
