void Balancer::run(){
{ // init stuff, don't want to do at static init
StringBuilder buf;
buf << ourHostname << ":" << cmdLine.port;
_myid = buf.str();
log(1) << "balancer myid: " << _myid << endl;
_started = time(0);
Shard::reloadShardInfo();
}
_ping();
_checkOIDs();
while ( ! inShutdown() ){
sleepsecs( 10 );
try {
ScopedDbConnection conn( configServer.getPrimary() );
_ping( conn.conn() );
if ( ! _checkOIDs() ){
uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
}
vector<CandidateChunkPtr> candidateChunks;
if ( _shouldIBalance( conn.conn() ) ){
log(1) << "balancer: start balancing round" << endl;
candidateChunks.clear();
_doBalanceRound( conn.conn() , &candidateChunks );
if ( candidateChunks.size() == 0 ) {
log(1) << "balancer: no need to move any chunk" << endl;
} else {
_balancedLastTime = _moveChunks( &candidateChunks );
log(1) << "balancer: end balancing round" << endl;
}
}
conn.done();
}
catch ( std::exception& e ){
log() << "caught exception while doing balance: " << e.what() << endl;
// It's possible this shard was removed
Shard::reloadShardInfo();
continue;
}
}
}
bool Balancer::_init(OperationContext* txn) {
log() << "about to contact config servers and shards";
try {
// Contact the config server and refresh shard information. Checks that each shard is indeed
// a different process (no hostname mixup)
// these checks are redundant in that they're redone at every new round but we want to do
// them initially here so to catch any problem soon
grid.shardRegistry()->reload(txn);
if (!_checkOIDs(txn)) {
return false;
}
log() << "config servers and shards contacted successfully";
StringBuilder buf;
buf << getHostNameCached() << ":" << serverGlobalParams.port;
_myid = buf.str();
log() << "balancer id: " << _myid << " started";
return true;
} catch (const std::exception& e) {
warning() << "could not initialize balancer, please check that all shards and config "
"servers are up: " << e.what();
return false;
}
}
bool Balancer::_init() {
try {
log() << "about to contact config servers and shards" << endl;
// contact the config server and refresh shard information
// checks that each shard is indeed a different process (no hostname mixup)
// these checks are redundant in that they're redone at every new round but we want to do them initially here
// so to catch any problem soon
Shard::reloadShardInfo();
_checkOIDs();
log() << "config servers and shards contacted successfully" << endl;
StringBuilder buf;
buf << getHostNameCached() << ":" << cmdLine.port;
_myid = buf.str();
_started = time(0);
log() << "balancer id: " << _myid << " started at " << time_t_to_String_short(_started) << endl;
return true;
}
catch ( std::exception& e ) {
warning() << "could not initialize balancer, please check that all shards and config servers are up: " << e.what() << endl;
return false;
}
}
void Balancer::run() {
Client::initThread("Balancer");
// This is the body of a BackgroundJob so if we throw here we're basically ending the balancer
// thread prematurely.
while (!inShutdown()) {
auto txn = cc().makeOperationContext();
if (!_init(txn.get())) {
log() << "will retry to initialize balancer in one minute";
sleepsecs(60);
continue;
}
break;
}
Seconds balanceRoundInterval(kBalanceRoundDefaultInterval);
while (!inShutdown()) {
auto txn = cc().makeOperationContext();
BalanceRoundDetails roundDetails;
try {
// ping has to be first so we keep things in the config server in sync
_ping(txn.get(), false);
MONGO_FAIL_POINT_BLOCK(balancerRoundIntervalSetting, scopedBalancerRoundInterval) {
const BSONObj& data = scopedBalancerRoundInterval.getData();
balanceRoundInterval = Seconds(data["sleepSecs"].numberInt());
}
// Use fresh shard state and balancer settings
Grid::get(txn.get())->shardRegistry()->reload(txn.get());
auto balancerConfig = Grid::get(txn.get())->getBalancerConfiguration();
Status refreshStatus = balancerConfig->refreshAndCheck(txn.get());
if (!refreshStatus.isOK()) {
warning() << "Skipping balancing round" << causedBy(refreshStatus);
sleepFor(balanceRoundInterval);
continue;
}
// now make sure we should even be running
if (!balancerConfig->isBalancerActive() || MONGO_FAIL_POINT(skipBalanceRound)) {
LOG(1) << "skipping balancing round because balancing is disabled";
// Ping again so scripts can determine if we're active without waiting
_ping(txn.get(), true);
sleepFor(balanceRoundInterval);
continue;
}
uassert(13258, "oids broken after resetting!", _checkOIDs(txn.get()));
{
auto scopedDistLock = grid.catalogManager(txn.get())
->distLock(txn.get(),
"balancer",
"doing balance round",
DistLockManager::kSingleLockAttemptTimeout);
if (!scopedDistLock.isOK()) {
LOG(1) << "skipping balancing round" << causedBy(scopedDistLock.getStatus());
// Ping again so scripts can determine if we're active without waiting
_ping(txn.get(), true);
sleepFor(balanceRoundInterval); // no need to wake up soon
continue;
}
LOG(1) << "*** start balancing round. "
<< "waitForDelete: " << balancerConfig->waitForDelete()
<< ", secondaryThrottle: "
<< balancerConfig->getSecondaryThrottle().toBSON();
OCCASIONALLY warnOnMultiVersion(
uassertStatusOK(_clusterStats->getStats(txn.get())));
Status status = _enforceTagRanges(txn.get());
if (!status.isOK()) {
warning() << "Failed to enforce tag ranges" << causedBy(status);
} else {
LOG(1) << "Done enforcing tag range boundaries.";
}
const auto candidateChunks = uassertStatusOK(
_chunkSelectionPolicy->selectChunksToMove(txn.get(), _balancedLastTime));
if (candidateChunks.empty()) {
LOG(1) << "no need to move any chunk";
_balancedLastTime = 0;
} else {
_balancedLastTime = _moveChunks(txn.get(),
candidateChunks,
balancerConfig->getSecondaryThrottle(),
balancerConfig->waitForDelete());
roundDetails.setSucceeded(static_cast<int>(candidateChunks.size()),
_balancedLastTime);
grid.catalogManager(txn.get())
->logAction(txn.get(), "balancer.round", "", roundDetails.toBSON());
}
LOG(1) << "*** End of balancing round";
}
// Ping again so scripts can determine if we're active without waiting
_ping(txn.get(), true);
sleepFor(_balancedLastTime ? kShortBalanceRoundInterval : balanceRoundInterval);
} catch (const std::exception& e) {
log() << "caught exception while doing balance: " << e.what();
// Just to match the opening statement if in log level 1
LOG(1) << "*** End of balancing round";
// This round failed, tell the world!
roundDetails.setFailed(e.what());
grid.catalogManager(txn.get())
->logAction(txn.get(), "balancer.round", "", roundDetails.toBSON());
// Sleep a fair amount before retrying because of the error
sleepFor(balanceRoundInterval);
}
}
}
void Balancer::run() {
// this is the body of a BackgroundJob so if we throw here we're basically ending the balancer thread prematurely
while ( ! inShutdown() ) {
if ( ! _init() ) {
log() << "will retry to initialize balancer in one minute" << endl;
sleepsecs( 60 );
continue;
}
break;
}
int sleepTime = 30;
// getConnectioString and dist lock constructor does not throw, which is what we expect on while
// on the balancer thread
ConnectionString config = configServer.getConnectionString();
DistributedLock balanceLock( config , "balancer" );
while ( ! inShutdown() ) {
try {
scoped_ptr<ScopedDbConnection> connPtr(
ScopedDbConnection::getInternalScopedDbConnection( config.toString() ) );
ScopedDbConnection& conn = *connPtr;
// ping has to be first so we keep things in the config server in sync
_ping( conn.conn() );
// use fresh shard state
Shard::reloadShardInfo();
// refresh chunk size (even though another balancer might be active)
Chunk::refreshChunkSize();
BSONObj balancerConfig;
// now make sure we should even be running
if ( ! grid.shouldBalance( "", &balancerConfig ) ) {
LOG(1) << "skipping balancing round because balancing is disabled" << endl;
// Ping again so scripts can determine if we're active without waiting
_ping( conn.conn(), true );
conn.done();
sleepsecs( sleepTime );
continue;
}
sleepTime = balancerConfig["_nosleep"].trueValue() ? 30 : 6;
uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
{
dist_lock_try lk( &balanceLock , "doing balance round" );
if ( ! lk.got() ) {
LOG(1) << "skipping balancing round because another balancer is active" << endl;
// Ping again so scripts can determine if we're active without waiting
_ping( conn.conn(), true );
conn.done();
sleepsecs( sleepTime ); // no need to wake up soon
continue;
}
LOG(1) << "*** start balancing round" << endl;
vector<CandidateChunkPtr> candidateChunks;
_doBalanceRound( conn.conn() , &candidateChunks );
if ( candidateChunks.size() == 0 ) {
LOG(1) << "no need to move any chunk" << endl;
_balancedLastTime = 0;
}
else {
_balancedLastTime = _moveChunks( &candidateChunks, balancerConfig["_secondaryThrottle"].trueValue() );
}
LOG(1) << "*** end of balancing round" << endl;
}
// Ping again so scripts can determine if we're active without waiting
_ping( conn.conn(), true );
conn.done();
sleepsecs( _balancedLastTime ? sleepTime / 6 : sleepTime );
}
catch ( std::exception& e ) {
log() << "caught exception while doing balance: " << e.what() << endl;
// Just to match the opening statement if in log level 1
LOG(1) << "*** End of balancing round" << endl;
sleepsecs( sleepTime ); // sleep a fair amount b/c of error
continue;
}
}
}
void Balancer::run() {
// this is the body of a BackgroundJob so if we throw here we're basically ending the balancer thread prematurely
while ( ! inShutdown() ) {
if ( ! _init() ) {
log() << "will retry to initialize balancer in one minute" << endl;
sleepsecs( 60 );
continue;
}
break;
}
int sleepTime = 10;
// getConnectioString and dist lock constructor does not throw, which is what we expect on while
// on the balancer thread
ConnectionString config = configServer.getConnectionString();
DistributedLock balanceLock( config , "balancer" );
while ( ! inShutdown() ) {
try {
ScopedDbConnection conn(config.toString(), 30);
// ping has to be first so we keep things in the config server in sync
_ping();
// use fresh shard state
Shard::reloadShardInfo();
// refresh chunk size (even though another balancer might be active)
Chunk::refreshChunkSize();
SettingsType balancerConfig;
string errMsg;
if (!grid.getBalancerSettings(&balancerConfig, &errMsg)) {
warning() << errMsg;
return ;
}
// now make sure we should even be running
if ((balancerConfig.isKeySet() && // balancer config doc exists
!grid.shouldBalance(balancerConfig)) ||
MONGO_FAIL_POINT(skipBalanceRound)) {
LOG(1) << "skipping balancing round because balancing is disabled" << endl;
// Ping again so scripts can determine if we're active without waiting
_ping( true );
conn.done();
sleepsecs( sleepTime );
continue;
}
uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
{
dist_lock_try lk( &balanceLock , "doing balance round" );
if ( ! lk.got() ) {
LOG(1) << "skipping balancing round because another balancer is active" << endl;
// Ping again so scripts can determine if we're active without waiting
_ping( true );
conn.done();
sleepsecs( sleepTime ); // no need to wake up soon
continue;
}
if ( !isConfigServerConsistent() ) {
conn.done();
warning() << "Skipping balancing round because data inconsistency"
<< " was detected amongst the config servers." << endl;
sleepsecs( sleepTime );
continue;
}
const bool waitForDelete = (balancerConfig.isWaitForDeleteSet() ?
balancerConfig.getWaitForDelete() : false);
scoped_ptr<WriteConcernOptions> writeConcern;
if (balancerConfig.isKeySet()) { // if balancer doc exists.
StatusWith<WriteConcernOptions*> extractStatus =
balancerConfig.extractWriteConcern();
if (extractStatus.isOK()) {
writeConcern.reset(extractStatus.getValue());
}
else {
warning() << extractStatus.toString();
}
}
LOG(1) << "*** start balancing round. "
<< "waitForDelete: " << waitForDelete
<< ", secondaryThrottle: "
<< (writeConcern.get() ? writeConcern->toBSON().toString() : "default")
<< endl;
vector<CandidateChunkPtr> candidateChunks;
_doBalanceRound( conn.conn() , &candidateChunks );
if ( candidateChunks.size() == 0 ) {
LOG(1) << "no need to move any chunk" << endl;
_balancedLastTime = 0;
}
else {
_balancedLastTime = _moveChunks(&candidateChunks,
writeConcern.get(),
waitForDelete );
}
LOG(1) << "*** end of balancing round" << endl;
}
// Ping again so scripts can determine if we're active without waiting
_ping( true );
conn.done();
sleepsecs( _balancedLastTime ? sleepTime / 10 : sleepTime );
}
catch ( std::exception& e ) {
log() << "caught exception while doing balance: " << e.what() << endl;
// Just to match the opening statement if in log level 1
LOG(1) << "*** End of balancing round" << endl;
sleepsecs( sleepTime ); // sleep a fair amount b/c of error
continue;
}
}
}
void Balancer::run() {
Client::initThread("Balancer");
// This is the body of a BackgroundJob so if we throw here we're basically ending the balancer
// thread prematurely.
while (!inShutdown()) {
auto txn = cc().makeOperationContext();
if (!_init(txn.get())) {
log() << "will retry to initialize balancer in one minute";
sleepsecs(60);
continue;
}
break;
}
Seconds balanceRoundInterval(kBalanceRoundDefaultInterval);
while (!inShutdown()) {
auto txn = cc().makeOperationContext();
BalanceRoundDetails roundDetails;
try {
// ping has to be first so we keep things in the config server in sync
_ping(txn.get());
MONGO_FAIL_POINT_BLOCK(balancerRoundIntervalSetting, scopedBalancerRoundInterval) {
const BSONObj& data = scopedBalancerRoundInterval.getData();
balanceRoundInterval = Seconds(data["sleepSecs"].numberInt());
}
BSONObj balancerResult;
// use fresh shard state
grid.shardRegistry()->reload(txn.get());
// refresh chunk size (even though another balancer might be active)
Chunk::refreshChunkSize(txn.get());
auto balSettingsResult = grid.catalogManager(txn.get())->getGlobalSettings(
txn.get(), SettingsType::BalancerDocKey);
const bool isBalSettingsAbsent =
balSettingsResult.getStatus() == ErrorCodes::NoMatchingDocument;
if (!balSettingsResult.isOK() && !isBalSettingsAbsent) {
warning() << balSettingsResult.getStatus();
return;
}
const SettingsType& balancerConfig =
isBalSettingsAbsent ? SettingsType{} : balSettingsResult.getValue();
// now make sure we should even be running
if ((!isBalSettingsAbsent && !Chunk::shouldBalance(balancerConfig)) ||
MONGO_FAIL_POINT(skipBalanceRound)) {
LOG(1) << "skipping balancing round because balancing is disabled";
// Ping again so scripts can determine if we're active without waiting
_ping(txn.get(), true);
sleepFor(balanceRoundInterval);
continue;
}
uassert(13258, "oids broken after resetting!", _checkOIDs(txn.get()));
{
auto scopedDistLock = grid.catalogManager(txn.get())
->distLock(txn.get(),
"balancer",
"doing balance round",
DistLockManager::kSingleLockAttemptTimeout);
if (!scopedDistLock.isOK()) {
LOG(1) << "skipping balancing round" << causedBy(scopedDistLock.getStatus());
// Ping again so scripts can determine if we're active without waiting
_ping(txn.get(), true);
sleepFor(balanceRoundInterval); // no need to wake up soon
continue;
}
const bool waitForDelete =
(balancerConfig.isWaitForDeleteSet() ? balancerConfig.getWaitForDelete()
: false);
MigrationSecondaryThrottleOptions secondaryThrottle(
MigrationSecondaryThrottleOptions::create(
MigrationSecondaryThrottleOptions::kDefault));
if (balancerConfig.isKeySet()) {
secondaryThrottle =
uassertStatusOK(MigrationSecondaryThrottleOptions::createFromBalancerConfig(
balancerConfig.toBSON()));
}
LOG(1) << "*** start balancing round. "
<< "waitForDelete: " << waitForDelete
<< ", secondaryThrottle: " << secondaryThrottle.toBSON();
const auto candidateChunks = uassertStatusOK(_getCandidateChunks(txn.get()));
if (candidateChunks.empty()) {
LOG(1) << "no need to move any chunk";
_balancedLastTime = 0;
} else {
_balancedLastTime =
_moveChunks(txn.get(), candidateChunks, secondaryThrottle, waitForDelete);
roundDetails.setSucceeded(static_cast<int>(candidateChunks.size()),
_balancedLastTime);
grid.catalogManager(txn.get())
->logAction(txn.get(), "balancer.round", "", roundDetails.toBSON());
}
LOG(1) << "*** End of balancing round";
}
// Ping again so scripts can determine if we're active without waiting
_ping(txn.get(), true);
sleepFor(_balancedLastTime ? kShortBalanceRoundInterval : balanceRoundInterval);
} catch (const std::exception& e) {
log() << "caught exception while doing balance: " << e.what();
// Just to match the opening statement if in log level 1
LOG(1) << "*** End of balancing round";
// This round failed, tell the world!
roundDetails.setFailed(e.what());
grid.catalogManager(txn.get())
->logAction(txn.get(), "balancer.round", "", roundDetails.toBSON());
// Sleep a fair amount before retrying because of the error
sleepFor(balanceRoundInterval);
}
}
}
void Balancer::run() {
// this is the body of a BackgroundJob so if we throw here we're basically ending the balancer thread prematurely
while ( ! inShutdown() ) {
if ( ! _init() ) {
log() << "will retry to initialize balancer in one minute" << endl;
sleepsecs( 60 );
continue;
}
break;
}
// getConnectioString and dist lock constructor does not throw, which is what we expect on while
// on the balancer thread
ConnectionString config = configServer.getConnectionString();
DistributedLock balanceLock( config , "balancer" );
while ( ! inShutdown() ) {
try {
ScopedDbConnection conn( config );
// ping has to be first so we keep things in the config server in sync
_ping( conn.conn() );
// now make sure we should even be running
if ( ! grid.shouldBalance() ) {
LOG(1) << "skipping balancing round because balancing is disabled" << endl;
conn.done();
sleepsecs( 30 );
continue;
}
uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
// use fresh shard state
Shard::reloadShardInfo();
{
dist_lock_try lk( &balanceLock , "doing balance round" );
if ( ! lk.got() ) {
LOG(1) << "skipping balancing round because another balancer is active" << endl;
conn.done();
sleepsecs( 30 ); // no need to wake up soon
continue;
}
LOG(1) << "*** start balancing round" << endl;
vector<CandidateChunkPtr> candidateChunks;
_doBalanceRound( conn.conn() , &candidateChunks );
if ( candidateChunks.size() == 0 ) {
LOG(1) << "no need to move any chunk" << endl;
}
else {
_balancedLastTime = _moveChunks( &candidateChunks );
}
LOG(1) << "*** end of balancing round" << endl;
}
conn.done();
sleepsecs( _balancedLastTime ? 5 : 10 );
}
catch ( std::exception& e ) {
log() << "caught exception while doing balance: " << e.what() << endl;
// Just to match the opening statement if in log level 1
LOG(1) << "*** End of balancing round" << endl;
sleepsecs( 30 ); // sleep a fair amount b/c of error
continue;
}
}
}
