void HbmLB::ResumeClients(int balancing)
{
#if CMK_LBDB_ON
DEBUGF(("[%d] ResumeClients. \n", CkMyPe()));
theLbdb->incStep();
// reset
LevelData *lData = levelData[0];
lData->clear();
if (CkMyPe() == 0 && balancing) {
double end_lb_time = CkWallTimer();
if (_lb_args.debug())
CkPrintf("[%s] Load balancing step %d finished at %f duration %f\n",
lbName(), step()-1,end_lb_time,end_lb_time - start_lb_time);
}
if (balancing && _lb_args.printSummary()) {
int count = 1;
LBInfo info(count);
LDStats *stats = &myStats;
info.getInfo(stats, count, 0); // no comm cost
LBRealType mLoad, mCpuLoad, totalLoad;
info.getSummary(mLoad, mCpuLoad, totalLoad);
int nmsgs, nbytes;
stats->computeNonlocalComm(nmsgs, nbytes);
CkPrintf("[%d] Load with %d objs: max (with comm): %f max (obj only): %f total: %f on %d processors at step %d useMem: %fKB nonlocal: %d %.2fKB.\n", CkMyPe(), stats->n_objs, mLoad, mCpuLoad, totalLoad, count, step()-1, (1.0*useMem())/1024, nmsgs, nbytes/1024.0);
thisProxy[0].reportLBQulity(mLoad, mCpuLoad, totalLoad, nmsgs, 1.0*nbytes/1024.0);
}
// zero out stats
theLbdb->ClearLoads();
theLbdb->ResumeClients();
#endif
}
void HybridBaseLB::ResumeClients(int balancing)
{
#if CMK_LBDB_ON
DEBUGF(("[%d] ResumeClients. \n", CkMyPe()));
double end_lb_time = CkWallTimer();
if (CkMyPe() == 0 && balancing) {
if (_lb_args.debug())
CkPrintf("[%s] Load balancing step %d finished at %f duration %f\n",
lbName(), step()-1,end_lb_time,end_lb_time - start_lb_time);
}
// zero out stats
theLbdb->ClearLoads();
theLbdb->ResumeClients();
theLbdb->SetMigrationCost(end_lb_time - start_lb_time);
#endif
}
void HbmLB::ProcessAtSync()
{
#if CMK_LBDB_ON
int i;
start_lb_time = 0;
if (CkMyPe() == 0) {
start_lb_time = CkWallTimer();
if (_lb_args.debug())
CkPrintf("[%s] Load balancing step %d starting at %f\n",
lbName(), step(), CkWallTimer());
}
// build LDStats
LBRealType total_walltime, total_cputime, idletime, bg_walltime, bg_cputime;
theLbdb->TotalTime(&total_walltime,&total_cputime);
theLbdb->IdleTime(&idletime);
theLbdb->BackgroundLoad(&bg_walltime,&bg_cputime);
myStats.n_objs = theLbdb->GetObjDataSz();
myStats.objData.resize(myStats.n_objs);
myStats.from_proc.resize(myStats.n_objs);
myStats.to_proc.resize(myStats.n_objs);
theLbdb->GetObjData(myStats.objData.getVec());
for (i=0; i<myStats.n_objs; i++)
myStats.from_proc[i] = myStats.to_proc[i] = 0; // only one PE
myStats.n_comm = theLbdb->GetCommDataSz();
myStats.commData.resize(myStats.n_comm);
theLbdb->GetCommData(myStats.commData.getVec());
myStats.complete_flag = 0;
// send to parent
DEBUGF(("[%d] Send stats to parent %d\n", CkMyPe(), levelData[0]->parent));
double tload = 0.0;
for (i=0; i<myStats.n_objs; i++) tload += myStats.objData[i].wallTime;
thisProxy[levelData[0]->parent].ReceiveStats(tload, CkMyPe(), 0);
#endif
}
void HybridBaseLB::ProcessAtSync()
{
#if CMK_LBDB_ON
start_lb_time = 0;
if (CkMyPe() == 0) {
start_lb_time = CkWallTimer();
if (_lb_args.debug())
CkPrintf("[%s] Load balancing step %d starting at %f\n",
lbName(), step(), CkWallTimer());
}
// assemble LB database
CLBStatsMsg* msg = AssembleStats();
CkMarshalledCLBStatsMessage marshmsg(msg);
// send to parent
thisProxy[levelData[0]->parent].ReceiveStats(marshmsg, 0);
DEBUGF(("[%d] Send stats to myself\n", CkMyPe()));
#endif
}
// LDStats data sent to parent contains real PE
// LDStats in parent should contain relative PE
void HybridBaseLB::Loadbalancing(int atlevel)
{
int i;
CmiAssert(atlevel >= 1);
CmiAssert(tree->isroot(CkMyPe(), atlevel));
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
CmiAssert(statsData);
// at this time, all objects processor location is relative, and
// all incoming objects from outside group belongs to the fake root proc.
// clear background load if needed
if (_lb_args.ignoreBgLoad()) statsData->clearBgLoad();
currentLevel = atlevel;
int nclients = lData->nChildren;
DEBUGF(("[%d] Calling Strategy ... \n", CkMyPe()));
double start_lb_time, strat_end_time;
start_lb_time = CkWallTimer();
if ((statsStrategy == SHRINK || statsStrategy == SHRINK_NULL) && atlevel == tree->numLevels()-1) {
// no obj and comm data
LBVectorMigrateMsg* migrateMsg = VectorStrategy(statsData);
strat_end_time = CkWallTimer();
// send to children
thisProxy.ReceiveVectorMigration(migrateMsg, nclients, lData->children);
}
else {
LBMigrateMsg* migrateMsg = Strategy(statsData);
strat_end_time = CkWallTimer();
// send to children
//CmiPrintf("[%d] level: %d nclients:%d children: %d %d\n", CkMyPe(), atlevel, nclients, lData->children[0], lData->children[1]);
if (!group1_created)
thisProxy.ReceiveMigration(migrateMsg, nclients, lData->children);
else {
// send in multicast tree
thisProxy.ReceiveMigration(migrateMsg, group1);
//CkSendMsgBranchGroup(CkIndex_HybridBaseLB::ReceiveMigration(NULL), migrateMsg, thisgroup, group1);
}
// CkPrintf("[%d] ReceiveMigration takes %f \n", CkMyPe(), CkWallTimer()-strat_end_time);
}
if (_lb_args.debug()>0){
CkPrintf("[%d] Loadbalancing Level %d (%d children) started at %f, elapsed time %f\n", CkMyPe(), atlevel, lData->nChildren, start_lb_time, strat_end_time-start_lb_time);
if (atlevel == tree->numLevels()-1) {
CkPrintf("[%d] %s memUsage: %.2fKB\n", CkMyPe(), lbName(), (1.0*useMem())/1024);
}
}
// inform new objects that are from outside group
if (atlevel < tree->numLevels()-1) {
for (i=0; i<statsData->n_objs; i++) {
CmiAssert(statsData->from_proc[i] != -1); // ???
if (statsData->from_proc[i] == nclients) { // from outside
CmiAssert(statsData->to_proc[i] < nclients);
int tope = lData->children[statsData->to_proc[i]];
// comm data
CkVec<LDCommData> comms;
// collectCommData(i, comms, atlevel);
thisProxy[tope].ObjMigrated(statsData->objData[i], comms.getVec(), comms.size(), atlevel-1);
}
}
}
}
