void HbmLB::ResumeClients(int balancing)
{
#if CMK_LBDB_ON
DEBUGF(("[%d] ResumeClients. \n", CkMyPe()));
theLbdb->incStep();
// reset
LevelData *lData = levelData[0];
lData->clear();
if (CkMyPe() == 0 && balancing) {
double end_lb_time = CkWallTimer();
if (_lb_args.debug())
CkPrintf("[%s] Load balancing step %d finished at %f duration %f\n",
lbName(), step()-1,end_lb_time,end_lb_time - start_lb_time);
}
if (balancing && _lb_args.printSummary()) {
int count = 1;
LBInfo info(count);
LDStats *stats = &myStats;
info.getInfo(stats, count, 0); // no comm cost
LBRealType mLoad, mCpuLoad, totalLoad;
info.getSummary(mLoad, mCpuLoad, totalLoad);
int nmsgs, nbytes;
stats->computeNonlocalComm(nmsgs, nbytes);
CkPrintf("[%d] Load with %d objs: max (with comm): %f max (obj only): %f total: %f on %d processors at step %d useMem: %fKB nonlocal: %d %.2fKB.\n", CkMyPe(), stats->n_objs, mLoad, mCpuLoad, totalLoad, count, step()-1, (1.0*useMem())/1024, nmsgs, nbytes/1024.0);
thisProxy[0].reportLBQulity(mLoad, mCpuLoad, totalLoad, nmsgs, 1.0*nbytes/1024.0);
}
// zero out stats
theLbdb->ClearLoads();
theLbdb->ResumeClients();
#endif
}
// objects arrives with only objdata
void HybridBaseLB::ObjsMigrated(CkVec<LDObjData>& datas, int m,
LDCommData *cdata, int n, int atlevel)
{
int i;
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
if (statsData != NULL) {
CkVec<LDObjData> &oData = statsData->objData;
for (i=0; i<m; i++)
{
// copy into LDStats
LDObjData &data = datas[i];
oData.push_back(data);
statsData->n_objs++;
if (data.migratable) statsData->n_migrateobjs++;
// an incoming object to the root
// pretend this object belongs to it
statsData->from_proc.push_back(lData->nChildren);
statsData->to_proc.push_back(lData->nChildren);
}
// copy into comm data
if (n) {
CkVec<LDCommData> &cData = statsData->commData;
for (int i=0; i<n; i++)
cData.push_back(cdata[i]);
statsData->n_comm += n;
statsData->deleteCommHash();
}
}
else { // leaf node, from which proc is unknown at this time
for (i=0; i<m; i++) {
LDObjData &data = datas[i];
LDObjKey key;
key.omID() = data.omID();
key.objID() = data.objID();
newObjs.push_back(Location(key, -1));
}
}
lData->obj_completed+=m;
if (lData->migrationDone()) {
StatsDone(atlevel);
}
}
// assmebly all stats messages from children
void HybridBaseLB::buildStats(int atlevel)
{
#if CMK_LBDB_ON
// build LDStats
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
CLBStatsMsg **statsMsgsList = lData->statsMsgsList;
int stats_msg_count = lData->stats_msg_count;
// statsMsgsList
DEBUGF(("[%d] buildStats for %d nobj:%d\n", CkMyPe(), stats_msg_count, statsData->n_objs));
statsData->nprocs() = stats_msg_count;
statsData->objData.resize(statsData->n_objs);
statsData->commData.resize(statsData->n_comm);
statsData->from_proc.resize(statsData->n_objs);
statsData->to_proc.resize(statsData->n_objs);
int nobj = 0;
int nmigobj = 0;
int ncom = 0;
for (int n=0; n<stats_msg_count; n++) {
int i;
CLBStatsMsg *msg = statsMsgsList[n];
int pe = msg->from_pe;
for (i=0; i<msg->n_objs; i++) {
// need to map index to relative index
statsData->from_proc[nobj] = statsData->to_proc[nobj] = NeighborIndex(pe, atlevel);
statsData->objData[nobj] = msg->objData[i];
if (msg->objData[i].migratable) nmigobj++;
nobj++;
}
for (i=0; i<msg->n_comm; i++) {
statsData->commData[ncom] = msg->commData[i];
ncom++;
}
// free the message
delete msg;
statsMsgsList[n]=0;
}
if (_lb_args.debug()>1) {
CmiPrintf("[%d] n_obj:%d migratable:%d ncom:%d at level %d at %f.\n", CkMyPe(), nobj, nmigobj, ncom, atlevel, CkWallTimer());
}
CmiAssert(statsData->n_objs == nobj);
CmiAssert(statsData->n_comm == ncom);
statsData->n_migrateobjs = nmigobj;
#endif
}
// LDStats data sent to parent contains real PE
// LDStats in parent should contain relative PE
void HbmLB::Loadbalancing(int atlevel)
{
CmiAssert(atlevel >= 1);
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
CmiAssert(statsData);
// at this time, all objects processor location is relative, and
// all incoming objects from outside group belongs to the fake root proc.
// clear background load if needed
if (_lb_args.ignoreBgLoad()) statsData->clearBgLoad();
currentLevel = atlevel;
double start_lb_time(CkWallTimer());
double lload = lData->statsList[0];
double rload = lData->statsList[1];
double diff = myabs(lload-rload);
double maxl = mymax(lload, rload);
double avg = (lload+rload)/2.0;
CkPrintf("[%d] lload: %f rload: %f atlevel: %d\n", CkMyPe(), lload, rload, atlevel);
if (diff/avg > 0.02) {
// we need to perform load balancing
int numpes = (int)pow(2.0, atlevel);
double delta = myabs(lload-rload) / numpes;
int overloaded = lData->children[0];
if (lload < rload) {
overloaded = lData->children[1];
}
DEBUGF(("[%d] branch %d is overloaded by %f... \n", CkMyPe(), overloaded, delta));
thisProxy[overloaded].ReceiveMigrationDelta(delta, atlevel, atlevel);
}
else {
LoadbalancingDone(atlevel);
}
}
// migrate only object LDStat in group
// leaf nodes actually migrate objects
void HybridBaseLB::ReceiveMigration(LBMigrateMsg *msg)
{
#if CMK_LBDB_ON
#if CMK_MEM_CHECKPOINT
CkResetInLdb();
#endif
FindNeighbors();
int atlevel = msg->level - 1;
DEBUGF(("[%d] ReceiveMigration\n", CkMyPe()));
LevelData *lData = levelData[atlevel];
// only non NULL when level > 0
LDStats *statsData = lData->statsData;
// do LDStats migration
const int me = CkMyPe();
lData->migrates_expected = 0;
for(int i=0; i < msg->n_moves; i++) {
MigrateInfo& move = msg->moves[i];
// incoming
if (move.from_pe != me && move.to_pe == me) {
// I can not be the parent node
DEBUGF(("[%d] expecting LDStats object from %d\n",me,move.from_pe));
// will receive a ObjData message
lData->migrates_expected ++;
}
else if (move.from_pe == me) { // outgoing
if (statsData) { // this is inner node
// send objdata
int obj;
int found = 0;
for (obj = 0; obj<statsData->n_objs; obj++) {
if (move.obj.omID() == statsData->objData[obj].handle.omID() &&
move.obj.objID() == statsData->objData[obj].handle.objID())
{
DEBUGF(("[%d] level: %d sending objData %d to %d. \n", CkMyPe(), atlevel, obj, move.to_pe));
found = 1;
// TODO send comm data
CkVec<LDCommData> comms;
collectCommData(obj, comms, atlevel);
if (move.to_pe != -1) {
// this object migrates to another PE of the parent domain
thisProxy[move.to_pe].ObjMigrated(statsData->objData[obj], comms.getVec(), comms.size(), atlevel);
}
lData->outObjs.push_back(MigrationRecord(move.obj, lData->children[statsData->from_proc[obj]], -1));
statsData->removeObject(obj);
break;
}
}
CmiAssert(found == 1);
}
else { // this is leave node
if (move.to_pe == -1) {
lData->outObjs.push_back(MigrationRecord(move.obj, CkMyPe(), -1));
}
else {
// migrate the object
theLbdb->Migrate(move.obj,move.to_pe);
}
}
} // end if
}
if (lData->migrationDone())
StatsDone(atlevel);
#endif
}
// LDStats data sent to parent contains real PE
// LDStats in parent should contain relative PE
void HybridBaseLB::Loadbalancing(int atlevel)
{
int i;
CmiAssert(atlevel >= 1);
CmiAssert(tree->isroot(CkMyPe(), atlevel));
LevelData *lData = levelData[atlevel];
LDStats *statsData = lData->statsData;
CmiAssert(statsData);
// at this time, all objects processor location is relative, and
// all incoming objects from outside group belongs to the fake root proc.
// clear background load if needed
if (_lb_args.ignoreBgLoad()) statsData->clearBgLoad();
currentLevel = atlevel;
int nclients = lData->nChildren;
DEBUGF(("[%d] Calling Strategy ... \n", CkMyPe()));
double start_lb_time, strat_end_time;
start_lb_time = CkWallTimer();
if ((statsStrategy == SHRINK || statsStrategy == SHRINK_NULL) && atlevel == tree->numLevels()-1) {
// no obj and comm data
LBVectorMigrateMsg* migrateMsg = VectorStrategy(statsData);
strat_end_time = CkWallTimer();
// send to children
thisProxy.ReceiveVectorMigration(migrateMsg, nclients, lData->children);
}
else {
LBMigrateMsg* migrateMsg = Strategy(statsData);
strat_end_time = CkWallTimer();
// send to children
//CmiPrintf("[%d] level: %d nclients:%d children: %d %d\n", CkMyPe(), atlevel, nclients, lData->children[0], lData->children[1]);
if (!group1_created)
thisProxy.ReceiveMigration(migrateMsg, nclients, lData->children);
else {
// send in multicast tree
thisProxy.ReceiveMigration(migrateMsg, group1);
//CkSendMsgBranchGroup(CkIndex_HybridBaseLB::ReceiveMigration(NULL), migrateMsg, thisgroup, group1);
}
// CkPrintf("[%d] ReceiveMigration takes %f \n", CkMyPe(), CkWallTimer()-strat_end_time);
}
if (_lb_args.debug()>0){
CkPrintf("[%d] Loadbalancing Level %d (%d children) started at %f, elapsed time %f\n", CkMyPe(), atlevel, lData->nChildren, start_lb_time, strat_end_time-start_lb_time);
if (atlevel == tree->numLevels()-1) {
CkPrintf("[%d] %s memUsage: %.2fKB\n", CkMyPe(), lbName(), (1.0*useMem())/1024);
}
}
// inform new objects that are from outside group
if (atlevel < tree->numLevels()-1) {
for (i=0; i<statsData->n_objs; i++) {
CmiAssert(statsData->from_proc[i] != -1); // ???
if (statsData->from_proc[i] == nclients) { // from outside
CmiAssert(statsData->to_proc[i] < nclients);
int tope = lData->children[statsData->to_proc[i]];
// comm data
CkVec<LDCommData> comms;
// collectCommData(i, comms, atlevel);
thisProxy[tope].ObjMigrated(statsData->objData[i], comms.getVec(), comms.size(), atlevel-1);
}
}
}
}
// build a message based on our LDStats for sending to parent
// shrink if neccessary
CLBStatsMsg * HybridBaseLB::buildCombinedLBStatsMessage(int atlevel)
{
#if CMK_LBDB_ON
int i;
double obj_walltime, obj_nmwalltime;
#if CMK_LB_CPUTIMER
double obj_cputime, obj_nmcputime;
#endif
LDStats *statsData = levelData[atlevel]->statsData;
CmiAssert(statsData);
CLBStatsMsg* cmsg;
int osz = statsData->n_objs;
int csz = statsData->n_comm;
int shrink = 0;
if ((statsStrategy == SHRINK || statsStrategy == SHRINK_NULL) && atlevel == tree->numLevels()-2)
{
shrink = 1;
obj_walltime = obj_nmwalltime = 0.0;
#if CMK_LB_CPUTIMER
obj_cputime = obj_nmcputime = 0.0;
#endif
for (i=0; i<osz; i++) {
if (statsData->objData[i].migratable) {
obj_walltime += statsData->objData[i].wallTime;
#if CMK_LB_CPUTIMER
obj_cputime += statsData->objData[i].cpuTime;
#endif
}
else {
obj_nmwalltime += statsData->objData[i].wallTime;
#if CMK_LB_CPUTIMER
obj_nmcputime += statsData->objData[i].cpuTime;
#endif
}
}
// skip obj and comm data
osz = csz = 0;
}
cmsg = new CLBStatsMsg(osz, csz);
int mype = CkMyPe();
cmsg->from_pe = mype; // real PE
// Get stats
cmsg->pe_speed = 0;
cmsg->total_walltime = 0.0;
cmsg->idletime = 0.0;
cmsg->bg_walltime = 0.0;
#if CMK_LB_CPUTIMER
cmsg->total_cputime = 0.0;
cmsg->bg_cputime = 0.0;
#endif
for (int pe=0; pe<statsData->nprocs(); pe++) {
struct ProcStats &procStat = statsData->procs[pe];
cmsg->pe_speed += procStat.pe_speed; // important
cmsg->total_walltime += procStat.total_walltime;
cmsg->idletime += procStat.idletime;
cmsg->bg_walltime += procStat.bg_walltime;
#if CMK_LB_CPUTIMER
cmsg->total_cputime += procStat.total_cputime;
cmsg->bg_cputime += procStat.bg_cputime;
#endif
}
/*
cmsg->idletime = 0.0;
cmsg->bg_walltime = 0.0;
cmsg->bg_cputime = 0.0;
*/
// copy obj data
cmsg->n_objs = osz;
for (i=0; i<osz; i++) {
cmsg->objData[i] = statsData->objData[i];
}
// copy comm data
cmsg->n_comm = csz;
for (i=0; i<csz; i++) {
LDCommData &commData = statsData->commData[i];
cmsg->commData[i] = commData;
// modify processor to be this real pe
if (commData.from_proc()) cmsg->commData[i].src_proc = mype;
if (commData.receiver.get_type() == LD_PROC_MSG) cmsg->commData[i].receiver.setProc(mype);
}
if (shrink) {
cmsg->total_walltime = obj_walltime;
cmsg->bg_walltime += obj_nmwalltime;
#if CMK_LB_CPUTIMER
cmsg->total_cputime = obj_cputime;
cmsg->bg_cputime += obj_nmcputime;
#endif
}
return cmsg;
#else
return NULL;
#endif
}