void WSLB::AtSync()
{
#if CMK_LBDB_ON
// CkPrintf("[%d] WSLB At Sync step %d!!!!\n",CkMyPe(),mystep);
if (CkMyPe() == 0) {
start_lb_time = CkWallTimer();
CkPrintf("Load balancing step %d starting at %f\n",
step(),start_lb_time);
}
if (neighbor_pes == 0) FindNeighbors();
if (!QueryBalanceNow(step()) || mig_msgs_expected == 0) {
MigrationDone();
return;
}
WSLBStatsMsg* msg = AssembleStats();
thisProxy.ReceiveStats(msg,mig_msgs_expected,neighbor_pes);
// Tell our own node that we are ready
ReceiveStats((WSLBStatsMsg*)0);
#endif
}
void CentralLB::Migrated(LDObjHandle h, int waitBarrier)
{
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
if(donotCountMigration){
return ;
}
#endif
#if CMK_LBDB_ON
if (waitBarrier) {
migrates_completed++;
DEBUGF(("[%d] An object migrated! %d %d\n",CkMyPe(),migrates_completed,migrates_expected));
if (migrates_completed == migrates_expected) {
MigrationDone(1);
}
}
else {
future_migrates_completed ++;
DEBUGF(("[%d] An object migrated with no barrier! %d expected: %d\n",CkMyPe(),future_migrates_completed,future_migrates_expected));
if (future_migrates_completed == future_migrates_expected) {
CheckMigrationComplete();
}
}
#endif
}
// an object arrives with only objdata
void HbmLB::ObjMigrated(LDObjData data, LDCommData *cdata, int n)
{
LevelData *lData = levelData[0];
// change LDStats
CkVec<LDObjData> &oData = myStats.objData;
// need to update LDObjHandle later
lData->obj_completed++;
data.handle.handle = -100;
oData.push_back(data);
myStats.n_objs++;
if (data.migratable) myStats.n_migrateobjs++;
myStats.from_proc.push_back(-1); // not important
myStats.to_proc.push_back(0);
// copy into comm data
if (n) {
CkVec<LDCommData> &cData = myStats.commData;
for (int i=0; i<n; i++)
cData.push_back(cdata[i]);
myStats.n_comm += n;
myStats.deleteCommHash();
}
if (lData->migrationDone()) {
// migration done finally
MigrationDone(1);
}
}
void WSLB::Migrated(LDObjHandle h, int waitBarrier)
{
#if CMK_LBDB_ON
migrates_completed++;
// CkPrintf("[%d] An object migrated! %d %d\n",
// CkMyPe(),migrates_completed,migrates_expected);
if (migrates_completed == migrates_expected) {
MigrationDone();
}
#endif
}
void HbmLB::ReceiveMigrationCount(int count, int lblevel)
{
lbLevel = lblevel;
LevelData *lData = levelData[0];
lData->migrates_expected = count;
if (lData->migrationDone()) {
// migration done finally
MigrationDone(1);
}
}
void HbmLB::Migrated(LDObjHandle h, int waitBarrier)
{
LevelData *lData = levelData[0];
lData->migrates_completed++;
newObjs.push_back(h);
DEBUGF(("[%d] An object migrated! %d %d\n", CkMyPe(),lData->migrates_completed,lData->migrates_expected));
if (lData->migrationDone()) {
// migration done finally
MigrationDone(1);
}
}
void HybridBaseLB::Migrated(LDObjHandle h, int waitBarrier)
{
LevelData *lData = levelData[0];
lData->migrates_completed++;
DEBUGF(("[%d] An object migrated! %d %d\n", CkMyPe(),lData->migrates_completed,lData->migrates_expected));
if (lData->migrationDone()) {
if (!lData->resumeAfterMigration) {
StatsDone(0);
}
else {
// migration done finally
MigrationDone(1);
}
}
}
void HybridBaseLB::AtSync()
{
#if CMK_LBDB_ON
// CkPrintf("[%d] HybridBaseLB At Sync step %d!!!!\n",CkMyPe(),mystep);
FindNeighbors();
// if num of processor is only 1, nothing should happen
if (!QueryBalanceNow(step()) || CkNumPes() == 1) {
MigrationDone(0);
return;
}
thisProxy[CkMyPe()].ProcessAtSync();
#endif
}
void WSLB::ReceiveMigration(LBMigrateMsg *msg)
{
#if CMK_LBDB_ON
if (neighbor_pes == 0) FindNeighbors();
if (mig_msgs_received == 0) migrates_expected = 0;
mig_msgs[mig_msgs_received] = msg;
mig_msgs_received++;
// CkPrintf("[%d] Received migration msg %d of %d\n",
// CkMyPe(),mig_msgs_received,mig_msgs_expected);
if (mig_msgs_received > mig_msgs_expected) {
CkPrintf("[%d] WSLB Error! Too many migration messages received\n",
CkMyPe());
}
if (mig_msgs_received != mig_msgs_expected) {
return;
}
// CkPrintf("[%d] in ReceiveMigration %d moves\n",CkMyPe(),msg->n_moves);
for(int neigh=0; neigh < mig_msgs_received;neigh++) {
LBMigrateMsg* m = mig_msgs[neigh];
for(int i=0; i < m->n_moves; i++) {
MigrateInfo& move = m->moves[i];
const int me = CkMyPe();
if (move.from_pe != me && move.to_pe == me) {
migrates_expected++;
}
}
delete m;
mig_msgs[neigh]=0;
}
// CkPrintf("[%d] in ReceiveMigration %d expected\n",
// CkMyPe(),migrates_expected);
mig_msgs_received = 0;
if (migrates_expected == 0 || migrates_expected == migrates_completed)
MigrationDone();
#endif
}
void CentralLB::AtSync()
{
#if CMK_LBDB_ON
DEBUGF(("[%d] CentralLB AtSync step %d!!!!!\n",CkMyPe(),step()));
#if CMK_MEM_CHECKPOINT
CkSetInLdb();
#endif
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CpvAccess(_currentObj)=this;
#endif
// if num of processor is only 1, nothing should happen
if (!QueryBalanceNow(step()) || CkNumPes() == 1) {
MigrationDone(0);
return;
}
if(CmiNodeAlive(CkMyPe())){
thisProxy [CkMyPe()].ProcessAtSync();
}
#endif
}
void CentralLB::ProcessReceiveMigration(CkReductionMsg *msg)
{
#if CMK_LBDB_ON
int i;
LBMigrateMsg *m = storedMigrateMsg;
CmiAssert(m!=NULL);
delete msg;
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
int *dummyCounts;
DEBUGF(("[%d] Starting ReceiveMigration WITH step %d m->step %d\n",CkMyPe(),step(),m->step));
// CmiPrintf("[%d] Starting ReceiveMigration step %d m->step %d\n",CkMyPe(),step(),m->step);
if(step() > m->step){
char str[100];
envelope *env = UsrToEnv(m);
return;
}
lbDecisionCount = m->lbDecisionCount;
#endif
if (_lb_args.debug() > 1)
if (CkMyPe()%1024==0) CmiPrintf("[%d] Starting ReceiveMigration step %d at %f\n",CkMyPe(),step(), CmiWallTimer());
for (i=0; i<CkNumPes(); i++) theLbdb->lastLBInfo.expectedLoad[i] = m->expectedLoad[i];
CmiAssert(migrates_expected <= 0 || migrates_completed == migrates_expected);
/*FAULT_EVAC*/
if(!CmiNodeAlive(CkMyPe())){
delete m;
return;
}
migrates_expected = 0;
future_migrates_expected = 0;
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
int sending=0;
int dummy=0;
LBDB *_myLBDB = theLbdb->getLBDB();
if(_restartFlag){
dummyCounts = new int[CmiNumPes()];
bzero(dummyCounts,sizeof(int)*CmiNumPes());
}
#endif
for(i=0; i < m->n_moves; i++) {
MigrateInfo& move = m->moves[i];
const int me = CkMyPe();
if (move.from_pe == me && move.to_pe != me) {
DEBUGF(("[%d] migrating object to %d\n",move.from_pe,move.to_pe));
// migrate object, in case it is already gone, inform toPe
#if (!defined(_FAULT_MLOG_) && !defined(_FAULT_CAUSAL_))
if (theLbdb->Migrate(move.obj,move.to_pe) == 0)
thisProxy[move.to_pe].MissMigrate(!move.async_arrival);
#else
if(_restartFlag == 0){
DEBUG(CmiPrintf("[%d] need to move object from %d to %d \n",CkMyPe(),move.from_pe,move.to_pe));
theLbdb->Migrate(move.obj,move.to_pe);
sending++;
}else{
if(_myLBDB->validObjHandle(move.obj)){
DEBUG(CmiPrintf("[%d] need to move object from %d to %d \n",CkMyPe(),move.from_pe,move.to_pe));
theLbdb->Migrate(move.obj,move.to_pe);
sending++;
}else{
DEBUG(CmiPrintf("[%d] dummy move to pe %d detected after restart \n",CmiMyPe(),move.to_pe));
dummyCounts[move.to_pe]++;
dummy++;
}
}
#endif
} else if (move.from_pe != me && move.to_pe == me) {
DEBUGF(("[%d] expecting object from %d\n",move.to_pe,move.from_pe));
if (!move.async_arrival) migrates_expected++;
else future_migrates_expected++;
}
else {
#if CMK_GLOBAL_LOCATION_UPDATE
UpdateLocation(move);
#endif
}
}
DEBUGF(("[%d] in ReceiveMigration %d moves expected: %d future expected: %d\n",CkMyPe(),m->n_moves, migrates_expected, future_migrates_expected));
// if (_lb_debug) CkPrintf("[%d] expecting %d objects migrating.\n", CkMyPe(), migrates_expected);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
if(_restartFlag){
sendDummyMigrationCounts(dummyCounts);
_restartFlag =0;
delete []dummyCounts;
}
#endif
#if 0
if (m->n_moves ==0) {
theLbdb->SetLBPeriod(theLbdb->GetLBPeriod()*2);
}
#endif
cur_ld_balancer = m->next_lb;
if((CkMyPe() == cur_ld_balancer) && (cur_ld_balancer != 0)){
LBDatabaseObj()->set_avail_vector(m->avail_vector, -2);
}
if (migrates_expected == 0 || migrates_completed == migrates_expected)
MigrationDone(1);
delete m;
// CkEvacuatedElement();
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
// migrates_expected = 0;
// // ResumeClients(1);
#endif
#endif
}
void HybridBaseLB::PropagateInfo(Location *loc, int n, int fromlevel)
{
#if CMK_LBDB_ON
int i, obj;
int atlevel = fromlevel - 1;
LevelData *lData = levelData[atlevel];
CkVec<Location> &matchedObjs = lData->matchedObjs;
//CkVec<Location> &unmatchedObjs = lData->unmatchedObjs;
std::map<LDObjKey, int> &unmatchedObjs = lData->unmatchedObjs;
if (atlevel > 0) {
if (_lb_args.debug() > 1)
CkPrintf("[%d] PropagateInfo at level %d started at %f\n",
CkMyPe(), atlevel, CkWallTimer());
// search in unmatched
#if 0
for (i=0; i<n; i++) {
// search and see if we have answer, put to matched
// store in unknown
for (obj=0; obj<unmatchedObjs.size(); obj++) {
if (loc[i].key == unmatchedObjs[obj].key) {
// answer must exist now
CmiAssert(unmatchedObjs[obj].loc != -1 || loc[i].loc != -1);
if (unmatchedObjs[obj].loc == -1) unmatchedObjs[obj].loc = loc[i].loc;
matchedObjs.push_back(unmatchedObjs[obj]);
unmatchedObjs.remove(obj);
break;
}
}
}
#else
for (int i=0; i<n; i++) {
// search and see if we have answer, put to matched
const LDObjKey key = loc[i].key;
std::map<LDObjKey, int>::iterator iter = unmatchedObjs.find(key);
if (iter != unmatchedObjs.end()) {
// answer must exist now
CmiAssert(iter->second != -1 || loc[i].loc != -1);
if (loc[i].loc == -1) loc[i].loc = iter->second;
matchedObjs.push_back(loc[i]);
unmatchedObjs.erase(iter);
}
}
#endif
CmiAssert(unmatchedObjs.size() == 0);
DEBUGF(("[%d] level %d PropagateInfo had %d matchedObjs. \n", CkMyPe(), atlevel, matchedObjs.size()));
// send down
thisProxy.PropagateInfo(matchedObjs.getVec(), matchedObjs.size(), atlevel, lData->nChildren, lData->children);
lData->statsData->clear();
matchedObjs.free();
}
else { // leaf node
// now start to migrate
CkVec<MigrationRecord> & outObjs = lData->outObjs;
int migs = outObjs.size() + newObjs.size();
for (i=0; i<outObjs.size(); i++) {
if (outObjs[i].toPe == -1) {
for (obj=0; obj<n; obj++) {
if (loc[obj].key.omID() == outObjs[i].handle.omID() &&
loc[obj].key.objID() == outObjs[i].handle.objID()) {
outObjs[i].toPe = loc[obj].loc;
break;
}
}
CmiAssert(obj < n);
}
CmiAssert(outObjs[i].toPe != -1);
// migrate now!
theLbdb->Migrate(outObjs[i].handle,outObjs[i].toPe);
} // end for out
// incoming
lData->migrates_expected = 0;
future_migrates_expected = 0;
for (i=0; i<newObjs.size(); i++) {
if (newObjs[i].loc == -1) {
for (obj=0; obj<n; obj++) {
if (loc[obj].key == newObjs[i].key) {
newObjs[i].loc = loc[obj].loc;
break;
}
}
CmiAssert(obj < n);
}
CmiAssert(newObjs[i].loc != -1);
lData->migrates_expected++;
} // end of for
DEBUGF(("[%d] expecting %d\n", CkMyPe(), lData->migrates_expected));
if (lData->migrationDone()) {
MigrationDone(1);
}
}
#endif
}