/// Basic Constructor
strat::strat()
{
#ifndef CMK_OPTIMIZE
if(pose_config.stats)
localStats = (localStat *)CkLocalBranch(theLocalStats);
#endif
eq = NULL;
userObj = NULL;
parent = NULL;
currentEvent = targetEvent = NULL;
#ifndef SEQUENTIAL_POSE
localPVT = (PVT *)CkLocalBranch(ThePVT);
#endif
STRAT_T = INIT_T;
}
/// PUP routine
void GVT::pup(PUP::er &p) {
p|estGVT; p|inactive; p|inactiveTime; p|nextLBstart;
p|lastEarliest; p|lastSends; p|lastRecvs; p|reportsExpected;
p|optGVT; p|conGVT; p|done; p|startOffset;
p|gvtIterationCount;
if (p.isUnpacking()) {
#ifndef CMK_OPTIMIZE
localStats = (localStat *)CkLocalBranch(theLocalStats);
#endif
}
int nullFlag;
if (SRs == NULL) {
nullFlag = 1;
} else {
nullFlag = 0;
}
p|nullFlag;
if (p.isUnpacking()) {
if (nullFlag) {
SRs = NULL;
} else {
SRs = new SRentry();
SRs->pup(p);
}
} else {
if (!nullFlag) {
SRs->pup(p);
}
}
}
/// Basic Constructor
GVT::GVT()
{
#ifdef VERBOSE_DEBUG
CkPrintf("[%d] constructing GVT\n",CkMyPe());
#endif
optGVT = POSE_UnsetTS, conGVT = POSE_UnsetTS;
done=0;
SRs = NULL;
startOffset = 0;
gvtIterationCount = 0;
#ifndef CMK_OPTIMIZE
localStats = (localStat *)CkLocalBranch(theLocalStats);
#endif
#ifndef SEQUENTIAL_POSE
if(pose_config.lb_on)
nextLBstart = pose_config.lb_skip - 1;
#endif
estGVT = lastEarliest = inactiveTime = POSE_UnsetTS;
lastSends = lastRecvs = inactive = 0;
reportsExpected = 1;
if (CkNumPes() >= 2) reportsExpected = 2;
// CkPrintf("GVT expects %d reports!\n", reportsExpected);
if (CkMyPe() == 0) { // start the PVT phase of the GVT algorithm
CProxy_PVT p(ThePVT);
prioBcMsg *startMsg = new (8*sizeof(int)) prioBcMsg;
startMsg->bc = 1;
*((int *)CkPriorityPtr(startMsg)) = 0;
CkSetQueueing(startMsg, CK_QUEUEING_IFIFO);
p.startPhase(startMsg); // broadcast PVT calculation to all PVT branches
}
}
/// PUP routine
void PVT::pup(PUP::er &p) {
p|optPVT; p|conPVT; p|estGVT; p|repPVT;
p|simdone; p|iterMin; p|waitForFirst;
p|reportTo; p|reportsExpected; p|reportReduceTo; p|reportEnd;
p|gvtTurn; p|specEventCount; p|eventCount;
p|startPhaseActive; p|parStartTime; p|parCheckpointInProgress;
p|parLastCheckpointGVT; p|parLastCheckpointTime;
p|parLBInProgress; p|parLastLBGVT;
p|optGVT; p|conGVT; p|rdone;
if (p.isUnpacking()) {
parStartTime = parLastCheckpointTime;
#ifndef CMK_OPTIMIZE
localStats = (localStat *)CkLocalBranch(theLocalStats);
#endif
#ifdef MEM_TEMPORAL
localTimePool = (TimePool *)CkLocalBranch(TempMemID);
#endif
SendsAndRecvs = new SRtable();
}
SendsAndRecvs->pup(p);
int nullFlag;
if (SRs == NULL) {
nullFlag = 1;
} else {
nullFlag = 0;
}
p|nullFlag;
if (p.isUnpacking()) {
if (nullFlag) {
SRs = NULL;
} else {
SRs = new SRentry();
SRs->pup(p);
}
} else {
if (!nullFlag) {
SRs->pup(p);
}
}
}
// converse handler
// send converse message to avoid QD detection
static void migrationDone(envelope *env, CkCoreState *ck)
{
// Since migrationsDone will deal with Charm++ messages,
// the LB must obey the CkMessageWatcher orders.
if (ck->watcher!=NULL) {
if (!ck->watcher->processMessage(&env,ck)) return;
}
NullLB *lb = (NullLB*)CkLocalBranch(_theNullLB);
lb->migrationsDone();
CkFreeSysMsg(EnvToUsr(env));
}
void LBDatabase::UpdateDataAfterLB(double mLoad, double mCpuLoad, double avgLoad) {
#if CMK_LBDB_ON
if (_lb_args.metaLbOn()) {
if (metabalancer == NULL) {
metabalancer = (MetaBalancer *)CkLocalBranch(_metalb);
}
if (metabalancer != NULL) {
metabalancer->UpdateAfterLBData(mLoad, mCpuLoad, avgLoad);
}
}
#endif
}
void LBDatabase::SetStrategyCost(double cost) {
#if CMK_LBDB_ON
if (_lb_args.metaLbOn()) {
if (metabalancer == NULL) {
metabalancer = (MetaBalancer *)CkLocalBranch(_metalb);
}
if (metabalancer != NULL) {
metabalancer->SetStrategyCost(cost);
}
}
#endif
}
/// Pack/unpack/sizing operator
void sim::pup(PUP::er &p) {
// pup simple types
p(active); p(myPVTidx); p(myLBidx); p(sync); p(DOs); p(UNDOs);
// pup event queue
if (p.isUnpacking()) {
eq = new eventQueue();
}
eq->pup(p);
// pup cancellations
cancels.pup(p);
if (p.isUnpacking()) { // reactivate migrated object
#if !CMK_TRACE_DISABLED
localStats = (localStat *)CkLocalBranch(theLocalStats);
#endif
#ifndef SEQUENTIAL_POSE
localPVT = (PVT *)CkLocalBranch(ThePVT);
myPVTidx = localPVT->objRegister(thisIndex, localPVT->getGVT(), sync, this);
if(pose_config.lb_on){
localLBG = TheLBG.ckLocalBranch();
myLBidx = localLBG->objRegister(thisIndex, sync, this);
}
#endif
active = 0;
}
PUParray(p, basicStats, 2);
// pup checkpoint info for sequential mode using sim 0 only
#ifdef SEQUENTIAL_POSE
if (thisIndex == 0) {
p|seqCheckpointInProgress;
p|seqLastCheckpointGVT;
p|seqLastCheckpointTime;
p|seqStartTime;
p|POSE_Skipped_Events;
p|poseIndexOfStopEvent;
if (p.isUnpacking()) {
seqStartTime = seqLastCheckpointTime;
}
}
#endif
}
/// Timestamps event message, sets priority, and records in spawned list
void rep::registerTimestamp(int idx, eventMsg *m, POSE_TimeType offset)
{
#ifndef SEQUENTIAL_POSE
PVT *localPVT = (PVT *)CkLocalBranch(ThePVT);
CmiAssert(ovt+offset >= localPVT->getGVT());
#endif
m->Timestamp(ovt+offset);
m->setPriority(ovt+offset-POSE_TimeMax);
//m->evID.setObj(myHandle);
#ifndef SEQUENTIAL_POSE
parent->registerSent(ovt+offset);
#endif
}
/// Basic Constructor
eventQueue::eventQueue()
{
lastLoggedVT = 0;
Event *e;
eqh = new EqHeap(); // create the heap for incoming events
largest = POSE_UnsetTS;
mem_usage = 0;
eventCount = 0;
tsOfLastInserted = 0;
// create the front sentinel node
e = new Event();
e->timestamp = POSE_UnsetTS;
e->done = -1;
e->fnIdx = -99;
e->msg = NULL;
e->commitBfr = NULL;
e->spawnedList = NULL;
e->commitBfrLen = 0;
e->next = e->prev = NULL;
frontPtr = e;
// create the back sentinel node
e = new Event();
e->timestamp=POSE_UnsetTS;
e->done = -1;
e->fnIdx = -100;
e->msg = NULL;
e->commitBfr = NULL;
e->spawnedList = NULL;
e->commitBfrLen = 0;
e->next = e->prev = NULL;
currentPtr = backPtr = e; // when no unprocessed events, currentPtr=backPtr
// link them together
frontPtr->next = backPtr;
backPtr->prev = frontPtr;
RBevent = NULL;
// other variables
recentAvgEventSparsity = 1;
sparsityStartTime = 0;
sparsityCalcCount = 0;
tsDiffCount = 0;
lastCommittedTS = 0;
for (int i = 0; i < DIFFS_TO_STORE; i++) {
tsCommitDiffs[i] = 0;
}
#ifdef MEM_TEMPORAL
localTimePool = (TimePool *)CkLocalBranch(TempMemID);
#endif
#ifdef EQ_SANITIZE
sanitize();
#endif
}
/// Basic Constructor
sim::sim()
{
#ifdef VERBOSE_DEBUG
CkPrintf("[%d] constructing sim %d\n",CkMyPe(), thisIndex);
#endif
#ifndef SEQUENTIAL_POSE
localPVT = (PVT *)CkLocalBranch(ThePVT);
if(pose_config.lb_on)
localLBG = TheLBG.ckLocalBranch();
#endif
#if !CMK_TRACE_DISABLED
if(pose_config.stats)
localStats = (localStat *)CkLocalBranch(theLocalStats);
#endif
basicStats[0] = basicStats[1] = 0LL;
lastGVT = active = DOs = UNDOs = 0;
srVector = (int *)malloc(CkNumPes() * sizeof(int));
for (int i=0; i<CkNumPes(); i++) srVector[i] = 0;
eq = new eventQueue();
myStrat = NULL;
objID = NULL;
}
void LBDatabase::pup(PUP::er& p)
{
IrrGroup::pup(p);
// the memory should be already allocated
int np;
if (!p.isUnpacking()) np = CkNumPes();
p|np;
// in case number of processors changes
if (p.isUnpacking()) {
CmiLock(avail_vector_lock);
if(!avail_vector_set){
avail_vector_set = true;
CmiAssert(avail_vector);
if(np>CkNumPes()){
delete [] avail_vector;
avail_vector = new char[np];
for (int i=0; i<np; i++) avail_vector[i] = 1;
}
p(avail_vector, np);
} else{
char * tmp_avail_vector = new char[np];
p(tmp_avail_vector, np);
delete [] tmp_avail_vector;
}
CmiUnlock(avail_vector_lock);
} else{
CmiAssert(avail_vector);
p(avail_vector, np);
}
p|mystep;
if(p.isUnpacking()) {
nloadbalancers = 0;
if (_lb_args.metaLbOn()) {
// if unpacking set metabalancer using the id
metabalancer = (MetaBalancer*)CkLocalBranch(_metalb);
}
}
}
/// Basic Constructor
PVT::PVT()
{
#ifdef VERBOSE_DEBUG
CkPrintf("[%d] constructing PVT\n",CkMyPe());
#endif
CpvInitialize(int, stateRecovery);
CpvAccess(stateRecovery) = 0;
CpvInitialize(eventID, theEventID);
CpvAccess(theEventID)=eventID();
// CpvAccess(theEventID).dump();
//LBTurnInstrumentOff();
optGVT = POSE_UnsetTS; conGVT = POSE_UnsetTS;
rdone=0;
SRs=NULL;
#ifdef POSE_COMM_ON
//com_debug = 1;
#endif
#ifndef CMK_OPTIMIZE
localStats = (localStat *)CkLocalBranch(theLocalStats);
if (pose_config.stats) {
localStats->TimerStart(GVT_TIMER);
}
#endif
#ifdef MEM_TEMPORAL
localTimePool = (TimePool *)CkLocalBranch(TempMemID);
CkPrintf("NOTE: Temporal memory manager is ON!\n");
#endif
optPVT = conPVT = estGVT = POSE_UnsetTS;
startPhaseActive = gvtTurn = simdone = 0;
SendsAndRecvs = new SRtable();
SendsAndRecvs->Initialize();
specEventCount = eventCount = waitForFirst = 0;
iterMin = POSE_UnsetTS;
int P=CkNumPes(), N=CkMyPe();
reportReduceTo = -1;
if ((N < P-2) && (N%2 == 1)) { //odd
reportTo = N-1;
reportsExpected = reportEnd = 0;
}
else if (N < P-2) { //even
reportTo = N;
reportsExpected = 2;
if (N == P-3)
reportsExpected = 1;
reportEnd = 0;
if (N < (P-2)/2)
reportReduceTo = P-2;
else reportReduceTo = P-1;
}
if (N == P-2) {
reportTo = N;
reportEnd = 1;
reportsExpected = 1 + (P-2)/4 + ((P-2)%4)/2;
}
else if (N == P-1) {
reportTo = N;
reportEnd = 1;
if (P==1) reportsExpected = 1;
else reportsExpected = 1 + (P-2)/4 + (P-2)%2;
}
// CkPrintf("PE %d reports to %d, receives %d reports, reduces and sends to %d, and reports directly to GVT if %d = 1!\n", CkMyPe(), reportTo, reportsExpected, reportReduceTo, reportEnd);
parCheckpointInProgress = 0;
parLastCheckpointGVT = 0;
parLastCheckpointTime = parStartTime = 0.0;
parLBInProgress = 0;
parLastLBGVT = 0;
// debugBufferLoc = debugBufferWrapped = debugBufferDumped = 0;
#ifndef CMK_OPTIMIZE
if(pose_config.stats)
localStats->TimerStop();
#endif
LBDatabase::Object()->AddMigrationDoneFn(staticDoneLB, this);
}
void eventQueue::CommitStatsHelper(sim *obj, Event *commitPtr) {
#if !CMK_TRACE_DISABLED
localStat *localStats = (localStat *)CkLocalBranch(theLocalStats);
if (pose_config.stats) {
localStats->Commit();
}
if (pose_config.dop) {
// if more than one event occurs at the same virtual time on this object,
// only count the first event
if (lastLoggedVT >= commitPtr->svt) {
commitPtr->svt = commitPtr->evt = -1;
} else {
lastLoggedVT = commitPtr->evt;
}
localStats->WriteDopData(commitPtr->srt, commitPtr->ert, commitPtr->svt, commitPtr->evt);
localStats->SetMaximums(commitPtr->evt, commitPtr->ert);
}
#endif
// only execute if sync strategy is adapt5
if (obj->myStrat->STRAT_T == ADAPT5_T) {
sparsityCalcCount++;
// run the sparsity calculation and store in strat
if (sparsityCalcCount >= EVQ_SPARSE_CALC_PERIOD) {
recentAvgEventSparsity = (int)((commitPtr->timestamp - sparsityStartTime) / sparsityCalcCount);
if (recentAvgEventSparsity < 1) {
recentAvgEventSparsity = 1;
}
((adapt5 *)obj->myStrat)->setRecentAvgEventSparsity(recentAvgEventSparsity);
sparsityCalcCount = 0;
sparsityStartTime = commitPtr->timestamp;
}
// calculate the diffs for one of the adapt5 algorithms
POSE_TimeType diff = commitPtr->timestamp - lastCommittedTS;
lastCommittedTS = commitPtr->timestamp;
for (int i = 0; i < DIFFS_TO_STORE; i++) {
if (diff > tsCommitDiffs[i]) {
// insert in list
for (int j = DIFFS_TO_STORE - 2; j > i; j--) {
tsCommitDiffs[j+1] = tsCommitDiffs[j];
}
if (i < DIFFS_TO_STORE - 1) {
// not at the end of the array
tsCommitDiffs[i+1] = tsCommitDiffs[i];
}
tsCommitDiffs[i] = diff;
break;
}
}
tsDiffCount++;
// calculate the timeleash and store in strat every TS_DIFF_WIN_SIZE events
if (tsDiffCount >= TS_DIFF_WIN_SIZE) {
POSE_TimeType totalDiff = 0;
for (int i = HIGHEST_DIFFS_TO_IGNORE; i < DIFFS_TO_STORE; i++) {
totalDiff += tsCommitDiffs[i];
}
POSE_TimeType avgDiff = totalDiff / NUM_DIFFS_TO_AVERAGE;
((adapt5 *)obj->myStrat)->setTimeLeash(DIFFS_TO_STORE * avgDiff);
tsDiffCount = 0;
for (int i = 0; i < DIFFS_TO_STORE; i++) {
tsCommitDiffs[i] = 0;
}
}
}
}
