void CTimer::resume(void)
{
if (suspended)
{
lastTime = getTime();
traceBegin(name);
}
suspended = false;
}
void LdbCoordinator::initialize(PatchMap *pMap, ComputeMap *cMap, int reinit)
{
const SimParameters *simParams = Node::Object()->simParameters;
#if 0
static int lbcreated = 0; // XXX static variables are unsafe for SMP
// PE0 first time Create a load balancer
if (CkMyPe() == 0 && !lbcreated) {
if (simParams->ldbStrategy == LDBSTRAT_ALGNBOR)
CreateNamdNborLB();
else {
// CreateCentralLB();
CreateNamdCentLB();
}
lbcreated = 1;
}
#endif
// DebugM(10,"stepsPerLdbCycle initialized\n");
stepsPerLdbCycle = simParams->ldbPeriod;
firstLdbStep = simParams->firstLdbStep;
int lastLdbStep = simParams->lastLdbStep;
int stepsPerCycle = simParams->stepsPerCycle;
computeMap = cMap;
patchMap = pMap;
// Set the number of received messages correctly for node 0
nStatsMessagesExpected = Node::Object()->numNodes();
nStatsMessagesReceived = 0;
if (patchNAtoms)
delete [] patchNAtoms; // Depends on delete NULL to do nothing
nPatches = patchMap->numPatches();
patchNAtoms = new int[nPatches];
typedef Sequencer *seqPtr;
if ( ! reinit ) {
delete [] sequencerThreads; // Depends on delete NULL to do nothing
sequencerThreads = new seqPtr[nPatches];
}
nLocalPatches=0;
int i;
for(i=0;i<nPatches;i++)
{
if (patchMap->node(i) == Node::Object()->myid())
{
nLocalPatches++;
patchNAtoms[i]=0;
} else {
patchNAtoms[i]=-1;
}
if ( ! reinit ) sequencerThreads[i]=NULL;
}
if ( ! reinit ) controllerThread = NULL;
if (nLocalPatches != patchMap->numHomePatches())
NAMD_die("Disaggreement in patchMap data.\n");
const int oldNumComputes = numComputes;
nLocalComputes = 0;
numComputes = computeMap->numComputes();
for(i=0;i<numComputes;i++) {
if ( (computeMap->node(i) == Node::Object()->myid())
&& ( 0
#ifndef NAMD_CUDA
|| (computeMap->type(i) == computeNonbondedSelfType)
|| (computeMap->type(i) == computeNonbondedPairType)
#endif
|| (computeMap->type(i) == computeLCPOType)
|| (computeMap->type(i) == computeSelfExclsType)
|| (computeMap->type(i) == computeSelfBondsType)
|| (computeMap->type(i) == computeSelfAnglesType)
|| (computeMap->type(i) == computeSelfDihedralsType)
|| (computeMap->type(i) == computeSelfImpropersType)
|| (computeMap->type(i) == computeSelfTholeType)
|| (computeMap->type(i) == computeSelfAnisoType)
|| (computeMap->type(i) == computeSelfCrosstermsType)
|| (computeMap->type(i) == computeBondsType)
|| (computeMap->type(i) == computeExclsType)
|| (computeMap->type(i) == computeAnglesType)
|| (computeMap->type(i) == computeDihedralsType)
|| (computeMap->type(i) == computeImpropersType)
|| (computeMap->type(i) == computeTholeType)
|| (computeMap->type(i) == computeAnisoType)
|| (computeMap->type(i) == computeCrosstermsType)
) ) {
nLocalComputes++;
}
}
// New LB frameworks registration
// Allocate data structure to save incoming migrations. Processor
// zero will get all migrations
// If this is the first time through, we need it register patches
if (ldbCycleNum == reg_all_objs) {
if ( Node::Object()->simParameters->ldBalancer == LDBAL_CENTRALIZED ) {
reg_all_objs = 3;
}
// Tell the lbdb that I'm registering objects, until I'm done
// registering them.
theLbdb->RegisteringObjects(myHandle);
if ( ldbCycleNum == 1 ) {
patchHandles = new LDObjHandle[nLocalPatches];
int patch_count=0;
int i;
for(i=0;i<nPatches;i++)
if (patchMap->node(i) == Node::Object()->myid()) {
LDObjid elemID;
elemID.id[0] = i;
elemID.id[1] = elemID.id[2] = elemID.id[3] = -2;
if (patch_count >= nLocalPatches) {
iout << iFILE << iERROR << iPE
<< "LdbCoordinator found too many local patches!" << endi;
CkExit();
}
HomePatch *p = patchMap->homePatch(i);
p->ldObjHandle =
patchHandles[patch_count]
= theLbdb->RegisterObj(myHandle,elemID,0,0);
patch_count++;
}
}
if ( numComputes > oldNumComputes ) {
// Register computes
for(i=oldNumComputes; i<numComputes; i++) {
if ( computeMap->node(i) == Node::Object()->myid())
{
if ( 0
#ifndef NAMD_CUDA
|| (computeMap->type(i) == computeNonbondedSelfType)
|| (computeMap->type(i) == computeNonbondedPairType)
#endif
|| (computeMap->type(i) == computeLCPOType)
|| (computeMap->type(i) == computeSelfExclsType)
|| (computeMap->type(i) == computeSelfBondsType)
|| (computeMap->type(i) == computeSelfAnglesType)
|| (computeMap->type(i) == computeSelfDihedralsType)
|| (computeMap->type(i) == computeSelfImpropersType)
|| (computeMap->type(i) == computeSelfTholeType)
|| (computeMap->type(i) == computeSelfAnisoType)
|| (computeMap->type(i) == computeSelfCrosstermsType)
) {
// Register the object with the load balancer
// Store the depended patch IDs in the rest of the element ID
LDObjid elemID;
elemID.id[0] = i;
if (computeMap->numPids(i) > 2)
elemID.id[3] = computeMap->pid(i,2);
else elemID.id[3] = -1;
if (computeMap->numPids(i) > 1)
elemID.id[2] = computeMap->pid(i,1);
else elemID.id[2] = -1;
if (computeMap->numPids(i) > 0)
elemID.id[1] = computeMap->pid(i,0);
else elemID.id[1] = -1;
Compute *c = computeMap->compute(i);
if ( ! c ) NAMD_bug("LdbCoordinator::initialize() null compute pointer");
c->ldObjHandle = theLbdb->RegisterObj(myHandle,elemID,0,1);
}
else if ( (computeMap->type(i) == computeBondsType)
|| (computeMap->type(i) == computeExclsType)
|| (computeMap->type(i) == computeAnglesType)
|| (computeMap->type(i) == computeDihedralsType)
|| (computeMap->type(i) == computeImpropersType)
|| (computeMap->type(i) == computeTholeType)
|| (computeMap->type(i) == computeAnisoType)
|| (computeMap->type(i) == computeCrosstermsType)
) {
// Register the object with the load balancer
// Store the depended patch IDs in the rest of the element ID
LDObjid elemID;
elemID.id[0] = i;
elemID.id[1] = elemID.id[2] = elemID.id[3] = -3;
Compute *c = computeMap->compute(i);
if ( ! c ) NAMD_bug("LdbCoordinator::initialize() null compute pointer");
c->ldObjHandle = theLbdb->RegisterObj(myHandle,elemID,0,0);
}
}
}
}
theLbdb->DoneRegisteringObjects(myHandle);
}
// process saved migration messages, if any
while ( migrateMsgs ) {
LdbMigrateMsg *m = migrateMsgs;
migrateMsgs = m->next;
Compute *c = computeMap->compute(m->handle.id.id[0]);
if ( ! c ) NAMD_bug("LdbCoordinator::initialize() null compute pointer 2");
c->ldObjHandle = m->handle;
delete m;
}
// Fixup to take care of the extra timestep at startup
// This is pretty ugly here, but it makes the count correct
// iout << "LDB Cycle Num: " << ldbCycleNum << "\n";
if ( simParams->ldBalancer == LDBAL_CENTRALIZED ) {
if (ldbCycleNum == 1 || ldbCycleNum == 3) {
numStepsToRun = stepsPerCycle;
totalStepsDone += numStepsToRun;
takingLdbData = 0;
theLbdb->CollectStatsOff();
} else if (ldbCycleNum == 2 || ldbCycleNum == 4) {
numStepsToRun = firstLdbStep - stepsPerCycle;
while ( numStepsToRun <= 0 ) numStepsToRun += stepsPerCycle;
totalStepsDone += numStepsToRun;
takingLdbData = 1;
theLbdb->CollectStatsOn();
} else if ( (ldbCycleNum <= 6) || !takingLdbData )
{
totalStepsDone += firstLdbStep;
if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
numStepsToRun = -1;
takingLdbData = 0;
theLbdb->CollectStatsOff();
} else {
numStepsToRun = firstLdbStep;
takingLdbData = 1;
theLbdb->CollectStatsOn();
}
}
else
{
totalStepsDone += stepsPerLdbCycle - firstLdbStep;
if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
numStepsToRun = -1;
takingLdbData = 0;
theLbdb->CollectStatsOff();
} else {
numStepsToRun = stepsPerLdbCycle - firstLdbStep;
takingLdbData = 0;
theLbdb->CollectStatsOff();
}
}
} else {
if (ldbCycleNum==1)
{
totalStepsDone += firstLdbStep;
numStepsToRun = firstLdbStep;
takingLdbData = 0;
theLbdb->CollectStatsOff();
}
else if ( (ldbCycleNum <= 4) || !takingLdbData )
{
totalStepsDone += firstLdbStep;
if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
numStepsToRun = -1;
takingLdbData = 0;
theLbdb->CollectStatsOff();
} else {
numStepsToRun = firstLdbStep;
takingLdbData = 1;
theLbdb->CollectStatsOn();
}
}
else
{
totalStepsDone += stepsPerLdbCycle - firstLdbStep;
if(lastLdbStep != -1 && totalStepsDone > lastLdbStep) {
numStepsToRun = -1;
takingLdbData = 0;
theLbdb->CollectStatsOff();
} else {
numStepsToRun = stepsPerLdbCycle - firstLdbStep;
takingLdbData = 0;
theLbdb->CollectStatsOff();
}
}
}
/*-----------------------------------------------------------------------------*
* --------------------------------------------------------------------------- *
* Comments inserted by Abhinav to clarify relation between ldbCycleNum, *
* load balancing step numbers (printed by the step() function) and *
* tracing of the steps *
* --------------------------------------------------------------------------- *
* If trace is turned off in the beginning, then tracing is turned on *
* at ldbCycleNum = 4 and turned off at ldbCycleNum = 8. ldbCycleNum can *
* be adjusted by specifying firstLdbStep and ldbPeriod which are set by *
* default to 5*stepspercycle and 200*stepspercycle if not specified. *
* *
* If we choose firstLdbStep = 20 and ldbPeriod = 100, we have the *
* following timeline (for these particular numbers): *
* *
* Tracing : <------ off ------><------------- on -----------><-- off *
* Ldb Step() No : 1 2 3 4 5 6 7 *
* Iteration Steps : 00====20====40====60====80======160====180=====260====280 *
* ldbCycleNum : 1 2 3 4 5 6 7 8 9 *
* Instrumention : Inst Inst Inst Inst Inst *
* LDB Strategy : TLB RLB RLB RLB RLB *
* *
* TLB = TorusLB *
* RLB = RefineTorusLB *
* Inst = Instrumentation Phase (no real load balancing) *
* --------------------------------------------------------------------------- *
*-----------------------------------------------------------------------------*
*/
#if 0 //replaced by traceBarrier at Controller and Sequencer
if (traceAvailable()) {
static int specialTracing = 0; // XXX static variables are unsafe for SMP
if (ldbCycleNum == 1 && traceIsOn() == 0) specialTracing = 1;
if (specialTracing) {
if (ldbCycleNum == 4) traceBegin();
if (ldbCycleNum == 8) traceEnd();
}
}
#endif
nPatchesReported = 0;
nPatchesExpected = nLocalPatches;
nComputesReported = 0;
nComputesExpected = nLocalComputes * numStepsToRun;
controllerReported = 0;
controllerExpected = ! CkMyPe();
if (CkMyPe() == 0)
{
if (computeArray == NULL)
computeArray = new computeInfo[numComputes];
if (patchArray == NULL)
patchArray = new patchInfo[nPatches];
if (processorArray == NULL)
processorArray = new processorInfo[CkNumPes()];
}
theLbdb->ClearLoads();
}
void startTrace() { traceBegin(); }