// Receive an unoccupied psi, and split off the computation of all associated f
// vectors across the node using CkLoop.
void PsiCache::computeFs(PsiMessage* msg) {
double start = CmiWallTimer();
if (msg->spin_index != 0) {
CkAbort("Error: We don't support multiple spins yet!\n");
}
CkAssert(msg->size == psi_size);
// Compute ikq index and the associated umklapp factor
// TODO: This should just be a table lookup
unsigned ikq;
int umklapp[3];
kqIndex(msg->k_index, ikq, umklapp);
bool uproc = false;
if (umklapp[0] != 0 || umklapp[1] != 0 || umklapp[2] != 0) {
uproc = true;
computeUmklappFactor(umklapp);
}
GWBSE* gwbse = GWBSE::get();
double*** e_occ = gwbse->gw_epsilon.Eocc;
double*** e_occ_shifted = gwbse->gw_epsilon.Eocc_shifted;
double*** e_unocc = gwbse->gw_epsilon.Eunocc;
// Create the FComputePacket for this set of f vectors and start CkLoop
f_packet.size = psi_size;
f_packet.unocc_psi = msg->psi;
if ( qindex == 0 ) {
f_packet.occ_psis = psis_shifted[ikq];
f_packet.e_occ = e_occ_shifted[msg->spin_index][ikq];
}
else {
f_packet.occ_psis = psis[ikq];
f_packet.e_occ = e_occ[msg->spin_index][ikq];
}
f_packet.e_unocc = e_unocc[msg->spin_index][msg->k_index][msg->state_index-L];
f_packet.fs = fs + (L*psi_size*(received_chunks%pipeline_stages));
if (uproc) { f_packet.umklapp_factor = umklapp_factor; }
else { f_packet.umklapp_factor = NULL; }
#ifdef USE_CKLOOP
CkLoop_Parallelize(computeF, 1, &f_packet, L, 0, L - 1);
#else
for (int l = 0; l < L; l++) {
computeF(l,l,NULL,1,&f_packet);
}
#endif
received_chunks++;
#ifdef TESTING
{
FVectorCache *fvec_cache = fvector_cache_proxy.ckLocalBranch();
fvec_cache->computeFTilde(fs);
// fvec_cache->applyCutoff(msg->accept_size, msg->accept);
// fvec_cache->init(140);
//compute ftilde first - similar to ckloop above for all L's
fvec_cache->putFVec(msg->state_index-L, fs);
}
#endif
// Let the matrix chares know that the f vectors are ready
CkCallback cb(CkReductionTarget(PMatrix, applyFs), pmatrix2D_proxy);
contribute(cb);
// Cleanup
delete msg;
total_time += CmiWallTimer() - start;
}
/// Busy wait for n
void POSE_busy_wait(double n)
{
double start = CmiWallTimer();
while (CmiWallTimer() - start < n) ;
}
void POSE_init(int IDflag, int ET) // can specify both
{
CkPrintf("Initializing POSE... \n");
POSEreadCmdLine();
if (pose_config.checkpoint_gvt_interval) {
CkPrintf("POSE checkpointing interval set to %lld GVT ticks\n", pose_config.checkpoint_gvt_interval);
}
if (pose_config.checkpoint_time_interval) {
CkPrintf("POSE checkpointing interval set to %d seconds\n", pose_config.checkpoint_time_interval);
}
if (pose_config.dop) {
CkPrintf("POSE DOP analysis enabled...deleting dop log files...\n");
char fName[32];
for (int i = 0; i < CkNumPes(); i++) {
sprintf(fName, "dop%d.log", i);
unlink(fName);
}
sprintf(fName, "dop_mod.out");
unlink(fName);
sprintf(fName, "dop_sim.out");
unlink(fName);
}
POSE_inactDetect = IDflag;
totalNumPosers = 0;
POSE_endtime = ET;
#ifdef SEQUENTIAL_POSE
_POSE_SEQUENTIAL = 1;
#else
_POSE_SEQUENTIAL = 0;
#endif
#if !CMK_TRACE_DISABLED
traceRegisterUserEvent("Forward Execution", 10);
traceRegisterUserEvent("Cancellation", 20);
traceRegisterUserEvent("Cancel Spawn", 30);
traceRegisterUserEvent("Rollback", 40);
traceRegisterUserEvent("Commit", 50);
traceRegisterUserEvent("OptSync", 60);
#endif
#ifndef SEQUENTIAL_POSE
#ifdef POSE_COMM_ON
// Create the communication library for POSE
POSE_commlib_insthndl = CkGetComlibInstance();
// Create the communication strategy for POSE
StreamingStrategy *strategy = new StreamingStrategy(COMM_TIMEOUT,COMM_MAXMSG);
//MeshStreamingStrategy *strategy = new MeshStreamingStrategy(COMM_TIMEOUT,COMM_MAXMSG);
//PrioStreaming *strategy = new PrioStreaming(COMM_TIMEOUT,COMM_MAXMSG);
//Register the strategy
POSE_commlib_insthndl.setStrategy(strategy);
//com_debug=1;
//CkPrintf("Simulation run with PrioStreaming(%d,%d) for communication optimization...\n", COMM_TIMEOUT, COMM_MAXMSG);
CkPrintf("Simulation run with StreamingStrategy(%d,%d) for communication optimization...\n", COMM_TIMEOUT, COMM_MAXMSG);
//CkPrintf("Simulation run with MeshStreaming(%d,%d) for communication optimization...\n", COMM_TIMEOUT, COMM_MAXMSG);
#endif
// Create a MemoryPool with global handle for memory recycling
MemPoolID = CProxy_MemoryPool::ckNew();
// Create a Temporal Memory Manager
TempMemID = CProxy_TimePool::ckNew();
#endif
// Initialize statistics collection if desired
#if !CMK_TRACE_DISABLED
theLocalStats = CProxy_localStat::ckNew();
CProxy_globalStat::ckNew(&theGlobalStats);
#endif
#ifndef SEQUENTIAL_POSE
// Initialize global handles to GVT and PVT
ThePVT = CProxy_PVT::ckNew();
TheGVT = CProxy_GVT::ckNew();
// Start off using normal forward execution
if(pose_config.lb_on)
{
// Initialize the load balancer
TheLBG = CProxy_LBgroup::ckNew();
TheLBstrategy = CProxy_LBstrategy::ckNew();
CkPrintf("Load balancing is ON.\n");
}
#endif
CProxy_pose::ckNew(&POSE_Coordinator_ID, 0);
// Create array to hold all POSE objects
#ifdef POSE_COMM_ON
POSE_Objects_RO = CProxy_sim::ckNew();
POSE_Objects = POSE_Objects_RO;
#else
POSE_Objects = CProxy_sim::ckNew();
#endif
//#ifndef SEQUENTIAL_POSE
//#ifdef POSE_COMM_ON
// Make POSE_Objects use the comm lib
// ComlibDelegateProxy(&POSE_Objects);
//#endif
//#endif
#ifdef SEQUENTIAL_POSE
if (CkNumPes() > 1) CkAbort("ERROR: Cannot run a sequential simulation on more than one processor!\n");
CkPrintf("NOTE: POSE running in sequential simulation mode!\n");
int fnIdx = CkIndex_pose::stop();
CkStartQD(fnIdx, &POSE_Coordinator_ID);
POSE_GlobalClock = 0;
POSE_GlobalTS = 0;
seqCheckpointInProgress = 0;
seqLastCheckpointGVT = 0;
seqLastCheckpointTime = seqStartTime = 0.0;
poseIndexOfStopEvent = -1;
#else
/* CkPrintf("WARNING: Charm Quiescence termination enabled!\n");
int fnIdx = CkIndex_pose::stop();
CkStartQD(fnIdx, &POSE_Coordinator_ID);
*/
#endif
CkPrintf("POSE initialization complete.\n");
if (POSE_inactDetect) CkPrintf("Using Inactivity Detection for termination.\n");
else
#if USE_LONG_TIMESTAMPS
CkPrintf("Using endTime of %lld for termination.\n", POSE_endtime);
#else
CkPrintf("Using endTime of %d for termination.\n", POSE_endtime);
#endif
sim_timer = CmiWallTimer();
}
int ComputeNonbondedPair::noWork() {
if (patch[0]->flags.doGBIS) {
gbisPhase = 1 + (gbisPhase % 3);//1->2->3->1...
}
#ifndef NAMD_CUDA
if ( patch[0]->flags.doNonbonded && (numAtoms[0] && numAtoms[1]) ) {
return 0; // work to do, enqueue as usual
} else {
#else
{
#endif
if (patch[0]->flags.doGBIS) {
if (gbisPhase == 1) {
for (int i=0; i<2; i++) {
psiSumBox[i]->skip();
intRadBox[i]->skip();
}
if (patch[0]->flags.doNonbonded) return 1;
else gbisPhase = 2;
}
if (gbisPhase == 2) {
for (int i=0; i<2; i++) {
bornRadBox[i]->skip();
dEdaSumBox[i]->skip();
}
if (patch[0]->flags.doNonbonded) return 1;
else gbisPhase = 3;
}
if (gbisPhase == 3) {
for (int i=0; i<2; i++) {
dHdrPrefixBox[i]->skip();
}
}
}
// skip all boxes
for (int i=0; i<2; i++) {
positionBox[i]->skip();
forceBox[i]->skip();
if ( patch[0]->flags.doMolly ) avgPositionBox[i]->skip();
// BEGIN LA
if (patch[0]->flags.doLoweAndersen) velocityBox[i]->skip();
// END LA
}
reduction->item(REDUCTION_COMPUTE_CHECKSUM) += 1.;
reduction->submit();
if (accelMDOn) amd_reduction->submit();
if (pressureProfileOn)
pressureProfileReduction->submit();
#ifndef NAMD_CUDA
// Inform load balancer
LdbCoordinator::Object()->skipWork(ldObjHandle);
#endif
return 1; // no work to do, do not enqueue
}
}
void ComputeNonbondedPair::doForce(CompAtom* p[2], CompAtomExt* pExt[2], Results* r[2])
{
// Inform load balancer.
// I assume no threads will suspend until endWork is called
//single phase declarations
int doEnergy = patch[0]->flags.doEnergy;
int a = 0; int b = 1;
// swap to place more atoms in inner loop (second patch)
if ( numAtoms[0] > numAtoms[1] ) { a = 1; b = 0; }
CompAtom* v[2];
/*******************************************************************************
* Prepare Parameters
*******************************************************************************/
if (!patch[0]->flags.doGBIS || gbisPhase == 1) {
#ifdef TRACE_COMPUTE_OBJECTS
double traceObjStartTime = CmiWallTimer();
#endif
DebugM(2,"doForce() called.\n");
DebugM(2, numAtoms[0] << " patch #1 atoms and " <<
numAtoms[1] << " patch #2 atoms\n");
for ( int i = 0; i < reductionDataSize; ++i )
reductionData[i] = 0;
if (pressureProfileOn) {
int n = pressureProfileAtomTypes;
memset(pressureProfileData, 0, 3*n*n*pressureProfileSlabs*sizeof(BigReal));
// adjust lattice dimensions to allow constant pressure
const Lattice &lattice = patch[0]->lattice;
pressureProfileThickness = lattice.c().z / pressureProfileSlabs;
pressureProfileMin = lattice.origin().z - 0.5*lattice.c().z;
}
params.reduction = reductionData;
params.pressureProfileReduction = pressureProfileData;
params.minPart = minPart;
params.maxPart = maxPart;
params.numParts = numParts;
params.workArrays = workArrays;
params.pairlists = &pairlists;
params.savePairlists = 0;
params.usePairlists = 0;
if ( patch[0]->flags.savePairlists ) {
params.savePairlists = 1;
params.usePairlists = 1;
} else if ( patch[0]->flags.usePairlists && patch[1]->flags.usePairlists ) {
if ( ! pairlistsValid ||
( patch[0]->flags.maxAtomMovement +
patch[1]->flags.maxAtomMovement > pairlistTolerance ) ) {
reductionData[pairlistWarningIndex] += 1;
} else {
params.usePairlists = 1;
}
}
if ( ! params.usePairlists ) {
pairlistsValid = 0;
}
params.plcutoff = cutoff;
params.groupplcutoff = cutoff +
patch[0]->flags.maxGroupRadius + patch[1]->flags.maxGroupRadius;
if ( params.savePairlists ) {
pairlistsValid = 1;
pairlistTolerance = patch[0]->flags.pairlistTolerance +
patch[1]->flags.pairlistTolerance;
params.plcutoff += pairlistTolerance;
params.groupplcutoff += pairlistTolerance;
}
const Lattice &lattice = patch[0]->lattice;
params.offset = lattice.offset(trans[a]) - lattice.offset(trans[b]);
// Atom Sorting : If we are sorting the atoms along the line connecting
// the patch centers, then calculate a normalized vector pointing from
// patch a to patch b (i.e. outer loop patch to inner loop patch).
#if NAMD_ComputeNonbonded_SortAtoms != 0
// Center of patch a (outer-loop; i-loop) and patch b (inner-loop; j/k-loop)
PatchMap* patchMap = PatchMap::Object();
ScaledPosition p_a_center, p_b_center;
p_a_center.x = patchMap->min_a(patchID[a]);
p_a_center.y = patchMap->min_b(patchID[a]);
p_a_center.z = patchMap->min_c(patchID[a]);
p_b_center.x = patchMap->min_a(patchID[b]);
p_b_center.y = patchMap->min_b(patchID[b]);
p_b_center.z = patchMap->min_c(patchID[b]);
p_a_center.x += patchMap->max_a(patchID[a]);
p_a_center.y += patchMap->max_b(patchID[a]);
p_a_center.z += patchMap->max_c(patchID[a]);
p_b_center.x += patchMap->max_a(patchID[b]);
p_b_center.y += patchMap->max_b(patchID[b]);
p_b_center.z += patchMap->max_c(patchID[b]);
p_a_center *= (BigReal)0.5;
p_b_center *= (BigReal)0.5;
p_a_center = lattice.unscale(p_a_center);
p_b_center = lattice.unscale(p_b_center);
// Adjust patch a's center by the offset
p_a_center.x += params.offset.x;
p_a_center.y += params.offset.y;
p_a_center.z += params.offset.z;
// Calculate and fill in the projected line vector
params.projLineVec = p_b_center - p_a_center;
params.projLineVec /= params.projLineVec.length(); // Normalize the vector
#endif
params.p[0] = p[a];
params.p[1] = p[b];
params.pExt[0] = pExt[a];
params.pExt[1] = pExt[b];
// BEGIN LA
params.doLoweAndersen = patch[0]->flags.doLoweAndersen;
if (params.doLoweAndersen) {
DebugM(4, "opening velocity boxes\n");
v[0] = velocityBox[0]->open();
v[1] = velocityBox[1]->open();
params.v[0] = v[a];
params.v[1] = v[b];
}
// END LA
params.ff[0] = r[a]->f[Results::nbond];
params.ff[1] = r[b]->f[Results::nbond];
params.numAtoms[0] = numAtoms[a];
params.numAtoms[1] = numAtoms[b];
// DMK - Atom Separation (water vs. non-water)
#if NAMD_SeparateWaters != 0
params.numWaterAtoms[0] = numWaterAtoms[a];
params.numWaterAtoms[1] = numWaterAtoms[b];
#endif
/*******************************************************************************
* Call Nonbonded Functions
*******************************************************************************/
if (numAtoms[0] && numAtoms[1]) {//only do if has atoms since gbis noWork doesn't account for no atoms
//force calculation calls
if ( patch[0]->flags.doFullElectrostatics )
{
params.fullf[0] = r[a]->f[Results::slow];
params.fullf[1] = r[b]->f[Results::slow];
if ( patch[0]->flags.doMolly ) {
if ( doEnergy )
calcPairEnergy(¶ms);
else calcPair(¶ms);
CompAtom *p_avg[2];
p_avg[0] = avgPositionBox[0]->open();
p_avg[1] = avgPositionBox[1]->open();
params.p[0] = p_avg[a];
params.p[1] = p_avg[b];
if ( doEnergy ) calcSlowPairEnergy(¶ms);
else calcSlowPair(¶ms);
avgPositionBox[0]->close(&p_avg[0]);
avgPositionBox[1]->close(&p_avg[1]);
} else if ( patch[0]->flags.maxForceMerged == Results::slow ) {
if ( doEnergy ) calcMergePairEnergy(¶ms);
else calcMergePair(¶ms);
} else {
if ( doEnergy ) calcFullPairEnergy(¶ms);
else calcFullPair(¶ms);
}
}
else
if ( doEnergy ) calcPairEnergy(¶ms);
else calcPair(¶ms);
}//end if has atoms
// BEGIN LA
if (params.doLoweAndersen) {
DebugM(4, "closing velocity boxes\n");
velocityBox[0]->close(&v[0]);
velocityBox[1]->close(&v[1]);
}
// END LA
}// end not gbis
/*******************************************************************************
* gbis Loop
*******************************************************************************/
if (patch[0]->flags.doGBIS) {
SimParameters *simParams = Node::Object()->simParameters;
gbisParams.sequence = sequence();
gbisParams.doGBIS = patch[0]->flags.doGBIS;
gbisParams.numPatches = 2;//pair
gbisParams.gbisPhase = gbisPhase;
gbisParams.doFullElectrostatics = patch[0]->flags.doFullElectrostatics;
gbisParams.epsilon_s = simParams->solvent_dielectric;
gbisParams.epsilon_p = simParams->dielectric;
gbisParams.rho_0 = simParams->coulomb_radius_offset;
gbisParams.kappa = simParams->kappa;
gbisParams.cutoff = simParams->cutoff;
gbisParams.doSmoothing = simParams->switchingActive;
gbisParams.a_cut = simParams->alpha_cutoff;
gbisParams.delta = simParams->gbis_delta;
gbisParams.beta = simParams->gbis_beta;
gbisParams.gamma = simParams->gbis_gamma;
gbisParams.alpha_max = simParams->alpha_max;
gbisParams.cid = cid;
gbisParams.patchID[0] = patch[a]->getPatchID();
gbisParams.patchID[1] = patch[b]->getPatchID();
gbisParams.maxGroupRadius = patch[0]->flags.maxGroupRadius;
if (patch[1]->flags.maxGroupRadius > gbisParams.maxGroupRadius)
gbisParams.maxGroupRadius = patch[1]->flags.maxGroupRadius;
gbisParams.doEnergy = doEnergy;
gbisParams.fsMax = simParams->fsMax;
for (int i = 0; i < numGBISPairlists; i++)
gbisParams.gbisStepPairlists[i] = &gbisStepPairlists[i];
//open boxes
if (gbisPhase == 1) {
gbisParams.intRad[0] = intRadBox[a]->open();
gbisParams.intRad[1] = intRadBox[b]->open();
gbisParams.psiSum[0] = psiSumBox[a]->open();
gbisParams.psiSum[1] = psiSumBox[b]->open();
gbisParams.gbInterEnergy=0;
gbisParams.gbSelfEnergy=0;
} else if (gbisPhase == 2) {
gbisParams.bornRad[0] = bornRadBox[a]->open();
gbisParams.bornRad[1] = bornRadBox[b]->open();
gbisParams.dEdaSum[0] = dEdaSumBox[a]->open();
gbisParams.dEdaSum[1] = dEdaSumBox[b]->open();
} else if (gbisPhase == 3) {
gbisParams.dHdrPrefix[0] = dHdrPrefixBox[a]->open();
gbisParams.dHdrPrefix[1] = dHdrPrefixBox[b]->open();
}
//make call to calculate GBIS
if ( !ComputeNonbondedUtil::commOnly ) {
calcGBIS(¶ms,&gbisParams);
}
//close boxes
if (gbisPhase == 1) {
psiSumBox[0]->close(&(gbisParams.psiSum[a]));
psiSumBox[1]->close(&(gbisParams.psiSum[b]));
} else if (gbisPhase == 2) {
dEdaSumBox[0]->close(&(gbisParams.dEdaSum[a]));
dEdaSumBox[1]->close(&(gbisParams.dEdaSum[b]));
} else if (gbisPhase == 3) {
bornRadBox[0]->close(&(gbisParams.bornRad[a]));
bornRadBox[1]->close(&(gbisParams.bornRad[b]));
reduction->item(REDUCTION_ELECT_ENERGY) += gbisParams.gbInterEnergy;
reduction->item(REDUCTION_ELECT_ENERGY) += gbisParams.gbSelfEnergy;
intRadBox[0]->close(&(gbisParams.intRad[a]));
intRadBox[1]->close(&(gbisParams.intRad[b]));
dHdrPrefixBox[0]->close(&(gbisParams.dHdrPrefix[a]));
dHdrPrefixBox[1]->close(&(gbisParams.dHdrPrefix[b]));
}
}//end if doGBIS
if (!patch[0]->flags.doGBIS || gbisPhase == 3) {
submitReductionData(reductionData,reduction);
if (accelMDOn) submitReductionData(reductionData,amd_reduction);
if (pressureProfileOn)
submitPressureProfileData(pressureProfileData, pressureProfileReduction);
#ifdef TRACE_COMPUTE_OBJECTS
traceUserBracketEvent(TRACE_COMPOBJ_IDOFFSET+cid, traceObjStartTime, CmiWallTimer());
#endif
reduction->submit();
if (accelMDOn) amd_reduction->submit();
if (pressureProfileOn)
pressureProfileReduction->submit();
}//end gbis end phase
}//end do Force
/// Busy wait for busyWait
void POSE_busy_wait()
{
double start = CmiWallTimer();
while (CmiWallTimer() - start < busyWait) ;
}
/// Single forward execution step
void adapt5::Step()
{
Event *ev;
POSE_TimeType lastGVT = localPVT->getGVT();
POSE_TimeType maxTimeLeash, offset;
double critStart;
rbFlag = 0;
if (!parent->cancels.IsEmpty()) CancelUnexecutedEvents();
if (eq->RBevent) Rollback();
if (!parent->cancels.IsEmpty()) CancelEvents();
parent->Status();
/*
if (rbFlag) { // adjust leash according to rollback
timeLeash = avgRBoffset;
}
else if (timeLeash < theMaxLeash) { // adjust according to state
if (eq->currentPtr->timestamp > POSE_UnsetTS) { // adjust to next event
if (eq->currentPtr->timestamp - lastGVT > timeLeash)
timeLeash == eq->currentPtr->timestamp - lastGVT;
// else leave it alone
}
// no next event; leave it alone
}
// Put leash back into reasonable bounds
if (timeLeash > theMaxLeash) {
timeLeash = theMaxLeash;
}
else if (timeLeash < 0) timeLeash = 0;
*/
/*
int i = timeLeash >> 6;
if (i > 1) {
if (rbFlag) {
timeLeash -= i;
} else {
timeLeash += i;
}
} else {
if (rbFlag) {
timeLeash--;
} else {
timeLeash++;
}
}
if (timeLeash <= 0) {
timeLeash = 10;
}
*/
/*
if (rbFlag) {
timeLeash = timeLeash >> 1;
} else {
timeLeash++;
}
*/
// other possibilities:
// -run for a number of events with a fixed timeleash, then analyze
// and adjust
// -simply use recentAvgEventSparsity * avgEventsPerRB
// -add code that looks at the number of RBs this poser is responsible
// for and don't punish it if it doesn't have a lot
ev = eq->currentPtr;
if (parent->basicStats[1] > 0) {
avgEventsPerRB = (int)(parent->basicStats[0] / parent->basicStats[1]);
if (avgEventsPerRB < 1) {
avgEventsPerRB = 1;
}
}
// ======== attempt 1 ========
#if ALGORITHM_TO_USE == 1
if (rbFlag) {
recentAvgRBLeashCount++;
recentTotalRBLeash += timeLeash;
// initial rollback calculation to quickly set recentAvgRBLeash to a reasonable value
if (initialAvgRBLeashCalc) {
recentAvgRBLeash = avgRBoffset;
recentTotalRBLeash = 0;
recentAvgRBLeashCount = 0;
initialAvgRBLeashCalc = false;
}
// calculate the recent average timeleash when rollbacks occur
if (recentAvgRBLeashCount >= AVG_LEASH_CALC_PERIOD) {
recentAvgRBLeash = recentTotalRBLeash / recentAvgRBLeashCount;
recentTotalRBLeash = 0;
recentAvgRBLeashCount = 0;
}
if (timeLeash > recentAvgRBLeash) {
timeLeash = recentAvgRBLeash;
} else {
timeLeash = recentAvgRBLeash / 2;
}
} else {
timeLeash += recentAvgEventSparsity;
}
if (avgRBsPerGVTIter > MAX_RB_PER_GVT_ITER) {
maxTimeLeash = recentAvgRBLeash / 2;
} else {
maxTimeLeash = (POSE_TimeType)MAX_LEASH_MULTIPLIER * (POSE_TimeType)recentAvgEventSparsity * (POSE_TimeType)avgEventsPerRB;
}
if (maxTimeLeash > 50000) {
maxTimeLeash = 50000;
}
if (timeLeash > maxTimeLeash) {
timeLeash = maxTimeLeash;
}
if (timeLeash < 1) {
timeLeash = 1;
}
#endif
// ======== attempt 2 ========
#if ALGORITHM_TO_USE == 2
timeLeash = recentAvgEventSparsity * avgEventsPerRB;
if (timeLeash > 50000) {
timeLeash = 50000;
}
if (timeLeash < 1) {
timeLeash = 1;
}
#endif
// ======== attempt 3 ========
#if ALGORITHM_TO_USE == 3
timeLeash += recentAvgEventSparsity;
if (avgRBsPerGVTIter > MAX_RB_PER_GVT_ITER) {
maxTimeLeash = ((POSE_TimeType)MAX_LEASH_MULTIPLIER * (POSE_TimeType)recentAvgEventSparsity * (POSE_TimeType)avgEventsPerRB) / (4 * (POSE_TimeType)avgRBsPerGVTIter);
} else {
maxTimeLeash = 1000;
}
if (maxTimeLeash > 50000) {
maxTimeLeash = 50000;
}
if (timeLeash > maxTimeLeash) {
timeLeash = maxTimeLeash;
}
if (timeLeash < 1) {
timeLeash = 1;
}
#endif
// ======== attempt 4 ========
#if ALGORITHM_TO_USE == 4
if (timeLeash > 10000) {
timeLeash = 10000;
}
if (timeLeash < 1) {
timeLeash = 1;
}
#endif
/*
if (rbFlag) {
GVT *localGVT = (GVT *)CkLocalBranch(TheGVT);
int numRollbacks = parent->basicStats[1];
int numGVTIters = localGVT->gvtIterationCount;
if (userObj->myHandle == 32) {
CkPrintf("*** ROLLBACK: numRollbacks=%d numGVTIters=%d\n", numRollbacks, numGVTIters);
}
if ((numGVTIters > 0) && (((96 * numRollbacks) / numGVTIters) > 2)) {
timeLeash = avgRBoffset;
} else {
timeLeash++;
}
} else {
timeLeash++;
}
if (timeLeash > (avgRBoffset << 1)) {
timeLeash = avgRBoffset << 1;
}
*/
// can also just hard-code the time leash
// timeLeash = 1000;
// if (stepCalls == 0) {
// timeLeashTotal = 0LL;
// }
// if (timeLeash < 1000000) {
// stepCalls++;
// timeLeashTotal += timeLeash;
// }
/*
if (itersAllowed < 1) {
itersAllowed = 1;
}
else {
itersAllowed = (int)((double)specEventCount * specTol);
itersAllowed -= specEventCount - eventCount;
if (itersAllowed < 1) itersAllowed = 1;
}
*/
// Prepare to execute an event
offset = lastGVT + timeLeash;
// Shorten the leash as we near POSE_endtime
if ((POSE_endtime > POSE_UnsetTS) && ((offset > POSE_endtime) ||
(offset <= POSE_UnsetTS)))
offset = POSE_endtime;
while ((ev->timestamp > POSE_UnsetTS) && (ev->timestamp <= offset) ){
#ifdef MEM_COARSE
// Check to see if we should hold off on forward execution to save on
// memory.
// NOTE: to avoid deadlock, make sure we have executed something
// beyond current GVT before worrying about memory usage
if (((ev->timestamp > lastGVT) || (userObj->OVT() > lastGVT))
&& (eq->mem_usage > objUsage)) { // don't deadlock
break;
}
#endif
iter++;
currentEvent = ev;
ev->done = 2;
localPVT->incSpecEventCount();
localPVT->incEventCount();
specEventCount++;
eventCount++;
#ifndef CMK_OPTIMIZE
if(pose_config.trace)
critStart = CmiWallTimer(); // trace timing
#endif
parent->ResolveFn(ev->fnIdx, ev->msg); // execute it
#ifndef CMK_OPTIMIZE
if(pose_config.trace)
traceUserBracketEvent(10, critStart, CmiWallTimer());
#endif
ev->done = 1; // flag the event as executed
eq->mem_usage++;
eq->ShiftEvent(); // shift to next event
ev = eq->currentPtr;
}
#ifndef CMK_OPTIMIZE
if(pose_config.stats)
if (iter > 0) localStats->Loop();
#endif
}
void CmiSendMessagePxshm(char *msg, int size, int dstnode, int *refcount)
{
#if PXSHM_STATS
double _startSendTime = CmiWallTimer();
#endif
LrtsPrepareEnvelope(msg, size);
int dstRank = PxshmRank(dstnode);
MEMDEBUG(CmiMemoryCheck());
/*
MACHSTATE4(3,"Send Msg Pxshm ogm %p size %d dst %d dstRank %d",ogm,ogm->size,ogm->dst,dstRank);
MACHSTATE4(3,"Send Msg Pxshm ogm %p size %d dst %d dstRank %d",ogm,ogm->size,ogm->dst,dstRank);
*/
CmiAssert(dstRank >=0 && dstRank != pxshmContext->noderank);
sharedBufData *dstBuf = &(pxshmContext->sendBufs[dstRank]);
PxshmSendQ *sendQ = pxshmContext->sendQs[dstRank];
#if PXSHM_OSSPINLOCK
if(! OSSpinLockTry(&dstBuf->header->lock)){
#elif PXSHM_LOCK
if(sem_trywait(dstBuf->mutex) < 0){
#elif PXSHM_FENCE
dstBuf->header->flagSender = 1;
dstBuf->header->turn = RECEIVER;
CmiMemoryReadFence(0,0);
CmiMemoryWriteFence(0,0);
//if(dstBuf->header->flagReceiver && dstBuf->header->turn == RECEIVER){
if(dstBuf->header->flagReceiver){
dstBuf->header->flagSender = 0;
#endif
/**failed to get the lock
insert into q and retain the message*/
#if SENDQ_LIST
if (sendQ->numEntries == 0 && sendQ->next == -2) {
sendQ->next = sendQ_head_index;
sendQ_head_index = dstRank;
}
#endif
pushSendQ(pxshmContext->sendQs[dstRank], msg, size, refcount);
(*refcount)++;
MEMDEBUG(CmiMemoryCheck());
return;
}else{
/***
* We got the lock for this buffer
* first write all the messages in the sendQ and then write this guy
* */
if(pxshmContext->sendQs[dstRank]->numEntries == 0){
// send message user event
int ret = sendMessage(msg,size,refcount,dstBuf,pxshmContext->sendQs[dstRank]);
#if SENDQ_LIST
if (sendQ->numEntries > 0 && sendQ->next == -2)
{
sendQ->next = sendQ_head_index;
sendQ_head_index = dstRank;
}
#endif
MACHSTATE(3,"Pxshm Send succeeded immediately");
}else{
(*refcount)+=2;/*this message should not get deleted when the queue is flushed*/
pushSendQ(pxshmContext->sendQs[dstRank],msg,size,refcount);
// MACHSTATE3(3,"Pxshm ogm %p pushed to sendQ length %d refcount %d",ogm,pxshmContext->sendQs[dstRank]->numEntries,ogm->refcount);
int sent = flushSendQ(sendQ);
(*refcount)--; /*if it has been sent, can be deleted by caller, if not will be deleted when queue is flushed*/
MACHSTATE1(3,"Pxshm flushSendQ sent %d messages",sent);
}
/* unlock the recvbuffer*/
#if PXSHM_OSSPINLOCK
OSSpinLockUnlock(&dstBuf->header->lock);
#elif PXSHM_LOCK
sem_post(dstBuf->mutex);
#elif PXSHM_FENCE
CmiMemoryReadFence(0,0);
CmiMemoryWriteFence(0,0);
dstBuf->header->flagSender = 0;
#endif
}
#if PXSHM_STATS
pxshmContext->sendCount ++;
pxshmContext->sendTime += (CmiWallTimer()-_startSendTime);
#endif
MEMDEBUG(CmiMemoryCheck());
};
inline void emptyAllRecvBufs();
inline void flushAllSendQs();
/**********
* Extract all the messages from the recvBuffers you can
* Flush all sendQs
* ***/
inline void CommunicationServerPxshm(){
#if PXSHM_STATS
double _startCommServerTime =CmiWallTimer();
#endif
MEMDEBUG(CmiMemoryCheck());
emptyAllRecvBufs();
flushAllSendQs();
#if PXSHM_STATS
pxshmContext->commServerTime += (CmiWallTimer()-_startCommServerTime);
#endif
MEMDEBUG(CmiMemoryCheck());
};
static void CmiNotifyStillIdlePxshm(CmiIdleState *s){
CommunicationServerPxshm();
}
static void CmiNotifyBeginIdlePxshm(CmiIdleState *s)
{
CmiNotifyStillIdle(s);
}
// broadcast
void CkCheckpointMgr::Checkpoint(const char *dirname, CkCallback& cb){
chkptStartTimer = CmiWallTimer();
// every body make dir in case it is local directory
CmiMkdir(dirname);
if (CkMyPe() == 0) {
checkpointOne(dirname, cb);
}
char fileName[1024];
#ifndef CMK_CHARE_USE_PTR
// save groups into Chares.dat
sprintf(fileName,"%s/Chares_%d.dat",dirname,CkMyPe());
FILE* fChares = CmiFopen(fileName,"wb");
if(!fChares) CkAbort("Failed to create checkpoint file for chares!");
PUP::toDisk pChares(fChares);
CkPupChareData(pChares);
CmiFclose(fChares);
#endif
// save groups into Groups.dat
// content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
sprintf(fileName,"%s/Groups_%d.dat",dirname,CkMyPe());
FILE* fGroups = CmiFopen(fileName,"wb");
if(!fGroups) CkAbort("Failed to create checkpoint file for group table!");
PUP::toDisk pGroups(fGroups);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CkPupGroupData(pGroups,CmiTrue);
#else
CkPupGroupData(pGroups);
#endif
CmiFclose(fGroups);
// save nodegroups into NodeGroups.dat
// content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
if (CkMyRank() == 0) {
sprintf(fileName,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
FILE* fNodeGroups = CmiFopen(fileName,"wb");
if(!fNodeGroups)
CkAbort("Failed to create checkpoint file for nodegroup table!");
PUP::toDisk pNodeGroups(fNodeGroups);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CkPupNodeGroupData(pNodeGroups,CmiTrue);
#else
CkPupNodeGroupData(pNodeGroups);
#endif
CmiFclose(fNodeGroups);
}
//DEBCHK("[%d]CkCheckpointMgr::Checkpoint called dirname={%s}\n",CkMyPe(),dirname);
sprintf(fileName,"%s/arr_%d.dat",dirname, CkMyPe());
FILE *datFile=CmiFopen(fileName,"wb");
if (datFile==NULL) CkAbort("Could not create data file");
PUP::toDisk p(datFile);
CkPupArrayElementsData(p);
CmiFclose(datFile);
#if CMK_HAS_SYNC && ! CMK_DISABLE_SYNC
system("sync");
#endif
restartCB = cb;
DEBCHK("[%d]restartCB installed\n",CkMyPe());
CkCallback localcb(CkIndex_CkCheckpointMgr::SendRestartCB(NULL),0,thisgroup);
contribute(0,NULL,CkReduction::sum_int,localcb);
}
void CkCheckpointMgr::SendRestartCB(CkReductionMsg *m){
delete m;
DEBCHK("[%d]Sending out the cb\n",CkMyPe());
CkPrintf("Checkpoint to disk finished in %fs, sending out the cb...\n", CmiWallTimer() - chkptStartTimer);
restartCB.send();
}
// Each worker reports back to here when it completes an iteration
void report(int row, int col,int core,float mytime) {
recieve_count++;
totalTime[core] += mytime;
totalObjs[core]++;
double perIterTime = CkWallTimer() - perIterStartTime;
if (num_chares == recieve_count) {
if (++iterations == total_iterations) {
CkPrintf("Completed %d iterations; last iteration time: %.6lf\n", iterations, perIterTime);
totalIterTime += perIterTime;
//find maximum totalTIme
double totalProgramTime = (CmiWallTimer()-progStartTime);
//double timePerIter = totalProgramTime/iterations;
// calculate the max idle time
double maximum=0.0;
double minimum = totalTime[0];
int penum_min, penum_max;
for(int i=0;i<CkNumPes();i++){
if(totalTime[i] >= maximum){
maximum = totalTime[i];
penum_max = i;
}
if(totalTime[i] <= minimum){
minimum = totalTime[i];
penum_min = i;
}
}
double maxIdleTime = maximum - minimum;
double percentageIdleTime = (maxIdleTime/totalProgramTime)*100.00;
// calculate the average idle time
double totalidletime=0.0;
for(int i=0;i<CkNumPes();i++){
totalidletime += (maximum-totalTime[i]);
}
double averageidletime = totalidletime/ CkNumPes();
double percentageAvIdleTime = (averageidletime / totalProgramTime)*100;
CkPrintf("PE\tTotal Time\tobjs\n");
CkPrintf("-----------------------------------------------------\n");
for(int i=0;i<CkNumPes();i++)
{
CkPrintf("%d\t%lf\t%d\n",i,totalTime[i], totalObjs[i]);
}
CkPrintf("\n\n");
CkPrintf("Total Program Time:%.15lf \n", totalProgramTime);
CkPrintf("Total Iteration Time:%.15lf \n", totalIterTime);
CkPrintf("lbd Overhead Iteration Time:%.15lf \n", lbdOverhead);
CkPrintf("Max idle time:%.15lf \n", maxIdleTime);
CkPrintf("Average idle time:%.15lf\n", averageidletime);
CkPrintf("Percentage max idle time:%.15lf \n", percentageIdleTime);
CkPrintf("Percentage average idle time:%.15lf\n", percentageAvIdleTime);
CkPrintf("max pe (%d): min pe (%d)\n\n\n", penum_max, penum_min);
CkPrintf("%.15lf:%.15lf:%.15lf:%.15lf:%.15lf:%.15lf\n",
totalProgramTime,totalIterTime, maxIdleTime,averageidletime, percentageIdleTime,percentageAvIdleTime );
CkExit();
} else {
CkPrintf("starting new iteration; iteration %d time: %.6lf\n", iterations, perIterTime);
if(iterations == LDB_ITER + 1) {
lbdOverhead = perIterTime;
} else {
totalIterTime += perIterTime;
}
recieve_count=0;
perIterStartTime = CkWallTimer();
if(iterations != LDB_ITER) {
array.begin_iteration();
} else {
CkPrintf("Before atsync\n");
for(int i=0;i<CkNumPes();i++) {
CkPrintf("%d\t%lf\t%d\n",i,totalTime[i], totalObjs[i]);
}
}
}
}
}
int NamdState::configListInit(ConfigList *cfgList) {
configList = cfgList;
if (!configList->okay()) {
NAMD_die("Simulation config file is incomplete or contains errors.");
}
DebugM(1,"NamdState::configFileInit configList okay\n");
char *currentdir = 0;
simParameters = new SimParameters(configList,currentdir);
fflush(stdout);
lattice = simParameters->lattice;
//Check features that are not supported in the memory optimized
//version. --Chao Mei
#ifdef MEM_OPT_VERSION
checkMemOptCompatibility();
#endif
//Check rigidBonds type when generating the compressed psf files.
if(simParameters->genCompressedPsf) {
if(simParameters->rigidBonds == RIGID_NONE){
//default to RIGID_ALL
simParameters->rigidBonds = RIGID_ALL;
}
}
StringList *molInfoFilename;
// If it's AMBER force field, read the AMBER style files;
// if it's GROMACS, read the GROMACS files;
// Otherwise read the CHARMM style files
if (simParameters->amberOn)
{ StringList *parmFilename = configList->find("parmfile");
molInfoFilename = parmFilename;
StringList *coorFilename = configList->find("ambercoor");
// "amber" is a temporary data structure, which records all
// the data from the parm file. After copying them into
// molecule, parameter and pdb structures, it will be deleted.
Ambertoppar *amber;
amber = new Ambertoppar;
if (amber->readparm(parmFilename->data))
{ parameters = new Parameters(amber, simParameters->vdwscale14);
molecule = new Molecule(simParameters, parameters, amber);
if (coorFilename != NULL)
pdb = new PDB(coorFilename->data,amber);
delete amber;
}
else
NAMD_die("Failed to read AMBER parm file!");
parameters->print_param_summary();
}
else if (simParameters->gromacsOn) {
StringList *topFilename = configList->find("grotopfile");
molInfoFilename = topFilename;
StringList *coorFilename = configList->find("grocoorfile");
// "gromacsFile" is a temporary data structure, which records all
// the data from the topology file. After copying it into the
// molecule and parameter and pdb, it will be deleted.
GromacsTopFile *gromacsFile;
gromacsFile = new GromacsTopFile(topFilename->data);
parameters = new Parameters(gromacsFile,simParameters->minimizeCGOn);
if (coorFilename != NULL)
pdb = new PDB(coorFilename->data,gromacsFile);
molecule = new Molecule(simParameters, parameters, gromacsFile);
// XXX does Molecule(needAll,these,arguments)?
delete gromacsFile; // XXX unimplemented
// XXX add error handling when the file doesn't exist
// XXX make sure the right things happen when the parameters are
// not even specified.
// NAMD_die("Failed to read AMBER parm file!");
parameters->print_param_summary();
}
else if (simParameters->usePluginIO){
#ifdef MEM_OPT_VERSION
NAMD_die("Using plugin IO is not supported in memory optimized version!");
#else
PluginIOMgr *pIOMgr = new PluginIOMgr();
iout << iWARN << "Plugin-based I/O is still in development and may still have bugs\n" << endi;
molfile_plugin_t *pIOHandle = pIOMgr->getPlugin();
if (pIOHandle == NULL) {
NAMD_die("ERROR: Failed to match requested plugin type");
}
if ( pIOHandle->open_file_read == NULL )
NAMD_die("ERROR: Selected plugin type cannot open files");
if ( pIOHandle->read_structure == NULL )
NAMD_die("ERROR: Selected plugin type cannot read structures");
if ( pIOHandle->read_next_timestep == NULL )
NAMD_die("ERROR: Selected plugin type cannot read coordinates");
StringList *moleculeFilename = configList->find("structure");
molInfoFilename = moleculeFilename;
StringList *parameterFilename = configList->find("parameters");
//****** BEGIN CHARMM/XPLOR type changes
// For AMBER use different constructor based on parm_struct!!! -JCP
parameters = new Parameters(simParameters, parameterFilename);
parameters->print_param_summary();
int numAtoms = 0;
//TODO: not sure about the name field in the handler
void *plgFile = pIOHandle->open_file_read(moleculeFilename->data,
pIOHandle->name, &numAtoms);
if(plgFile == NULL) {
NAMD_die("ERROR: Opening structure file failed!");
}
double fileReadTime = CmiWallTimer();
molecule = new Molecule(simParameters, parameters, pIOHandle, plgFile, numAtoms);
iout << iINFO << "TIME FOR LOAD MOLECULE STRUCTURE INFORMATION: " << CmiWallTimer() - fileReadTime << "\n" << endi;
/* If we are only generating compressed molecule information, the PDB object is not needed */
if(!simParameters->genCompressedPsf) {
fileReadTime = CmiWallTimer();
//get the occupancy data from the Molecule object and then free it
//as it is stored in the Molecule object.
pdb = new PDB(pIOHandle, plgFile, molecule->numAtoms, molecule->getOccupancyData(), molecule->getBFactorData());
molecule->freeOccupancyData();
molecule->freeBFactorData();
iout << iINFO << "TIME FOR LOADING ATOMS' COORDINATES INFORMATION: " << CmiWallTimer() - fileReadTime << "\n" << endi;
}
pIOHandle->close_file_read(plgFile);
delete pIOMgr;
#endif
}
else
{
StringList *moleculeFilename = configList->find("structure");
molInfoFilename = moleculeFilename;
StringList *parameterFilename = configList->find("parameters");
//****** BEGIN CHARMM/XPLOR type changes
// For AMBER use different constructor based on parm_struct!!! -JCP
parameters = new Parameters(simParameters, parameterFilename);
//****** END CHARMM/XPLOR type changes
parameters->print_param_summary();
double fileReadTime = CmiWallTimer();
molecule = new Molecule(simParameters, parameters, moleculeFilename->data, configList);
iout << iINFO << "TIME FOR READING PSF FILE: " << CmiWallTimer() - fileReadTime << "\n" << endi;
}
fflush(stdout);
#ifdef MEM_OPT_VERSION
//upon knowing the number of atoms, it's good time to estimate the number of
//input/output processors if their value is not set
if(simParameters->numinputprocs==0){
int numatoms = molecule->numAtoms;
long estval = (sizeof(InputAtom)+2*sizeof(int)+1)*((long)(numatoms));
int numprocs = estval>>26; //considering every input proc consumes about 64M.
if(numprocs==0){
numprocs=1;
}else if(numprocs>CkNumPes()){
numprocs=CkNumPes();
}
simParameters->numinputprocs=numprocs;
}
/* Use fast, accurate charm clock */
double msg_timer(void) {
return CmiWallTimer();
}
/**
threadInfo methods
*/
void commThreadInfo::run()
{
CpvAccess(CthResumeBigSimThreadIdx) = BgRegisterHandler((BgHandler)CthResumeNormalThread);
tSTARTTIME = CmiWallTimer();
if (!tSTARTED) {
tSTARTED = 1;
// InitHandlerTable();
BgNodeStart(BgGetArgc(), BgGetArgv());
/* bnv should be initialized */
}
threadQueue *commQ = myNode->commThQ;
//int recvd=0; //for debugging only
for (;;) {
char *msg = getFullBuffer();
if (!msg) {
// tCURRTIME += (CmiWallTimer()-tSTARTTIME);
commQ->enq(CthSelf());
DEBUGF(("[%d] comm thread suspend.\n", BgMyNode()));
CthSuspend();
DEBUGF(("[%d] comm thread assume.\n", BgMyNode()));
// tSTARTTIME = CmiWallTimer();
continue;
}
DEBUGF(("[%d] comm thread has a msg.\n", BgMyNode()));
//printf("on node %d, comm thread process a msg %p with type %d\n", BgMyNode(), msg, CmiBgMsgType(msg));
/* schedule a worker thread, if small work do it itself */
if (CmiBgMsgType(msg) == SMALL_WORK) {
if (CmiBgMsgRecvTime(msg) > tCURRTIME) tCURRTIME = CmiBgMsgRecvTime(msg);
// tSTARTTIME = CmiWallTimer();
/* call user registered handler function */
BgProcessMessage(this, msg);
}
else {
#if BIGSIM_TIMING
correctMsgTime(msg);
#endif
//recvd++;
//DEBUGM(4, ("[N%d] C[%d] will add a msg (handler=%d | cnt=%d", BgMyNode(), id, CmiBgMsgHandle(msg), recvd));
int msgLen = CmiBgMsgLength(msg);
DEBUGM(4, (" | len: %d | type: %d | node id: %d | src pe: %d\n" , msgLen, CmiBgMsgType(msg), CmiBgMsgNodeID(msg), CmiBgMsgSrcPe(msg)));
if (CmiBgMsgThreadID(msg) == ANYTHREAD) {
DEBUGF(("anythread, call addBgNodeMessage\n"));
addBgNodeMessage(msg); /* non-affinity message */
DEBUGM(4, ("The message is added to node\n\n"));
}
else {
DEBUGF(("[N%d] affinity msg, call addBgThreadMessage to tID:%d\n",
BgMyNode(), CmiBgMsgThreadID(msg)));
addBgThreadMessage(msg, CmiBgMsgThreadID(msg));
DEBUGM(4, ("The message is added to thread(%d)\n\n", CmiBgMsgThreadID(msg)));
}
}
/* let other communication thread do their jobs */
// tCURRTIME += (CmiWallTimer()-tSTARTTIME);
if (!schedule_flag) CthYield();
tSTARTTIME = CmiWallTimer();
}
}
void start() {
startTime = CmiWallTimer();
array.doStep();
}
TempAwareRefineLB::TempAwareRefineLB(const CkLBOptions &opt): CentralLB(opt)
{
#ifdef TEMP_LDB
starting=CmiWallTimer();
// procsPerNode=4;
migFile=fopen("migInfo","w");
numAvailFreqs = 11;
//numAvailFreqs = 14;
//numAvailFreqs = 7;
freqs=new int[numAvailFreqs];
freqsEffect=new int[numAvailFreqs];
// for might (lab machine)
/*
freqs[0] = 2262000;
freqs[1] = 2261000;
freqs[2] = 2128000;
freqs[3] = 1995000;
freqs[4] = 1862000;
freqs[5] = 1729000;
freqs[6] = 1596000;
*/
// for tarekc cluster
freqs[0] = 2395000;
freqs[1] = 2394000;
freqs[2] = 2261000;
freqs[3] = 2128000;
freqs[4] = 1995000;
freqs[5] = 1862000;
freqs[6] = 1729000;
freqs[7] = 1596000;
freqs[8] = 1463000;
freqs[9] = 1330000;
freqs[10] = 1197000;
freqsEffect[0] = 1979886;
freqsEffect[1] = 1943017;
freqsEffect[2] = 1910989;
freqsEffect[3] = 1876619;
freqsEffect[4] = 1824126;
freqsEffect[5] = 1763990;
freqsEffect[6] = 1666773;
freqsEffect[7] = 1560224;
freqsEffect[8] = 1443154;
freqsEffect[9] = 1317009;
freqsEffect[10] = 1200000;
/*
// for grace, humility etc (lab i7 machines)
freqs[0] = 2801000;
freqs[1] = 2800000;
freqs[2] = 2667000;
freqs[3] = 2533000;
freqs[4] = 2400000;
freqs[5] = 2267000;
freqs[6] = 2133000;
freqs[7] = 2000000;
freqs[8] = 1867000;
freqs[9] = 1733000;
freqs[10] = 1600000;
freqs[11] = 1467000;
freqs[12] = 1333000;
freqs[13] = 1200000;
*/
procFreqPtr = new int[CkNumPes()];
for(int i=0;i<CkNumPes();i++)
{
char newfreq[10];
sprintf(newfreq,"%d",freqs[0]);
cpufreq_sysfs_write(newfreq,i%physicalCoresPerNode);
procFreqPtr[i]=0;
}
// logicalCoresPerChip=4;
procFreq=NULL;
procTemp=NULL;
procFreqNew=NULL;
procFreqNewEffect = NULL;
avgChipTemp=NULL;
lbname = "TempAwareRefineLB";
if (CkMyPe()==0)
CkPrintf("[%d] TempAwareRefineLB created\n",CkMyPe());
char logFile[100];
snprintf(logFile, sizeof(logFile), "temp_freq.log.%d", CkMyPe());
if ((logFD = fopen(logFile, "a"))) {
fprintf(logFD, "Time, PE, Temperature, Frequency\n");
} else {
CkAbort("Couldn't open temperature/frequency log file");
}
CcdCallOnConditionKeep(CcdPERIODIC_1second, &printCurrentTemperature, this);
#else
CmiAbort("TEMPLB ERROR: not supported without TEMP_LDB flag.\n");
#endif
}
void startTraceBigSim(){
CkAssert(CkpvAccess(insideTraceBracket) == false);
CkpvAccess(insideTraceBracket) = true;
#if SPLIT_APART_CYCLE_ACCURATE
SimParameters *simParams = Node::Object()->simParameters;
if(simParams->bgSplitNumProcs != -1 && simParams->bgSplitMyProc!=-1){
(bgTraceCounter) ++;
if( ((bgTraceCounter) % simParams->bgSplitNumProcs) == simParams->bgSplitMyProc){
// Do slow mambo simulation for this case!
//CkPrintf("TRACEBIGSIM: Doing cycle accurate simulation for interesting event #%lu\n", (bgTraceCounter) );
start_time = begin(); // for MAMBO
}
}
#endif
#ifdef BIGSIM_PAPI
for(int i=0;i<NUM_PAPI_EVENTS;i++)
values[i] = 0;
events[0] = PAPI_FP_OPS;
events[1] = PAPI_TOT_INS;
events[2] = PAPI_L1_ICM;
events[3] = PAPI_L2_TCM;
events[4] = PAPI_L3_TCM;
events[5] = PAPI_TLB_TL;
events[6] = PAPI_LD_INS;
events[7] = PAPI_SR_INS; // store instructions
events[8] = PAPI_RES_STL; // resource stalls
/* Other available events:
PAPI_BR_INS,
PAPI_BR_MSP,
PAPI_FP_INS,
PAPI_TOT_INS,
PAPI_TOT_IIS,
PAPI_L1_DCM,
PAPI_L1_LDM,
PAPI_L2_TCM,
PAPI_L3_LDM,
PAPI_RES_STL,
PAPI_LD_INS,
PAPI_TLB_TL
*/
CkAssert(PAPI_start_counters(events, NUM_PAPI_EVENTS) == PAPI_OK);
#endif
#ifdef CMK_BIGSIM_CHARM
BgMark("startTraceBigSim %f\n");
#endif
#if WITH_MAMBO
// startTime = CmiWallTimer();
start_time = begin(); // for MAMBO
#else
CkpvAccess(start_time) = CmiWallTimer();
#endif
}
void TempAwareRefineLB::work(LDStats* stats)
{
#ifdef TEMP_LDB
////////////////////////////////////////////////////
numProcs=stats->nprocs();
numChips=numProcs/logicalCoresPerChip;
avgChipTemp=new float[numChips];
if(procFreq!=NULL) delete [] procFreq;
if(procFreqEffect!=NULL) delete [] procFreqEffect;
// if(procFreqPtr!=NULL) delete [] procFreqPtr;
if(procTemp!=NULL) delete [] procTemp;
if(procFreqNew!=NULL) delete [] procFreqNew;
if(procFreqNewEffect!=NULL) delete [] procFreqNewEffect;
if(avgChipTemp!=NULL) delete [] avgChipTemp;
procFreq = new int[numProcs];
procFreqEffect = new int[numProcs];
// procFreqPtr = new int[numProcs];
procTemp = new float[numProcs];
procFreqNew = new int[numProcs];
procFreqNewEffect = new int[numProcs];
avgChipTemp = new float[numChips];
for(int i=0;i<numChips;i++) avgChipTemp[i]=0;
for(int i=0;i<numProcs;i++)
{
procFreq[i] = stats->procs[i].pe_speed;
procTemp[i] = stats->procs[i].pe_temp;
// procFreqPtr[i] = getProcFreqPtr(freqs,numAvailFreqs,procFreq[i]);
avgChipTemp[i/logicalCoresPerChip] += procTemp[i];
}
for(int i=0;i<numChips;i++)
{
avgChipTemp[i]/=logicalCoresPerChip;
//CkPrintf("---- CHIP#%d has temp=%f ----------\n",i,avgChipTemp[i]);
}
for(int i=0;i<numChips;i++)
{
int over=0,under=0;
if(avgChipTemp[i] > MAX_TEMP)
{
over=1;
if(procFreqPtr[i*logicalCoresPerChip]==numAvailFreqs-1)
{
for(int j=i*logicalCoresPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++) procFreqNew[j] = freqs[procFreqPtr[j]];
CkPrintf("CHIP#%d RUNNING HOT EVEN WITH MIN FREQUENCY!!\n",i);
}
else
{
for(int j=i*logicalCoresPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++)
{
if(procFreqPtr[j]<numAvailFreqs-1) procFreqPtr[j]++;
#ifdef MAX_MIN
/// PLEASE COMMENT OUT .. TESTING ONLY
if(i==0) {procFreqPtr[j] = numAvailFreqs-1;/*CkPrintf("C for i:%d\n",j);*/}
//if(i<numChips-1) procFreqPtr[j]=0;
else procFreqPtr[j]=0;
/////////////////////////
#endif
procFreqNew[j] = freqs[procFreqPtr[j]];
}
#ifndef ORG_VERSION
CkPrintf("!!!!! Chip#%d running HOT shifting from %d to %d temp=%f\n",i,procFreq[i*logicalCoresPerChip],procFreqNew[i*logicalCoresPerChip],avgChipTemp[i]);
#endif
}
}
else
// if(avgChipTemp[i] < MAX_TEMP-1)
{
under=1;
if(procFreqPtr[i*logicalCoresPerChip]>0)
{
for(int j=i*logicalCoresPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++)
{
if(procFreqPtr[j]>0)
procFreqPtr[j]--;
#ifdef MAX_MIN
/// PLEASE COMMENT OUT .. TESTING ONLY
if(i==0) procFreqPtr[j] = numAvailFreqs-1;
//if(i<numChips-1) procFreqPtr[j]=0;
else procFreqPtr[j]=0;
/////////////////////////
#endif
procFreqNew[j] = freqs[procFreqPtr[j]];
}
#ifndef ORG_VERSION
CkPrintf("!!!!! Chip#%d running COLD shifting from %d to %d temp=%f\n",i,procFreq[i*logicalCoresPerChip],procFreqNew[i*logicalCoresPerChip],avgChipTemp[i]);
#endif
}
else
{
for(int j=i*logicalCoresPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++) procFreqNew[j] = freqs[procFreqPtr[j]];
}
}
/*
if(under==0 && over==0)
{
for(int j=i*logicalCoresPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++) procFreqNew[j] = freqs[procFreqPtr[j]];
}
*/
//if(i==5) for(int j=i*c(resPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++) procFreqNew[j] = freqs[numAvailFreqs-1];
//else
#ifdef ORG_VERSION
for(int j=i*logicalCoresPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++) procFreqNew[j] = freqs[0];
#endif
//for(int j=i*logicalCoresPerChip;j<i*logicalCoresPerChip+logicalCoresPerChip;j++) procFreqNew[j] = freqs[0];
}
//for(int x=0;x<numProcs;x+=logicalCoresPerChip) if(procFreq[x]!=procFreqNew[x]) thisProxy[x].changeFreq(procFreqNew[x]);
//for(int x=0;x<numProcs;x++) CkPrintf("Procs#%d freq %d\n",x,procFreqNew[x]);
////////////////////////////////////////////////////
#ifndef NO_TEMP_LB
int obj;
int n_pes = stats->nprocs();
// CkPrintf("[%d] RefineLB strategy\n",CkMyPe());
// RemoveNonMigratable(stats, n_pes);
// get original object mapping
int* from_procs = RefinerTemp::AllocProcs(n_pes, stats);
for(obj=0;obj<stats->n_objs;obj++) {
int pe = stats->from_proc[obj];
from_procs[obj] = pe;
}
// Get a new buffer to refine into
populateEffectiveFreq(numProcs);
int* to_procs = RefinerTemp::AllocProcs(n_pes, stats);
// RefinerTemp refiner(1.03,procFreqEffect,procFreqNewEffect,n_pes); // overload tolerance=1.05
RefinerTemp refiner(1.03,procFreq,procFreqNew,n_pes);
refiner.Refine(n_pes, stats, from_procs, to_procs);
// Save output
int migs=0;
int *numMigs = new int[numProcs];
int totE = 0;
for(int mm=0;mm<numProcs;mm++) numMigs[mm] = 0;
for(obj=0;obj<stats->n_objs;obj++) {
int pe = stats->from_proc[obj];
numMigs[to_procs[obj]]++;
//stats->objData[obj].objID();
LDObjData &odata = stats->objData[obj];
computeInfo *c1 = new computeInfo();
c1->id = odata.objID();
//if(to_procs[obj]==3) CkPrintf("[%d,%d] going to 3 totE:%d\n",c1->id.getID()[0],c1->id.getID()[1],totE++);//,(stats->objData[obj].objID().getID())[1],totE++);
if (to_procs[obj] != pe) {
migs++;
//if (_lb_args.debug()>=2)
{
// CkPrintf("[%d,%d] Obj %d migrating from %d to %d\n",
// c1->id.getID()[0],c1->id.getID()[1],obj,pe,to_procs[obj]);
}
stats->to_proc[obj] = to_procs[obj];
}
}
for(int mm=0;mm<numProcs;mm++)
{
//CkPrintf("PROC#%d freq:%d objs:%d ----------\n",mm,procFreqNew[mm],numMigs[mm]);
}
CkPrintf("TEMPLB INFO: Total Objs:%d migrations:%d time:%f \n",stats->n_objs,migs,CmiWallTimer()-starting);
fprintf(migFile,"%f %d\n",CmiWallTimer()-starting,migs);
// Free the refine buffers
RefinerTemp::FreeProcs(from_procs);
RefinerTemp::FreeProcs(to_procs);
#endif
//for(int x=0;x<numProcs;x++) CkPrintf("Procs#%d ------- freq %d\n",x,procFreqNew[x]);
/*
for(int x=0;x<numProcs;x+=logicalCoresPerChip)
{
if(procFreq[x]!=procFreqNew[x])
{
CkPrintf("Chaning the freq for PROC#%d\n",x);
thisProxy[x].changeFreq(procFreqNew[x]);
}
}
*/
for(int x=0;x<numProcs;x++)
{
//CkPrintf("--------- Proc#%d %d numProcs=%d\n",x,procFreqNew[x],numProcs);
if(procFreq[x]!=procFreqNew[x]) thisProxy[x].changeFreq(procFreqNew[x]);
}
#endif // TEMP_LDB endif
}
void endTraceBigSim_20param(char * eventname, int stepno, int num_params, double p1 , double p2 , double p3 , double p4 , double p5 , double p6 , double p7 , double p8 , double p9 , double p10 , double p11 , double p12 , double p13 , double p14 , double p15 , double p16 , double p17 , double p18 , double p19 , double p20 ) {
#if WITH_MAMBO
end_time=end();
// double endTime = CmiWallTimer();
#else
CkpvAccess(end_time) = CmiWallTimer();
#endif
CkAssert(CkpvAccess(insideTraceBracket) == true);
CkpvAccess(insideTraceBracket) = false;
#ifdef CMK_BIGSIM_CHARM
char perfCountString[1024];
perfCountString[0] = 0;
#endif
char params[2048];
if(num_params==0) sprintf(params, "");
if(num_params==1) sprintf(params, "%f", p1);
if(num_params==2) sprintf(params, "%f %f", p1, p2);
if(num_params==3) sprintf(params, "%f %f %f", p1, p2, p3);
if(num_params==4) sprintf(params, "%f %f %f %f", p1, p2, p3, p4);
if(num_params==5) sprintf(params, "%f %f %f %f %f", p1, p2, p3, p4, p5);
if(num_params==6) sprintf(params, "%f %f %f %f %f %f", p1, p2, p3, p4, p5, p6);
if(num_params==7) sprintf(params, "%f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7);
if(num_params==8) sprintf(params, "%f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8);
if(num_params==9) sprintf(params, "%f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9);
if(num_params==10) sprintf(params, "%f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10);
if(num_params==11) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11);
if(num_params==12) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12);
if(num_params==13) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13);
if(num_params==14) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14);
if(num_params==15) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15);
if(num_params==16) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16);
if(num_params==17) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17);
if(num_params==18) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18);
if(num_params==19) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19);
if(num_params==20) sprintf(params, "%f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f", p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20);
char paramString[2048];
sprintf(paramString, "params:{ %s }", params);
char eventNameString[1024];
sprintf(eventNameString, "event:{ %s }", eventname);
#ifdef BIGSIM_PAPI
CkAssert(PAPI_stop_counters(values, NUM_PAPI_EVENTS) == PAPI_OK);
sprintf(perfCountString, " PAPI:{ " );
for(int i=0;i<NUM_PAPI_EVENTS;i++){
sprintf(perfCountString+strlen(perfCountString), " %lld ", values[i] );
}
printf("value=%lld\n", values[0]);
sprintf(perfCountString+strlen(perfCountString), " }");
#endif
char timeString[512];
timeString[0] = 0;
char stepString[128];
stepString[0] = 0;
sprintf(stepString, "step:{ %d }", stepno);
#if ! CMK_BIGSIM_CHARM
#if WITH_MAMBO
// sprintf(timeString, "time:{ %f }", endTime-startTime);
sprintf(timeString, "time_in_cycles:{ %llu }", end_time-start_time);
#endif
#endif
if (CkpvAccess(bgfp) == NULL) {
if (CkpvAccess(outputParameters)) {
double t = CkpvAccess(end_time)-CkpvAccess(start_time);
if (t<0.0) {
CmiPrintf("time: %f\n", t);
t = 0.0;
}
CmiAssert(t >= 0.0);
sprintf(timeString, "time_in_us:{ %lf } %s %s %s\n", t*1e6, eventNameString, stepString, paramString);
CkpvAccess(eventsPool).insert(timeString);
}
}
#if SPLIT_APART_CYCLE_ACCURATE
SimParameters *simParams = Node::Object()->simParameters;
if(simParams->bgSplitNumProcs != -1 && simParams->bgSplitMyProc!=-1){
if( ((bgTraceCounter) % simParams->bgSplitNumProcs) == simParams->bgSplitMyProc){
// Do slow mambo simulation for this case!
// Counter is incremented only in startTraceBigSim()
}
}
#endif
#ifdef CMK_BIGSIM_CHARM
char sequenceString[128];
sequenceString[0] = 0;
BgMark("endTraceBigSim %f\n");
if (CkpvAccess(bgfp) != NULL) {
// write event ID
int seqno = tTIMELINEREC.length()-1;
if (seqno<0) CkAbort("Traces are not generated. Please run emulation with +bglog");
fprintf(CkpvAccess(bgfp),"%d ",seqno);
sprintf(sequenceString, "seqno:{ %d } ",seqno);
// fprintf(CkpvAccess(bgfp),"%s\n",params);
fprintf(CkpvAccess(bgfp), "TRACEBIGSIM: %s %s %s %s %s %s\n", eventNameString, stepString, sequenceString, timeString, perfCountString, paramString);
}
#else
/*
// printf("TRACEBIGSIM: %s %s %s %s %s\n", eventNameString, sequenceString, timeString, perfCountString, paramString);
if (CkpvAccess(bgfp) != NULL) {
fprintf(CkpvAccess(bgfp), "TRACEBIGSIM: %s %s %s %s %s\n", eventNameString, sequenceString, timeString, perfCountString, paramString);
}
*/
#endif
}
extern "C" void
init(void)
{
CkPrintf("init started\n");
double startTime=CmiWallTimer();
const char *eleName="mesh1.tet";//"adpmm/xxx.1.ele";//"88mesh/mesh1.tri";
const char *nodeName="mesh1.node";//"adpmm/xxx.1.node";//"88mesh/mesh1.node";
int nPts=0; //Number of nodes
vector3d *pts=0; //Node coordinates
int *bounds;
CkPrintf("Reading node coordinates from %s\n",nodeName);
//Open and read the node coordinate file
{
char line[1024];
FILE *f=fopen(nodeName,"r");
if (f==NULL) die("Can't open node file!");
fgets(line,1024,f);
if (1!=sscanf(line,"%d",&nPts)) die("Can't read number of points!");
pts=new vector3d[nPts];
bounds = new int[nPts];
for (int i=0;i<nPts;i++) {
int ptNo;
if (NULL==fgets(line,1024,f)) die("Can't read node input line!");
if (5!=sscanf(line,"%d%lf%lf%lf%d",&ptNo,&pts[i].x,&pts[i].y,&pts[i].z,&bounds[i]))
die("Can't parse node input line!");
}
fclose(f);
}
int nEle=0;
connRec *ele=NULL;
CkPrintf("Reading elements from %s\n",eleName);
//Open and read the element connectivity file
{
char line[10240];
FILE *f=fopen(eleName,"r");
if (f==NULL) die("Can't open element file!");
fgets(line,1024,f);
if (1!=sscanf(line,"%d",&nEle)) die("Can't read number of elements!");
ele=new connRec[nEle];
for (int i=0;i<nEle;i++) {
int elNo;
if (NULL==fgets(line,10240,f)) die("Can't read element input line!");
if (5!=sscanf(line,"%d%d%d%d%d",&elNo,&ele[i][0],&ele[i][1],&ele[i][2],&ele[i][3]))
die("Can't parse element input line!");
//ele[i][0]--;
//ele[i][1]--;
//ele[i][2]--;
//ele[i][3]--;
}
fclose(f);
}
int fem_mesh=FEM_Mesh_default_write(); // Tell framework we are writing to the mesh
CkPrintf("Passing node coords to framework\n");
FEM_Mesh_data(fem_mesh, // Add nodes to the current mesh
FEM_NODE, // We are registering nodes
FEM_DATA+0, // Register the point locations which are normally
// the first data elements for an FEM_NODE
(double *)pts, // The array of point locations
0, // 0 based indexing
nPts, // The number of points
FEM_DOUBLE, // Coordinates are doubles
3); // Points have dimension 3 (x,y,z)
CkPrintf("Passing node bounds to framework\n");
FEM_Mesh_data(fem_mesh, // Add nodes to the current mesh
FEM_NODE, // We are registering nodes
FEM_BOUNDARY, // Register the point bound info
// the first data elements for an FEM_NODE
(int *)bounds, // The array of point bound info
0, // 0 based indexing
nPts, // The number of points
FEM_INT, // bounds are ints
1); // Points have dimension 1
CkPrintf("Passing elements to framework\n");
FEM_Mesh_data(fem_mesh, // Add nodes to the current mesh
FEM_ELEM+0, // We are registering elements with type 0
// The next type of element could be registered with FEM_ELEM+1
FEM_CONN, // Register the connectivity table for this
// data elements for this type of FEM entity
(int *)ele, // The array of point locations
0, // 0 based indexing
nEle, // The number of elements
FEM_INDEX_0, // We use zero based node numbering
4); // Elements have degree 3, since triangles are defined
// by three nodes
// Register values to the elements so we can keep track of them after partitioning
double values[10240];
for(int i=0;i<10240;i++)values[i]=i;
// tets
FEM_Mesh_data(fem_mesh, // Add nodes to the current mesh
FEM_ELEM, // We are registering elements with type 1
FEM_DATA,
(int *)values, // The array of point locations
0, // 0 based indexing
nEle, // The number of elements
FEM_DOUBLE, // We use zero based node numbering
1);
//boundary conditions
FEM_Mesh_data(fem_mesh, // Add nodes to the current mesh
FEM_NODE, // We are registering elements with type 1
FEM_BOUNDARY,
(int *)bounds, // The array of point locations
0, // 0 based indexing
nPts, // The number of elements
FEM_INT, // We use zero based node numbering
1);
delete[] ele;
delete[] pts;
delete[] bounds;
double *sizingData = new double[nEle];
for (int i=0; i<nEle; i++) sizingData[i]=-1.0;
FEM_Mesh_data(fem_mesh, FEM_ELEM+0, FEM_MESH_SIZING, sizingData, 0, nEle,
FEM_DOUBLE, 1);
delete [] sizingData;
// add ghost layers
const int triangleFaces[4] = {0,1,2,3};
CkPrintf("Adding Ghost layers\n");
FEM_Add_ghost_layer(1,1);
FEM_Add_ghost_elem(0,4,triangleFaces);
CkPrintf("Finished with init (Reading took %.f s)\n",CmiWallTimer()-startTime);
}
void POSE_startTimer() {
CkPrintf("Starting simulation...\n");
sim_timer = CmiWallTimer();
}
/**
* This handler is used only during initialization. It receives messages from
* processor zero regarding Readonly Data (in one single message), Readonly Messages,
* Groups, and Nodegroups.
* The Readonly Data message also contains the total number of messages expected
* during the initialization phase.
* For Groups and Nodegroups, only messages with epoch=0 (meaning created from within
* a mainchare) are buffered for special creation, the other messages are buffered
* together with all the other regular messages by _bufferHandler (and will be flushed
* after all the initialization messages have been processed).
*/
static void _initHandler(void *msg, CkCoreState *ck)
{
CkAssert(CkMyPe()!=0);
register envelope *env = (envelope *) msg;
if (ck->watcher!=NULL) {
if (!ck->watcher->processMessage(&env,ck)) return;
}
switch (env->getMsgtype()) {
case BocInitMsg:
if (env->getGroupEpoch()==0) {
CkpvAccess(_numInitsRecd)++;
// _processBocInitMsg already handles QD
//CpvAccess(_qd)->process();
CkpvAccess(_bocInitVec)->insert(env->getGroupNum().idx, env);
} else _bufferHandler(msg);
break;
case NodeBocInitMsg:
if (env->getGroupEpoch()==0) {
CmiImmediateLock(CksvAccess(_nodeGroupTableImmLock));
CksvAccess(_numInitNodeMsgs)++;
CksvAccess(_nodeBocInitVec)->insert(env->getGroupNum().idx, env);
CmiImmediateUnlock(CksvAccess(_nodeGroupTableImmLock));
CpvAccess(_qd)->process();
} else _bufferHandler(msg);
break;
case ROMsgMsg:
CkpvAccess(_numInitsRecd)++;
CpvAccess(_qd)->process();
if(env->isPacked()) CkUnpackMessage(&env);
_processROMsgMsg(env);
break;
case RODataMsg:
CkpvAccess(_numInitsRecd)++;
CpvAccess(_qd)->process();
_numExpectInitMsgs = env->getCount();
_processRODataMsg(env);
break;
default:
CmiAbort("Internal Error: Unknown-msg-type. Contact Developers.\n");
}
DEBUGF(("[%d,%.6lf] _numExpectInitMsgs %d CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs) %d+%d\n",CmiMyPe(),CmiWallTimer(),_numExpectInitMsgs,CkpvAccess(_numInitsRecd),CksvAccess(_numInitNodeMsgs)));
if(_numExpectInitMsgs&&(CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs)==_numExpectInitMsgs)) {
_initDone();
}
}
TempAwareCommLB::TempAwareCommLB(const CkLBOptions &opt): CBase_TempAwareCommLB(opt)
{
#ifdef TEMP_LDB
lbname = "TempAwareCommLB";
if (CkMyPe()==0)
CkPrintf("[%d] TempAwareCommLB created\n",CkMyPe());
starting=CmiWallTimer();
migFile=fopen("migInfo","w");
numAvailFreqs = 11;
freqs=new int[numAvailFreqs];
freqsEffect=new int[numAvailFreqs];
// for tarekc cluster
freqs[0] = 2395000;
freqs[1] = 2394000;
freqs[2] = 2261000;
freqs[3] = 2128000;
freqs[4] = 1995000;
freqs[5] = 1862000;
freqs[6] = 1729000;
freqs[7] = 1596000;
freqs[8] = 1463000;
freqs[9] = 1330000;
freqs[10] = 1197000;
freqsEffect[0] = 1979886;
freqsEffect[1] = 1943017;
freqsEffect[2] = 1910989;
freqsEffect[3] = 1876619;
freqsEffect[4] = 1824126;
freqsEffect[5] = 1763990;
freqsEffect[6] = 1666773;
freqsEffect[7] = 1560224;
freqsEffect[8] = 1443154;
freqsEffect[9] = 1317009;
freqsEffect[10] = 1200000;
procFreqPtr = new int[CkNumPes()];
for(int i=0;i<CkNumPes();i++)
{
char newfreq[10];
sprintf(newfreq,"%d",freqs[0]);
cpufreq_sysfs_write(newfreq,i%physicalCoresPerNode);
procFreqPtr[i]=0;
}
procFreq=NULL;
procTemp=NULL;
procFreqNew=NULL;
procFreqNewEffect = NULL;
avgChipTemp=NULL;
char logFile[100];
snprintf(logFile, sizeof(logFile), "temp_freq.log.%d", CkMyPe());
if ((logFD = fopen(logFile, "a"))) {
fprintf(logFD, "Time, PE, Temperature, Frequency\n");
} else {
CkAbort("Couldn't open temperature/frequency log file");
}
CcdCallOnConditionKeep(CcdPERIODIC_1second, &printCurrentTemperature, this);
#else
#endif
}
