// Perform one iteration of work
// The first step is to send the local state to the neighbors
void begin_iteration(void) {
itercnt++;
//ckout << "iter=" << itercnt << ":(" << thisIndex.x << "," << thisIndex.y << ") is on " << CkMyPe() << endl;
// Copy left column and right column into temporary arrays
double *left_edge = new double[block_height];
double *right_edge = new double[block_height];
for(int i=0;i<block_height;++i){
left_edge[i] = temperature[i+1][1];
right_edge[i] = temperature[i+1][block_width];
}
// Send my left edge
thisProxy(wrap_x(thisIndex.x-1), thisIndex.y).ghostsFromRight(block_height, left_edge);
// Send my right edge
thisProxy(wrap_x(thisIndex.x+1), thisIndex.y).ghostsFromLeft(block_height, right_edge);
// Send my top edge
thisProxy(thisIndex.x, wrap_y(thisIndex.y-1)).ghostsFromBottom(block_width, &temperature[1][1]);
// Send my bottom edge
thisProxy(thisIndex.x, wrap_y(thisIndex.y+1)).ghostsFromTop(block_width, &temperature[block_height][1]);
hasSent=true;
check_and_compute();
delete [] right_edge;
delete [] left_edge;
}
void Patch::ResumeFromSync(){
// if (thisIndex.x+thisIndex.y+thisIndex.z == 0) CkPrintf("patch 0 ResumeFromSync at %f\n",CmiWallTimer());
thisProxy(thisIndex.x, thisIndex.y, thisIndex.z).start();
stepTime = CmiWallTimer();
LBTurnInstrumentOn();
//resumeCount = 0;
}
void Compute::notifyReceiver(int pe, CkIndex3D index, int arr) {
// --- B --- | --- C --- | --- A ---
if(arr == SENDA) {
rHandles[num_chare_x + num_chare_y + index.z] = CkDirect_createHandle(pe, &A[(index.z)*subBlockDimXz*blockDimY], sizeof(float)*subBlockDimXz*blockDimY, Compute::callBackRcvdA, (void *)this, OOB);
thisProxy(index.x, index.y, index.z).recvHandle(rHandles[num_chare_x + num_chare_y + index.z], thisIndex.z, arr);
}
if(arr == SENDB) {
rHandles[index.x] = CkDirect_createHandle(pe, &B[(index.x)*subBlockDimYx*blockDimZ], sizeof(float)*subBlockDimYx*blockDimZ, Compute::callBackRcvdB, (void *)this, OOB);
thisProxy(index.x, index.y, index.z).recvHandle(rHandles[index.x], thisIndex.x, arr);
}
if(arr == SENDC) {
rHandles[num_chare_x + index.y] = CkDirect_createHandle(pe, &tmpC[(index.y)*subBlockDimXy*blockDimZ], sizeof(float)*subBlockDimXy*blockDimZ, Compute::callBackRcvdC, (void *)this, OOB);
thisProxy(index.x, index.y, index.z).recvHandle(rHandles[num_chare_x + index.y], thisIndex.y, arr);
}
}
void Compute::setupChannels() {
int mype = CkMyPe();
int x = thisIndex.x;
int y = thisIndex.y;
int z = thisIndex.z;
for(int k=0; k<num_chare_z; k++)
if(k != z)
thisProxy(x, y, k).notifyReceiver(mype, thisIndex, SENDA);
for(int i=0; i<num_chare_x; i++)
if(i != x)
thisProxy(i, y, z).notifyReceiver(mype, thisIndex, SENDB);
for(int j=0; j<num_chare_y; j++)
if(j != y)
thisProxy(x, j, z).notifyReceiver(mype, thisIndex, SENDC);
}
void SayHi2(int hiNo)
{
CkPrintf("Hi 2 [%d] from element (%d,%d,%d)\n",hiNo,
thisIndex.x,thisIndex.y,thisIndex.z);
int y=thisIndex.y+1;
if (y < nElements) {
//Pass the hello on:
thisProxy(37,y,2*y+1).SayHi2(hiNo+1);
}
else
//We've been around once-- we're done.
mainProxy.done();
}
// Send ghost faces to the six neighbors
void begin_iteration(void) {
if (thisIndex.x == 0 && thisIndex.y == 0 && thisIndex.z == 0) {
// CkPrintf("Start of iteration %d\n", iterations);
if(iterations % PRINT_FREQ == 0) {
average = timing;
timing = CmiWallTimer();
average = (timing - average)/(double)PRINT_FREQ;
CkPrintf("time=%.2f it=%d avg=%.4f\n",timing,iterations,average);
}
}
iterations++;
// Copy different faces into messages
ghostMsg *leftMsg = new (blockDimY*blockDimZ) ghostMsg(RIGHT, blockDimY, blockDimZ);
ghostMsg *rightMsg = new (blockDimY*blockDimZ) ghostMsg(LEFT, blockDimY, blockDimZ);
ghostMsg *topMsg = new (blockDimX*blockDimZ) ghostMsg(BOTTOM, blockDimX, blockDimZ);
ghostMsg *bottomMsg = new (blockDimX*blockDimZ) ghostMsg(TOP, blockDimX, blockDimZ);
ghostMsg *frontMsg = new (blockDimX*blockDimY) ghostMsg(BACK, blockDimX, blockDimY);
ghostMsg *backMsg = new (blockDimX*blockDimY) ghostMsg(FRONT, blockDimX, blockDimY);
CkSetRefNum(leftMsg, iterations);
CkSetRefNum(rightMsg, iterations);
CkSetRefNum(topMsg, iterations);
CkSetRefNum(bottomMsg, iterations);
CkSetRefNum(frontMsg, iterations);
CkSetRefNum(backMsg, iterations);
for(int j=0; j<blockDimY; ++j)
for(int k=0; k<blockDimZ; ++k) {
leftMsg->gh[k*blockDimY+j] = temperature[index(1, j+1, k+1)];
rightMsg->gh[k*blockDimY+j] = temperature[index(blockDimX, j+1, k+1)];
}
for(int i=0; i<blockDimX; ++i)
for(int k=0; k<blockDimZ; ++k) {
topMsg->gh[k*blockDimX+i] = temperature[index(i+1, 1, k+1)];
bottomMsg->gh[k*blockDimX+i] = temperature[index(i+1, blockDimY, k+1)];
}
for(int i=0; i<blockDimX; ++i)
for(int j=0; j<blockDimY; ++j) {
frontMsg->gh[j*blockDimX+i] = temperature[index(i+1, j+1, 1)];
backMsg->gh[j*blockDimX+i] = temperature[index(i+1, j+1, blockDimZ)];
}
// Send my left face
thisProxy(wrap_x(thisIndex.x-1), thisIndex.y, thisIndex.z).receiveGhosts(leftMsg);
// Send my right face
thisProxy(wrap_x(thisIndex.x+1), thisIndex.y, thisIndex.z).receiveGhosts(rightMsg);
// Send my top face
thisProxy(thisIndex.x, wrap_y(thisIndex.y-1), thisIndex.z).receiveGhosts(topMsg);
// Send my bottom face
thisProxy(thisIndex.x, wrap_y(thisIndex.y+1), thisIndex.z).receiveGhosts(bottomMsg);
// Send my front face
thisProxy(thisIndex.x, thisIndex.y, wrap_z(thisIndex.z-1)).receiveGhosts(frontMsg);
// Send my back face
thisProxy(thisIndex.x, thisIndex.y, wrap_z(thisIndex.z+1)).receiveGhosts(backMsg);
}
void Patch::applyForces(){
int i, x, y, z, x1, y1, z1;
// if all forces are received, then it must recompute particles location
if (forceCount == numNbrs) {
CkVec<Particle> *outgoing = new CkVec<Particle>[numNbrs];
// Received all it's forces from the interactions.
forceCount = 0;
// Update properties on own particles
updateProperties();
// sending particles to neighboring cells
x = thisIndex.x;
y = thisIndex.y;
z = thisIndex.z;
if (stepCount > 0 && (stepCount % migrateStepCount) == 0){
for(i=0; i<particles.length(); i++) {
migrateToPatch(particles[i], x1, y1, z1);
if(x1 !=0 || y1!=0 || z1 !=0) {
//CkPrintf("PARTICLE MIGRATING!\n");
outgoing[(x1+1)*nbrsY*nbrsZ + (y1+1)*nbrsZ + (z1+1)].push_back(wrapAround(particles[i]));
particles.remove(i);
}
}
for(int num=0; num<numNbrs; num++) {
x1 = num / (nbrsY * nbrsZ) - nbrsX/2;
y1 = (num % (nbrsY * nbrsZ)) / nbrsZ - nbrsY/2;
z1 = num % nbrsZ - nbrsZ/2;
patchArray(WRAP_X(x+x1), WRAP_Y(y+y1), WRAP_Z(z+z1)).receiveParticles(outgoing[num]);
}
}
else
incomingFlag = true;
updateFlag = true;
// checking whether to proceed with next step
thisProxy(x, y, z).checkNextStep();
//checkNextStep();
delete [] outgoing;
// thisProxy(x, y, z).checkNextStep();
//checkNextStep();
}
// else { CkPrintf("forcecount = %d/%d on patch %d %d %d\n", forceCount, numNbrs, thisIndex.x, thisIndex.y, thisIndex.z); }
}
// Function that checks whether it must start the following step or wait until other messages are received
void Patch::checkNextStep(){
int i;
double timer;
if (updateFlag && incomingFlag) {
// resetting flags
updateFlag = false;
incomingFlag = false;
stepCount++;
// adding new elements
for (i = 0; i < incomingParticles.length(); i++)
particles.push_back(incomingParticles[i]);
incomingParticles.removeAll();
if (thisIndex.x == 0 && thisIndex.y == 0 && thisIndex.z == 0 && stepCount%NUM_STEPS == 0) {
timer = CmiWallTimer();
CkPrintf("Step %d Benchmark Time %f ms/step, Total Time Elapsed %f s\n", stepCount, ((timer - stepTime)/NUM_STEPS)*1000, timer);
stepTime = timer;
// if (stepCount == 300)
// traceBegin();
// if (stepCount == 400)
// traceEnd();
}
// if (stepCount == 300 && thisIndex.x*patchArrayDimY*patchArrayDimZ + thisIndex.y*patchArrayDimZ + thisIndex.z < 8)
// traceBegin();
// if (stepCount == 301 && thisIndex.x*patchArrayDimY*patchArrayDimZ + thisIndex.y*patchArrayDimZ + thisIndex.z < 8)
// traceEnd();
// checking for next step
if (stepCount >= finalStepCount) {
// CkPrintf("Final number of particles is %d on Patch [%d][%d][%d]\n", particles.length(), thisIndex.x, thisIndex.y, thisIndex.z);
print();
contribute(CkCallback(CkIndex_Main::allDone(), mainProxy));
} else {
if (perform_lb){
AtSync();
LBTurnInstrumentOff();
perform_lb=false;
}
else{
thisProxy(thisIndex.x, thisIndex.y, thisIndex.z).start();
//contribute(CkCallback(CkIndex_Main::lbBarrier(),mainProxy));
}
}
}
}
// Send ghost faces to the six neighbors
void begin_iteration(void) {
iterations++;
// Copy different faces into messages
double *leftGhost = new double[blockDimY*blockDimZ];
double *rightGhost = new double[blockDimY*blockDimZ];
double *topGhost = new double[blockDimX*blockDimZ];
double *bottomGhost = new double[blockDimX*blockDimZ];
double *frontGhost = new double[blockDimX*blockDimY];
double *backGhost = new double[blockDimX*blockDimY];
for(int k=0; k<blockDimZ; ++k)
for(int j=0; j<blockDimY; ++j) {
leftGhost[k*blockDimY+j] = temperature[1][j+1][k+1];
rightGhost[k*blockDimY+j] = temperature[blockDimX][j+1][k+1];
}
for(int k=0; k<blockDimZ; ++k)
for(int i=0; i<blockDimX; ++i) {
bottomGhost[k*blockDimX+i] = temperature[i+1][1][k+1];
topGhost[k*blockDimX+i] = temperature[i+1][blockDimY][k+1];
}
for(int j=0; j<blockDimY; ++j)
for(int i=0; i<blockDimX; ++i) {
backGhost[j*blockDimX+i] = temperature[i+1][j+1][1];
frontGhost[j*blockDimX+i] = temperature[i+1][j+1][blockDimZ];
}
int x = thisIndex.x, y = thisIndex.y, z = thisIndex.z;
if(!leftBound)
thisProxy(x-1,y,z).receiveGhosts(iterations, RIGHT, blockDimY, blockDimZ, leftGhost);
if(!rightBound)
thisProxy(x+1,y,z).receiveGhosts(iterations, LEFT, blockDimY, blockDimZ, rightGhost);
if(!topBound)
thisProxy(x,y+1,z).receiveGhosts(iterations, BOTTOM, blockDimX, blockDimZ, topGhost);
if(!bottomBound)
thisProxy(x,y-1,z).receiveGhosts(iterations, TOP, blockDimX, blockDimZ, bottomGhost);
if(!frontBound)
thisProxy(x,y,z+1).receiveGhosts(iterations, BACK, blockDimX, blockDimY, frontGhost);
if(!backBound)
thisProxy(x,y,z-1).receiveGhosts(iterations, FRONT, blockDimX, blockDimY, backGhost);
delete [] leftGhost;
delete [] rightGhost;
delete [] bottomGhost;
delete [] topGhost;
delete [] frontGhost;
delete [] backGhost;
}
void Patch::localCreateSection() {
#ifdef USE_SECTION_MULTICAST
CkVec<CkArrayIndex6D> elems;
for (int num=0; num<numNbrs; num++)
elems.push_back(CkArrayIndex6D(computesList[num][0], computesList[num][1], computesList[num][2], computesList[num][3], computesList[num][4], computesList[num][5]));
CkArrayID computeArrayID = computeArray.ckGetArrayID();
mCastSecProxy = CProxySection_Compute::ckNew(computeArrayID, elems.getVec(), elems.size());
CkMulticastMgr *mCastGrp = CProxy_CkMulticastMgr(mCastGrpID).ckLocalBranch();
mCastSecProxy.ckSectionDelegate(mCastGrp);
mCastGrp->setReductionClient(mCastSecProxy, new CkCallback(CkIndex_Patch::reduceForces(NULL), thisProxy(thisIndex.x, thisIndex.y, thisIndex.z)));
#endif
}
// Receive and handle RPC messages
//
void Network::RPCMsg(mRPC *msg) {
//CkPrintf("Received %" PRIidx" on %" PRIidx " in iteration %" PRIidx "\n", msg->command, prtidx, iter);
// Pausing/Checkpointing/Stepping
//
if (msg->command == RPCCOMMAND_PAUSE ||
msg->command == RPCCOMMAND_STOP) {
// Coordinate the synchronization iteration
synciter = iter;
if (prtiter % 2 == 0 && cadjprt[0] > cadjprt[1]) { ++synciter; }
if (prtiter % 2 == 1 && cadjprt[1] > cadjprt[0]) { ++synciter; }
// TODO make this print with debugging flag
//CkPrintf("Messages on %" PRIidx ": c0: %d, c1: %d, pi: % " PRIidx "\n", prtidx, cadjprt[0], cadjprt[1], prtiter);
//CkPrintf("Pausing %" PRIidx " in iteration %" PRIidx "\n", prtidx, synciter);
}
else if (msg->command == RPCCOMMAND_UNPAUSE) {
// Resume simulation
thisProxy(prtidx).Cycle();
}
else if (msg->command == RPCCOMMAND_CHECK) {
// Coordinate the synchronization iteration
synciter = iter;
// Perform checkpointing
cpflag = true;
thisProxy(prtidx).SaveNetwork();
}
else if (msg->command == RPCCOMMAND_STEP) {
if (msg->nrpcdata == 0) {
// Step for one iteration
synciter = iter + 1;
}
else {
// Coordinate the synchronization iteration
synciter = iter + (idx_t) (((tick_t) (msg->rpcdata[0]*TICKS_PER_MS))/tstep);
}
// Resume simulation until synchronization point
thisProxy(prtidx).Cycle();
}
// Stimulation
//
else if (msg->command == RPCCOMMAND_STIM) {
// Coordinate the syncronization iteration
synciter = iter;
if (prtiter % 2 == 0 && cadjprt[0] > cadjprt[1]) { ++synciter; }
if (prtiter % 2 == 1 && cadjprt[1] > cadjprt[0]) { ++synciter; }
// Apply Stimuli (with offset)
}
else if (msg->command == RPCCOMMAND_PSTIM) {
// Coordinate the syncronization iteration
synciter = iter;
// Apply Stimuli
//msg->nrpcdata
// Resync
thisProxy(prtidx).Cycle();
}
// YARP Port Streams
//
else if (msg->command == RPCCOMMAND_OPEN) {
// Coordinate the syncronization iteration
synciter = iter;
// Resync
thisProxy(prtidx).Cycle();
}
else if (msg->command == RPCCOMMAND_CLOSE) {
// Coordinate the syncronization iteration
synciter = iter;
// Resync
thisProxy(prtidx).Cycle();
}
// cleanup
delete msg;
}
