void ComputeMgr::sendBuildMICForceTable() {
CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
int pe = CkNodeFirst(CkMyNode());
int end = pe + CkNodeSize(CkMyNode());
for( ; pe != end; ++pe ) {
cm[pe].recvBuildMICForceTable();
}
}
void CkArrayReductionMgr::collectAllMessages(){
// if the queue isn't full, but there is at least one message and the
// reduction is streamable, do a partial reduction
bool partialReduction = count < size && my_msgs.length() > 0 && \
CkReduction::reducerTable[my_msgs.peek()->reducer].streamable;
if(count == size || partialReduction) {
ARPRINT("[%d] CollectAll messages for %d with %d on %p\n",CkMyNode(),redNo,count,this);
CkReductionMsg *result = reduceMessages();
if (partialReduction) {
my_msgs.enq(result);
return;
}
result->redNo = redNo;
/**keep a count of elements that contributed to the original reduction***/
contributeWithCounter(result,result->gcount);
count=0;
redNo++;
int n=my_futureMsgs.length();
for(int i=0;i<n;i++){
CkReductionMsg *elementMesg = my_futureMsgs.deq();
if(elementMesg->getRedNo() == redNo){
my_msgs.enq(elementMesg);
count++;
collectAllMessages();
}else{
my_futureMsgs.enq(elementMesg);
}
}
}
}
void CkArrayReductionMgr::setAttachedGroup(CkGroupID groupID){
attachedGroup = groupID;
ARPRINT("[%d] setAttachedGroup called with attachedGroup %d \n",CkMyNode(),attachedGroup);
if (alreadyStarted != -1) {
((CkNodeReductionMgr *)this)->restartLocalGroupReductions(alreadyStarted);
alreadyStarted = -1;
}
}
int CkArrayReductionMgr::getTotalGCount(){
int firstPE = CkNodeFirst(CkMyNode());
int totalGCount=0;
for(int i=0;i<size;i++){
CProxy_CkReductionMgr reductionMgrProxy(attachedGroup);
CkReductionMgr *mgrPtr = reductionMgrProxy[firstPE+i].ckLocalBranch();
CkAssert(mgrPtr != NULL);
totalGCount += mgrPtr->getGCount();
}
return totalGCount;
}
int CkArrayReductionMgr::startLocalGroupReductions(int number){
ARPRINT("[%d] startLocalGroupReductions for red No %d my group %d attached group %d number of rednMgrs %d on %p \n",CkMyNode(),number,thisgroup.idx, attachedGroup.idx,size,this);
if(attachedGroup.isZero()){
alreadyStarted = number;
return 0;
}
int firstPE = CkNodeFirst(CkMyNode());
for(int i=0;i<size;i++){
CProxy_CkReductionMgr reductionMgrProxy(attachedGroup);
reductionMgrProxy[firstPE+i].ReductionStarting(new CkReductionNumberMsg(number));
}
return 1;
}
// Converse immediate handler
void immediatering_startHandler(void *vmsg)
{
const int maxRings = 1000;
immediateMsg *msg=(immediateMsg *)vmsg;
if(0!=strcmp(msg->data,"Array!")) {
CkAbort("Message corrupted");
}
if(CkMyNode()==0)
msg->iter++;
if (msg->iter < maxRings) { /* Keep passing message around the ring */
/* This is questionable: */
sendImmediate((CkMyNode()+1) % CkNumNodes(),msg->iter);
} else /* msg->iter>=maxRings, so stop: */ {
/* megatest_finish isn't safe from an immediate, so send a
regular non-immediate message out: */
int size=CmiMsgHeaderSizeBytes;
void *msg=CmiAlloc(size);
CmiSetHandler(msg,immediatering_finishHandlerIdx);
CmiSyncSendAndFree(0,size,(char *)msg);
}
CmiFree(msg);
}
void immRing_nodegroup::start(immMessage *msg)
{
const int maxRings = 50;
//CkPrintf("[%d] start %d\n", thisIndex, msg->iter);
if(!msg->check()) {
CkError("Message corrupted!\n");
megatest_finish();
return;
}
if(CkMyNode()==0)
msg->iter++;
if (msg->iter < maxRings) {
thisProxy[(CkMyNode()+1) % CkNumNodes()].start(msg);
} else {
delete msg;
//megatest_finish();
int size=CmiMsgHeaderSizeBytes;
void *msg=CmiAlloc(size);
CmiSetHandler(msg,immediatering_finishHandlerIdx);
CmiSyncSendAndFree(0,size,(char *)msg);
}
}
void CkArrayReductionMgr::startNodeGroupReduction(int number,CkGroupID groupID){
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
Chare *oldObj =CpvAccess(_currentObj);
CpvAccess(_currentObj) = this;
#endif
ARPRINT("[%d] startNodeGroupReductions for red No %d my group %d attached group %d on %p \n",CkMyNode(),number,thisgroup.idx, attachedGroup.idx,this);
if(attachedGroup.isZero()){
setAttachedGroup(groupID);
}
startLocalGroupReductions(number);
CkReductionNumberMsg *msg = new CkReductionNumberMsg(number);
envelope::setSrcPe((char *)UsrToEnv(msg),CkMyNode());
((CkNodeReductionMgr *)this)->ReductionStarting(msg);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CpvAccess(_currentObj) = oldObj;
#endif
}
void CkArrayReductionMgr::contributeArrayReduction(CkReductionMsg *m){
ARPRINT("[%d]Contribute Array Reduction called for RedNo %d group %d \n",CkMyNode(),m->getRedNo(),thisgroup.idx);
/** store the contribution untill all procs have contributed. At that point reduce and
carry out a reduction among nodegroups*/
#if CMK_BIGSIM_CHARM
_TRACE_BG_TLINE_END(&(m->log));
#endif
CmiLock(lockCount);
if(m->getRedNo() == redNo){
my_msgs.enq(m);
count++;
collectAllMessages();
}else{
//ARPRINT("[%d][%d]Out of sequence messages for %d Present redNo %d \n",CkMyNode(),CkMyPe(),m->getRedNo(),redNo);
my_futureMsgs.enq(m);
}
CmiUnlock(lockCount);
}
FVectorCache::FVectorCache() {
eps_chares_x = 7;
eps_chares_y = 7;
totalSize = 0;
GWBSE *gwbse = GWBSE::get();
L = gwbse->gw_parallel.L;
int total_eps_chares = eps_chares_x*eps_chares_y;
my_chare_count = total_eps_chares/CkNumNodes();
my_chare_start = CkMyNode()*my_chare_count;
int remaining = total_eps_chares%CkNumNodes();
if(CkMyNode()>0)
my_chare_start += remaining;
if(CkMyNode()==0)
my_chare_count += remaining;
my_eps_chare_indices_x = new int[my_chare_count];
my_eps_chare_indices_y = new int[my_chare_count];
findIndices();
int count = 0;
for(int i=eps_start_chare_x;i<=eps_end_chare_x;i++){
int j = 0;
if(i==eps_start_chare_x)
j = eps_start_chare_y;
int j_end = eps_chares_y-1;
if(i==eps_end_chare_x)
j_end = eps_end_chare_y;
while(j<=j_end){
my_eps_chare_indices_x[count] = i;
my_eps_chare_indices_y[count++] = j;
j++;
}
}
ndata = gwbse->gw_parallel.n_elems;
data_size_x = ndata/eps_chares_x;
if(ndata%eps_chares_x > 0)
data_size_x += 2;
data_size_y = ndata/eps_chares_y;
if(ndata%eps_chares_y > 0)
data_size_y += 2;
data_offset_x = new int[my_chare_count];
data_offset_y = new int[my_chare_count];
for(int i=0;i<my_chare_count;i++){
data_offset_x[i] = my_eps_chare_indices_x[i]*data_size_x;
data_offset_y[i] = my_eps_chare_indices_y[i]*data_size_y;
}
int size_x = data_size_x;
int size_y = data_size_y;
local_offset = new int[my_chare_count*2];
global_offset = new int[my_chare_count*2];
for(int i=0;i<my_chare_count;i++){
global_offset[2*i] = data_offset_x[i];//totalSize;
local_offset[2*i] = totalSize;
totalSize += size_x;
global_offset[2*i+1] = data_offset_y[i];//totalSize;
local_offset[2*i+1] = totalSize;
totalSize += size_y;
}
fs = new complex[NSIZE*L*totalSize];
contribute(CkCallback(CkReductionTarget(Controller,fCacheReady), controller_proxy));
}
// TODO: improve this to only be called when REGISTER_REGIONS is active
// at the moment "#ifdef REGISTER REGIONS" doesn't work in controller.ci
void PsiCache::reportInfo() {
if(min_row != -1 && max_row != -1 && min_col != -1 && max_col != -1) {
CkPrintf("PsiCache: MyNode = %d\nminRow: %d, maxRow: %d, minCol: %d, maxCol: %d\n", CkMyNode(), min_row, max_row, min_col, max_col);
}
}
// broadcast
void CkCheckpointMgr::Checkpoint(const char *dirname, CkCallback& cb){
chkptStartTimer = CmiWallTimer();
// every body make dir in case it is local directory
CmiMkdir(dirname);
if (CkMyPe() == 0) {
checkpointOne(dirname, cb);
}
char fileName[1024];
#ifndef CMK_CHARE_USE_PTR
// save groups into Chares.dat
sprintf(fileName,"%s/Chares_%d.dat",dirname,CkMyPe());
FILE* fChares = CmiFopen(fileName,"wb");
if(!fChares) CkAbort("Failed to create checkpoint file for chares!");
PUP::toDisk pChares(fChares);
CkPupChareData(pChares);
CmiFclose(fChares);
#endif
// save groups into Groups.dat
// content of the file: numGroups, GroupInfo[numGroups], _groupTable(PUP'ed), groups(PUP'ed)
sprintf(fileName,"%s/Groups_%d.dat",dirname,CkMyPe());
FILE* fGroups = CmiFopen(fileName,"wb");
if(!fGroups) CkAbort("Failed to create checkpoint file for group table!");
PUP::toDisk pGroups(fGroups);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CkPupGroupData(pGroups,CmiTrue);
#else
CkPupGroupData(pGroups);
#endif
CmiFclose(fGroups);
// save nodegroups into NodeGroups.dat
// content of the file: numNodeGroups, GroupInfo[numNodeGroups], _nodeGroupTable(PUP'ed), nodegroups(PUP'ed)
if (CkMyRank() == 0) {
sprintf(fileName,"%s/NodeGroups_%d.dat",dirname,CkMyNode());
FILE* fNodeGroups = CmiFopen(fileName,"wb");
if(!fNodeGroups)
CkAbort("Failed to create checkpoint file for nodegroup table!");
PUP::toDisk pNodeGroups(fNodeGroups);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CkPupNodeGroupData(pNodeGroups,CmiTrue);
#else
CkPupNodeGroupData(pNodeGroups);
#endif
CmiFclose(fNodeGroups);
}
//DEBCHK("[%d]CkCheckpointMgr::Checkpoint called dirname={%s}\n",CkMyPe(),dirname);
sprintf(fileName,"%s/arr_%d.dat",dirname, CkMyPe());
FILE *datFile=CmiFopen(fileName,"wb");
if (datFile==NULL) CkAbort("Could not create data file");
PUP::toDisk p(datFile);
CkPupArrayElementsData(p);
CmiFclose(datFile);
#if CMK_HAS_SYNC && ! CMK_DISABLE_SYNC
system("sync");
#endif
restartCB = cb;
DEBCHK("[%d]restartCB installed\n",CkMyPe());
CkCallback localcb(CkIndex_CkCheckpointMgr::SendRestartCB(NULL),0,thisgroup);
contribute(0,NULL,CkReduction::sum_int,localcb);
}
