void *Communicate::getMessage(int PE, int tag)
{
if ( CmiMyRank() ) NAMD_bug("Communicate::getMessage called on non-rank-zero Pe\n");
int itag[2], rtag[2];
void *msg;
itag[0] = (PE==(-1)) ? (CmmWildCard) : PE;
itag[1] = (tag==(-1)) ? (CmmWildCard) : tag;
while((msg=CmmGet(CkpvAccess(CsmMessages),2,itag,rtag))==0) {
CmiDeliverMsgs(0);
}
char *ackmsg = (char *) CmiAlloc(CmiMsgHeaderSizeBytes);
CmiSetHandler(ackmsg, CsmAckHandlerIndex);
CmiSyncSend(CmiNodeFirst((CmiMyNode()-1)/2), CmiMsgHeaderSizeBytes, ackmsg);
while ( CkpvAccess(CsmAcks) < nchildren ) {
CmiDeliverMsgs(0);
}
CkpvAccess(CsmAcks) = 0;
int size = SIZEFIELD(msg);
for ( int i = 2; i >= 1; --i ) {
int node = CmiMyNode() * 2 + i;
if ( node < CmiNumNodes() ) {
CmiSyncSend(CmiNodeFirst(node),size,(char*)msg);
}
}
return msg;
}
/* called on PE 0 */
static void cpuAffinityHandler(void *m)
{
static int count = 0;
static int nodecount = 0;
hostnameMsg *rec;
hostnameMsg *msg = (hostnameMsg *)m;
hostnameMsg *tmpm;
int tag, tag1, pe, myrank;
int npes = CmiNumPes();
/* for debug
char str[128];
skt_print_ip(str, msg->ip);
printf("hostname: %d %s\n", msg->pe, str);
*/
CmiAssert(CmiMyPe()==0 && rankmsg != NULL);
tag = *(int*)&msg->ip;
pe = msg->pe;
if ((rec = (hostnameMsg *)CmmProbe(hostTable, 1, &tag, &tag1)) != NULL) {
CmiFree(msg);
}
else {
rec = msg;
rec->seq = nodecount;
nodecount++; /* a new node record */
CmmPut(hostTable, 1, &tag, msg);
}
myrank = rec->rank%rec->ncores;
while (in_exclude(myrank)) { /* skip excluded core */
myrank = (myrank+1)%rec->ncores;
rec->rank ++;
}
rankmsg->ranks[pe] = myrank; /* core rank */
rankmsg->nodes[pe] = rec->seq; /* on which node */
rec->rank ++;
count ++;
if (count == CmiNumPes()) {
/* CmiPrintf("Cpuaffinity> %d unique compute nodes detected! \n", CmmEntries(hostTable)); */
tag = CmmWildCard;
while (tmpm = CmmGet(hostTable, 1, &tag, &tag1)) CmiFree(tmpm);
CmmFree(hostTable);
#if 1
/* bubble sort ranks on each node according to the PE number */
{
int i,j;
for (i=0; i<npes-1; i++)
for(j=i+1; j<npes; j++) {
if (rankmsg->nodes[i] == rankmsg->nodes[j] &&
rankmsg->ranks[i] > rankmsg->ranks[j])
{
int tmp = rankmsg->ranks[i];
rankmsg->ranks[i] = rankmsg->ranks[j];
rankmsg->ranks[j] = tmp;
}
}
}
#endif
CmiSyncBroadcastAllAndFree(sizeof(rankMsg)+CmiNumPes()*sizeof(int)*2, (void *)rankmsg);
}
}
