Communicate::Communicate(void)
{
CkpvInitialize(CmmTable, CsmMessages);
CsmHandlerIndex = CmiRegisterHandler((CmiHandler) CsmHandler);
CsmAckHandlerIndex = CmiRegisterHandler((CmiHandler) CsmAckHandler);
CkpvAccess(CsmMessages) = CmmNew();
if ( CmiMyNode() * 2 + 2 < CmiNumNodes() ) nchildren = 2;
else if ( CmiMyNode() * 2 + 1 < CmiNumNodes() ) nchildren = 1;
else nchildren = 0;
CkpvInitialize(int, CsmAcks);
CkpvAccess(CsmAcks) = nchildren;
}
MBlockChunk::MBlockChunk(const CkArrayID &threads_)
{
commonInit();
threads=threads_;
migInit();
update.nRecd = 0;
update.wait_seqnum = -1;
messages = CmmNew();
thread->semaPut(MBLOCK_TCHARM_SEMAID,this);
}
void CmiInitCPUAffinity(char **argv)
{
static skt_ip_t myip;
int ret, i, exclude;
hostnameMsg *msg;
char *pemap = NULL;
char *commap = NULL;
char *pemapfile = NULL;
int show_affinity_flag;
int affinity_flag = CmiGetArgFlagDesc(argv,"+setcpuaffinity",
"set cpu affinity");
while (CmiGetArgIntDesc(argv,"+excludecore", &exclude, "avoid core when setting cpuaffinity")) {
if (CmiMyRank() == 0) add_exclude(exclude);
affinity_flag = 1;
}
if (CmiGetArgStringDesc(argv, "+pemapfile", &pemapfile, "define pe to core mapping file")) {
FILE *fp;
char buf[128];
pemap = (char*)malloc(1024);
fp = fopen(pemapfile, "r");
if (fp == NULL) CmiAbort("pemapfile does not exist");
while (!feof(fp)) {
if (fgets(buf, 128, fp)) {
if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
strcat(pemap, buf);
}
}
fclose(fp);
if (CmiMyPe()==0) CmiPrintf("Charm++> read from pemap file '%s': %s\n", pemapfile, pemap);
}
CmiGetArgStringDesc(argv, "+pemap", &pemap, "define pe to core mapping");
if (pemap!=NULL && excludecount>0)
CmiAbort("Charm++> +pemap can not be used with +excludecore.\n");
CmiGetArgStringDesc(argv, "+commap", &commap, "define comm threads to core mapping");
if (pemap!=NULL || commap!=NULL) affinity_flag = 1;
show_affinity_flag = CmiGetArgFlagDesc(argv,"+showcpuaffinity",
"print cpu affinity");
cpuAffinityHandlerIdx =
CmiRegisterHandler((CmiHandler)cpuAffinityHandler);
cpuAffinityRecvHandlerIdx =
CmiRegisterHandler((CmiHandler)cpuAffinityRecvHandler);
if (CmiMyRank() ==0) {
affLock = CmiCreateLock();
}
#if CMK_BLUEGENEP || CMK_BLUEGENEQ
if(affinity_flag){
affinity_flag = 0;
if(CmiMyPe()==0) CmiPrintf("Charm++> cpu affinity setting is not needed on Blue Gene, thus ignored.\n");
}
if(show_affinity_flag){
show_affinity_flag = 0;
if(CmiMyPe()==0) CmiPrintf("Charm++> printing cpu affinity is not supported on Blue Gene.\n");
}
#endif
if (!affinity_flag) {
if (show_affinity_flag) CmiPrintCPUAffinity();
return;
}
if (CmiMyPe() == 0) {
CmiPrintf("Charm++> cpu affinity enabled. \n");
if (excludecount > 0) {
CmiPrintf("Charm++> cpuaffinity excludes core: %d", excludecore[0]);
for (i=1; i<excludecount; i++) CmiPrintf(" %d", excludecore[i]);
CmiPrintf(".\n");
}
if (pemap!=NULL)
CmiPrintf("Charm++> cpuaffinity PE-core map : %s\n", pemap);
}
if (CmiMyPe() >= CmiNumPes()) { /* this is comm thread */
/* comm thread either can float around, or pin down to the last rank.
however it seems to be reportedly slower if it is floating */
CmiNodeAllBarrier();
if (commap != NULL) {
int mycore = search_pemap(commap, CmiMyPeGlobal()-CmiNumPesGlobal());
if(CmiMyPe()-CmiNumPes()==0) printf("Charm++> set comm %d on node %d to core #%d\n", CmiMyPe()-CmiNumPes(), CmiMyNode(), mycore);
if (-1 == CmiSetCPUAffinity(mycore))
CmiAbort("set_cpu_affinity abort!");
CmiNodeAllBarrier();
if (show_affinity_flag) CmiPrintCPUAffinity();
return; /* comm thread return */
}
else {
/* if (CmiSetCPUAffinity(CmiNumCores()-1) == -1) CmiAbort("set_cpu_affinity abort!"); */
#if !CMK_CRAYXT && !CMK_CRAYXE && !CMK_CRAYXC && !CMK_BLUEGENEQ
if (pemap == NULL) {
#if CMK_MACHINE_PROGRESS_DEFINED
while (affinity_doneflag < CmiMyNodeSize()) CmiNetworkProgress();
#else
#if CMK_SMP
#error "Machine progress call needs to be implemented for cpu affinity!"
#endif
#endif
}
#endif
#if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC
/* if both pemap and commmap are NULL, will compute one */
if (pemap != NULL)
#endif
{
CmiNodeAllBarrier();
if (show_affinity_flag) CmiPrintCPUAffinity();
return; /* comm thread return */
}
}
}
if (pemap != NULL && CmiMyPe()<CmiNumPes()) { /* work thread */
int mycore = search_pemap(pemap, CmiMyPeGlobal());
if(show_affinity_flag) CmiPrintf("Charm++> set PE %d on node %d to core #%d\n", CmiMyPe(), CmiMyNode(), mycore);
if (mycore >= CmiNumCores()) {
CmiPrintf("Error> Invalid core number %d, only have %d cores (0-%d) on the node. \n", mycore, CmiNumCores(), CmiNumCores()-1);
CmiAbort("Invalid core number");
}
if (CmiSetCPUAffinity(mycore) == -1) CmiAbort("set_cpu_affinity abort!");
CmiNodeAllBarrier();
CmiNodeAllBarrier();
/* if (show_affinity_flag) CmiPrintCPUAffinity(); */
return;
}
#if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC
{
int numCores = CmiNumCores();
int myid = getXTNodeID(CmiMyNodeGlobal(), CmiNumNodesGlobal());
int myrank;
int pe, mype = CmiMyPeGlobal();
int node = CmiMyNodeGlobal();
int nnodes = 0;
#if CMK_SMP
if (CmiMyPe() >= CmiNumPes()) { /* this is comm thread */
int node = CmiMyPe() - CmiNumPes();
mype = CmiGetPeGlobal(CmiNodeFirst(node) + CmiMyNodeSize() - 1, CmiMyPartition()); /* last pe on SMP node */
node = CmiGetNodeGlobal(node, CmiMyPartition());
}
#endif
pe = mype - 1;
while (pe >= 0) {
int n = CmiNodeOf(pe);
if (n != node) { nnodes++; node = n; }
if (getXTNodeID(n, CmiNumNodesGlobal()) != myid) break;
pe --;
}
CmiAssert(numCores > 0);
myrank = (mype - pe - 1 + nnodes)%numCores;
#if CMK_SMP
if (CmiMyPe() >= CmiNumPes())
myrank = (myrank + 1)%numCores;
#endif
if (-1 != CmiSetCPUAffinity(myrank)) {
DEBUGP(("Processor %d is bound to core #%d on node #%d\n", CmiMyPe(), myrank, mynode));
}
else{
CmiPrintf("Processor %d set affinity failed!\n", CmiMyPe());
CmiAbort("set cpu affinity abort!\n");
}
}
if (CmiMyPe() < CmiNumPes())
CmiNodeAllBarrier();
CmiNodeAllBarrier();
#else
/* get my ip address */
if (CmiMyRank() == 0)
{
#if CMK_HAS_GETHOSTNAME
myip = skt_my_ip(); /* not thread safe, so only calls on rank 0 */
#else
CmiAbort("Can not get unique name for the compute nodes. \n");
#endif
}
CmiNodeAllBarrier();
/* prepare a msg to send */
msg = (hostnameMsg *)CmiAlloc(sizeof(hostnameMsg));
CmiSetHandler((char *)msg, cpuAffinityHandlerIdx);
msg->pe = CmiMyPe();
msg->ip = myip;
msg->ncores = CmiNumCores();
DEBUGP(("PE %d's node has %d number of cores. \n", CmiMyPe(), msg->ncores));
msg->rank = 0;
CmiSyncSendAndFree(0, sizeof(hostnameMsg), (void *)msg);
if (CmiMyPe() == 0) {
int i;
hostTable = CmmNew();
rankmsg = (rankMsg *)CmiAlloc(sizeof(rankMsg)+CmiNumPes()*sizeof(int)*2);
CmiSetHandler((char *)rankmsg, cpuAffinityRecvHandlerIdx);
rankmsg->ranks = (int *)((char*)rankmsg + sizeof(rankMsg));
rankmsg->nodes = (int *)((char*)rankmsg + sizeof(rankMsg) + CmiNumPes()*sizeof(int));
for (i=0; i<CmiNumPes(); i++) {
rankmsg->ranks[i] = 0;
rankmsg->nodes[i] = -1;
}
for (i=0; i<CmiNumPes(); i++) CmiDeliverSpecificMsg(cpuAffinityHandlerIdx);
}
/* receive broadcast from PE 0 */
CmiDeliverSpecificMsg(cpuAffinityRecvHandlerIdx);
CmiLock(affLock);
affinity_doneflag++;
CmiUnlock(affLock);
CmiNodeAllBarrier();
#endif
if (show_affinity_flag) CmiPrintCPUAffinity();
}
