/** \function pidtonid
* finds nids for pids 1 to CmiNumPes and stores them in an array
* correspondingly also creates an array for nids to pids
*/
void pidtonid(int numpes) {
#if XT3_TOPOLOGY
cnos_nidpid_map_t *nidpid;
int ierr, i, j, nid;
nidpid = (cnos_nidpid_map_t *)malloc(sizeof(cnos_nidpid_map_t) * numpes);
pid2nid = (int *)malloc(sizeof(int) * numpes);
for(i=0; i<MAXNID; i++) {
nid2pid[i][0] = -1;
nid2pid[i][1] = -1;
}
ierr = cnos_get_nidpid_map(&nidpid);
for(i=0; i<numpes; i++) {
nid = nidpid[i].nid;
pid2nid[i] = nid;
/* if the first position on the node is not filled */
/* put it there (0) else at (1) */
if (nid2pid[nid][0] == -1)
nid2pid[nid][0] = i;
else
nid2pid[nid][1] = i;
}
/* free(nidpid); */
/* CORRECTION FOR MPICH_RANK_REORDER_METHOD */
int k = -1;
for(i=0; i<MAXNID; i++) {
if(nid2pid[i][0] != -1) {
nid2pid[i][0] = k++;
pid2nid[k] = i;
nid2pid[i][1] = k++;
pid2nid[k] = i;
}
}
#elif XT4_TOPOLOGY || XT5_TOPOLOGY
int i, l, nid;
pid2nid = (int *)malloc(sizeof(int) * numpes);
for(i=0; i<MAXNID; i++)
for(l=0; l<TDIM; l++)
nid2pid[i][l] = -1;
for (i=0; i<numpes; i++) {
PMI_Get_nid(i, &nid);
pid2nid[i] = nid;
l = 0;
while(nid2pid[nid][l] != -1)
l++;
nid2pid[nid][l] = i;
}
#endif
}
static int rte_init(void)
{
int rc;
int nprocs;
orte_dt_init();
/* Get our process information */
/* Procs in this environment are directly launched. Hence, there
* was no mpirun to create a jobid for us, and each app proc is
* going to have to fend for itself. For now, we assume that the
* jobid is some arbitrary number (say, 1).
*/
ORTE_PROC_MY_NAME->jobid = 1;
/* find our vpid from cnos */
ORTE_PROC_MY_NAME->vpid = (orte_vpid_t) cnos_get_rank();
/* Get the number of procs in the job from cnos */
orte_process_info.num_procs = (orte_std_cntr_t) cnos_get_size();
/* Get the nid map */
nprocs = cnos_get_nidpid_map(&map);
if (nprocs <= 0) {
opal_output(0, "%5d: cnos_get_nidpid_map() returned %d",
cnos_get_rank(), nprocs);
return ORTE_ERR_FATAL;
}
/* MPI_Init needs the grpcomm framework, so we have to init it */
if (ORTE_SUCCESS != (rc = orte_grpcomm_base_open())) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (ORTE_SUCCESS != (rc = orte_grpcomm_base_select())) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* that's all we need here */
return ORTE_SUCCESS;
}
/** \function getXTNodeID
* returns nodeID corresponding to the MPI rank (possibly obtained
* from CmiMyNode()/CmiNodeOf(pe)) passed to it
*/
int getXTNodeID(int mpirank, int nummpiranks) {
int nid = -1;
#if XT3_TOPOLOGY
cnos_nidpid_map_t *nidpid;
int ierr;
nidpid = (cnos_nidpid_map_t *)malloc(sizeof(cnos_nidpid_map_t) * nummpiranks);
ierr = cnos_get_nidpid_map(&nidpid);
nid = nidpid[mpirank].nid;
/* free(nidpid); */
#elif CMK_HAS_PMI_GET_NID /* if it is a XT4/5 */
PMI_Get_nid(mpirank, &nid);
#else
#error "Cannot get network topology information on a Cray build. Swap current module xt-mpt with xt-mpt/5.0.0 or higher and xt-asyncpe with xt-asyncpe/4.0 or higher and then rebuild"
#endif
return nid;
}
