/** \function pidtonid
* finds nids for pids 1 to CmiNumPes and stores them in an array
* correspondingly also creates an array for nids to pids
*/
void pidtonid(int numpes) {
CmiLock(cray_lock);
if (pid2nid != NULL) {
CmiUnlock(cray_lock);
return; /* did once already */
}
getDimension(&maxNID,&maxX,&maxY,&maxZ);
int numCores = CmiNumCores();
pid2nid = (int *)malloc(sizeof(int) * numpes);
#if XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
int i, nid, ret;
CmiAssert(rca_coords == NULL);
rca_coords = (rca_mesh_coord_t *)malloc(sizeof(rca_mesh_coord_t)*(maxNID+1));
for (i=0; i<maxNID; i++) {
rca_coords[i].mesh_x = rca_coords[i].mesh_y = rca_coords[i].mesh_z = -1;
}
for (i=0; i<numpes; i++) {
PMI_Get_nid(CmiGetNodeGlobal(CmiNodeOf(i),CmiMyPartition()), &nid);
pid2nid[i] = nid;
CmiAssert(nid < maxNID);
ret = rca_get_meshcoord(nid, &rca_coords[nid]);
CmiAssert(ret != -1);
}
#endif
CmiUnlock(cray_lock);
}
/** \function pidtonid
* finds nids for pids 1 to CmiNumPes and stores them in an array
* correspondingly also creates an array for nids to pids
*/
void pidtonid(int numpes) {
#if XT3_TOPOLOGY
cnos_nidpid_map_t *nidpid;
int ierr, i, j, nid;
nidpid = (cnos_nidpid_map_t *)malloc(sizeof(cnos_nidpid_map_t) * numpes);
pid2nid = (int *)malloc(sizeof(int) * numpes);
for(i=0; i<MAXNID; i++) {
nid2pid[i][0] = -1;
nid2pid[i][1] = -1;
}
ierr = cnos_get_nidpid_map(&nidpid);
for(i=0; i<numpes; i++) {
nid = nidpid[i].nid;
pid2nid[i] = nid;
/* if the first position on the node is not filled */
/* put it there (0) else at (1) */
if (nid2pid[nid][0] == -1)
nid2pid[nid][0] = i;
else
nid2pid[nid][1] = i;
}
/* free(nidpid); */
/* CORRECTION FOR MPICH_RANK_REORDER_METHOD */
int k = -1;
for(i=0; i<MAXNID; i++) {
if(nid2pid[i][0] != -1) {
nid2pid[i][0] = k++;
pid2nid[k] = i;
nid2pid[i][1] = k++;
pid2nid[k] = i;
}
}
#elif XT4_TOPOLOGY || XT5_TOPOLOGY
int i, l, nid;
pid2nid = (int *)malloc(sizeof(int) * numpes);
for(i=0; i<MAXNID; i++)
for(l=0; l<TDIM; l++)
nid2pid[i][l] = -1;
for (i=0; i<numpes; i++) {
PMI_Get_nid(i, &nid);
pid2nid[i] = nid;
l = 0;
while(nid2pid[nid][l] != -1)
l++;
nid2pid[nid][l] = i;
}
#endif
}
/** \function getXTNodeID
* returns nodeID corresponding to the MPI rank (possibly obtained
* from CmiMyNode()/CmiNodeOf(pe)) passed to it
*/
int getXTNodeID(int mpirank, int nummpiranks) {
int nid = -1;
#if CMK_HAS_PMI_GET_NID /* if it is a XT4/5 or XE */
PMI_Get_nid(mpirank, &nid);
#else
#error "Cannot get network topology information on a Cray build. Swap current module xt-mpt with xt-mpt/5.0.0 or higher and xt-asyncpe with xt-asyncpe/4.0 or higher and then rebuild"
#endif
return nid;
}
/** \function getXTNodeID
* returns nodeID corresponding to the MPI rank (possibly obtained
* from CmiMyNode()/CmiNodeOf(pe)) passed to it
*/
int getXTNodeID(int mpirank, int nummpiranks) {
int nid = -1;
#if XT3_TOPOLOGY
cnos_nidpid_map_t *nidpid;
int ierr;
nidpid = (cnos_nidpid_map_t *)malloc(sizeof(cnos_nidpid_map_t) * nummpiranks);
ierr = cnos_get_nidpid_map(&nidpid);
nid = nidpid[mpirank].nid;
/* free(nidpid); */
#elif CMK_HAS_PMI_GET_NID /* if it is a XT4/5 */
PMI_Get_nid(mpirank, &nid);
#else
#error "Cannot get network topology information on a Cray build. Swap current module xt-mpt with xt-mpt/5.0.0 or higher and xt-asyncpe with xt-asyncpe/4.0 or higher and then rebuild"
#endif
return nid;
}
int A1D_Initialize()
{
#ifdef DMAPPD_USES_MPI
int mpi_initialized, mpi_provided;
int mpi_status = MPI_SUCCESS;
int namelen;
char procname[MPI_MAX_PROCESSOR_NAME];
#endif
#ifdef __CRAYXE
int pmi_status = PMI_SUCCESS;
int nodeid = -1;
rca_mesh_coord_t rca_xyz;
dmapp_return_t dmapp_status = DMAPP_RC_SUCCESS;
dmapp_rma_attrs_ext_t dmapp_config_in, dmapp_config_out;
dmapp_jobinfo_t dmapp_info;
dmapp_pe_t dmapp_rank = -1;
int dmapp_size = -1;
#endif
int sheapflag = 0;
#ifdef DEBUG_FUNCTION_ENTER_EXIT
fprintf(stderr,"entering A1D_Initialize() \n");
#endif
#ifdef DMAPPD_USES_MPI
/***************************************************
*
* configure MPI
*
***************************************************/
/* MPI has to be Initialized for this implementation to work */
MPI_Initialized(&mpi_initialized);
assert(mpi_initialized==1);
/* MPI has to tolerate threads because A1 supports them */
MPI_Query_thread(&mpi_provided);
//assert(mpi_provided>MPI_THREAD_SINGLE);
/* have to use our own communicator for collectives to be proper */
mpi_status = MPI_Comm_dup(MPI_COMM_WORLD,&A1D_COMM_WORLD);
assert(mpi_status==0);
/* get my MPI rank */
mpi_status = MPI_Comm_rank(A1D_COMM_WORLD,&mpi_rank);
assert(mpi_status==0);
/* get MPI world size */
mpi_status = MPI_Comm_size(A1D_COMM_WORLD,&mpi_size);
assert(mpi_status==0);
/* in a perfect world, this would provide topology information like BG */
MPI_Get_processor_name( procname, &namelen );
printf( "%d: MPI_Get_processor_name = %s\n" , mpi_rank, procname );
fflush( stdout );
/* barrier to make sure MPI is ready everywhere */
mpi_status = MPI_Barrier(A1D_COMM_WORLD);
assert(mpi_status==0);
#endif
#ifdef __CRAYXE
/***************************************************
*
* query topology
*
***************************************************/
PMI_Get_nid( mpi_rank, &nodeid );
assert(pmi_status==PMI_SUCCESS);
rca_get_meshcoord((uint16_t)nodeid, &rca_xyz);
printf("%d: rca_get_meshcoord returns (%2u,%2u,%2u)\n", mpi_rank, rca_xyz.mesh_x, rca_xyz.mesh_y, rca_xyz.mesh_z );
#endif
#ifdef __CRAYXE
/***************************************************
*
* configure DMAPP
*
***************************************************/
dmapp_config_in.max_outstanding_nb = DMAPP_DEF_OUTSTANDING_NB; /* 512 */
dmapp_config_in.offload_threshold = DMAPP_OFFLOAD_THRESHOLD; /* 4096 */
#ifdef DETERMINISTIC_ROUTING
dmapp_config_in.put_relaxed_ordering = DMAPP_ROUTING_DETERMINISTIC;
dmapp_config_in.get_relaxed_ordering = DMAPP_ROUTING_DETERMINISTIC;
#else
dmapp_config_in.put_relaxed_ordering = DMAPP_ROUTING_ADAPTIVE;
dmapp_config_in.get_relaxed_ordering = DMAPP_ROUTING_ADAPTIVE;
#endif
dmapp_config_in.max_concurrency = 1; /* not thread-safe */
#ifdef FLUSH_IMPLEMENTED
dmapp_config_in.PI_ordering = DMAPP_PI_ORDERING_RELAXED;
#else
dmapp_config_in.PI_ordering = DMAPP_PI_ORDERING_STRICT;
#endif
dmapp_status = dmapp_init_ext( &dmapp_config_in, &dmapp_config_out );
assert(dmapp_status==DMAPP_RC_SUCCESS);
#ifndef FLUSH_IMPLEMENTED
/* without strict PI ordering, we have to flush remote stores with a get packet to force global visibility */
assert( dmapp_config_out.PI_ordering == DMAPP_PI_ORDERING_STRICT);
#endif
dmapp_status = dmapp_get_jobinfo(&dmapp_info);
assert(dmapp_status==DMAPP_RC_SUCCESS);
dmapp_rank = dmapp_info.pe;
dmapp_size = dmapp_info.npes;
A1D_Sheap_desc = dmapp_info.sheap_seg;
/* make sure PMI and DMAPP agree */
assert(mpi_rank==dmapp_rank);
assert(mpi_size==dmapp_size);
#endif
/***************************************************
*
* setup protocols
*
***************************************************/
#ifdef FLUSH_IMPLEMENTED
/* allocate Put list */
A1D_Put_flush_list = malloc( mpi_size * sizeof(int32_t) );
assert(A1D_Put_flush_list != NULL);
#endif
#ifdef __CRAYXE
A1D_Acc_lock = dmapp_sheap_malloc( sizeof(int64_t) );
#endif
A1D_Allreduce_issame64((size_t)A1D_Acc_lock, &sheapflag);
assert(sheapflag==1);
#ifdef DEBUG_FUNCTION_ENTER_EXIT
fprintf(stderr,"exiting A1D_Initialize() \n");
#endif
return(0);
}
int main(void)
{
int rc;
int rank, size;
PMI_BOOL initialized;
rc = PMI_Initialized(&initialized);
if (rc!=PMI_SUCCESS)
PMI_Abort(rc,"PMI_Initialized failed");
if (initialized!=PMI_TRUE)
{
int spawned;
rc = PMI_Init(&spawned);
if (rc!=PMI_SUCCESS)
PMI_Abort(rc,"PMI_Init failed");
}
rc = PMI_Get_rank(&rank);
if (rc!=PMI_SUCCESS)
PMI_Abort(rc,"PMI_Get_rank failed");
rc = PMI_Get_size(&size);
if (rc!=PMI_SUCCESS)
PMI_Abort(rc,"PMI_Get_size failed");
printf("rank %d of %d \n", rank, size);
int rpn; /* rpn = ranks per node */
rc = PMI_Get_clique_size(&rpn);
if (rc!=PMI_SUCCESS)
PMI_Abort(rc,"PMI_Get_clique_size failed");
printf("rank %d clique size %d \n", rank, rpn);
int * clique_ranks = malloc( rpn * sizeof(int) );
if (clique_ranks==NULL)
PMI_Abort(rpn,"malloc failed");
rc = PMI_Get_clique_ranks(clique_ranks, rpn);
if (rc!=PMI_SUCCESS)
PMI_Abort(rc,"PMI_Get_clique_ranks failed");
for(int i = 0; i<rpn; i++)
printf("rank %d clique[%d] = %d \n", rank, i, clique_ranks[i]);
int nid;
rc = PMI_Get_nid(rank, &nid);
if (rc!=PMI_SUCCESS)
PMI_Abort(rc,"PMI_Get_nid failed");
printf("rank %d PMI_Get_nid gives nid %d \n", rank, nid);
#if OLD
rca_mesh_coord_t xyz;
rca_get_meshcoord( (uint16_t) nid, &xyz);
printf("rank %d rca_get_meshcoord returns (%2u,%2u,%2u)\n",
rank, xyz.mesh_x, xyz.mesh_y, xyz.mesh_z);
#else // UNTESTED
pmi_mesh_coord_t xyz;
PMI_Get_meshcoord((uint16_t) nid, &xyz);
printf("rank %d PMI_Get_meshcoord returns (%2u,%2u,%2u)\n",
rank, xyz.mesh_x, xyz.mesh_y, xyz.mesh_z);
#endif
fflush(stdout);
return 0;
}
/* get size and dimension for XE machine */
void getDimension(int *maxnid, int *xdim, int *ydim, int *zdim)
{
int i = 0, nid, ret;
rca_mesh_coord_t dimsize;
CmiLock(cray_lock2);
if(maxNID != -1) {
*xdim = maxX;
*ydim = maxY;
*zdim = maxZ;
*maxnid = maxNID;
CmiUnlock(cray_lock2);
return;
}
#if CMK_HAS_RCA_MAX_DIMENSION
// rca_get_meshtopology(&mnid);
rca_get_max_dimension(&dimsize);
maxX = *xdim = dimsize.mesh_x+1;
maxY = *ydim = dimsize.mesh_y+1;
maxZ = *zdim = dimsize.mesh_z+1;
maxNID = 0;
for(i = 0; i < CmiNumNodesGlobal(); i++) {
PMI_Get_nid(i, &nid);
if(nid >= maxNID) maxNID = nid + 1;
}
*maxnid = maxNID;
#else
*xdim = *ydim = *zdim = 0;
/* loop until fails to find the max */
do {
int x, y, z;
ret = getMeshCoord(i, &x, &y, &z);
if (ret == -1) {
#if CMK_CRAY_MAXNID
if (i<=CMK_CRAY_MAXNID) {
i++;
ret = 0;
continue;
}
#endif
break;
}
if (x>*xdim) *xdim = x;
if (y>*ydim) *ydim = y;
if (z>*zdim) *zdim = z;
i++;
} while (ret == 0);
maxNID = *maxnid = i;
maxX = *xdim = *xdim+1;
maxY = *ydim = *ydim+1;
maxZ = *zdim = *zdim+1;
#endif
CmiUnlock(cray_lock2);
/* printf("%d %d %d %d\n", *maxnid, *xdim, *ydim, *zdim); */
}
