MPI_Comm ReorderRanksZCurve(MPI_Comm comm)
{
MPI_Status status;
int rank, size;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
int dim;
MPIX_Torus_ndims(&dim);
int* mycoords = new int[dim + 1];
MPIX_Rank2torus(rank, mycoords);
MPI_Send(mycoords, dim, MPI_INT, 0, 111, comm);
delete [] mycoords;
if (rank == 0)
{
int** coords = new int*[size];
for (int i = 0; i < size; i++)
{
coords[i] = new int[dim + 1];
coords[i][dim] = i;
MPI_Recv(coords[i], dim, MPI_INT, i, 111, comm, &status);
}
KdTreeSort(coords, 0, dim, size);
//DebugRankCoords(coords, dim, size);
for (int i = 0; i < size; i++)
{
MPI_Send(&coords[i][dim], 1, MPI_INT, i, 112, comm);
delete [] coords[i];
}
delete [] coords;
}
int new_rank;
MPI_Recv(&new_rank, 1, MPI_INT, 0, 112, comm, &status);
MPI_Comm new_comm;
MPI_Comm_split(comm, 0, new_rank, &new_comm);
return new_comm;
}
void F77_FUNC(mpix_rank2torus,MPIX_RANK2TORUS)(MPI_Fint * rank, MPI_Fint * coords, MPI_Fint * ierr)
{
int torusdim = -1;
int rc = MPIX_Torus_ndims(&torusdim);
if (rc!=MPI_SUCCESS) {
*ierr = (MPI_Fint)rc;
return;
}
int myrank = -1;
safecast(*rank, &myrank);
int * mycoords = malloc(torusdim*sizeof(int));
*ierr = (MPI_Fint)MPIX_Rank2torus(myrank, mycoords);
for (int i=0; i<torusdim; i++)
coords[i] = (MPI_Fint)mycoords[i];
free(mycoords);
return;
}
INLINE void PROFILER_INIT()
{
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
MPI_Comm_size(MPI_COMM_WORLD,&numranks);
int coords[6], tmasterRank;
MPIX_Rank2torus(myrank, coords);
/* choose the MPI rank on (0, 0, 0, 0, 0) [0] as the master rank */
if(coords[0]+coords[1]+coords[2]+coords[3]+coords[4]+coords[5] == 0) {
isMaster = 1;
printf("Init intercepted by bgqcounter unit\n");
} else {
isMaster = 0;
}
char *filename = getenv("BGQ_COUNTER_FILE");
if(isMaster) {
if(filename != NULL)
dataFile = fopen(filename,"w");
else
dataFile = stdout;
}
#if BGQ_DEBUG
if(isMaster) {
printf("File opened, Initializing BGPM\n");
}
#endif
Bgpm_Init(BGPM_MODE_SWDISTRIB);
#if BGQ_DEBUG
if(isMaster) {
printf("Initialized BGPM, Splitting communicator\n");
}
#endif
/* split communicator based on the T dimension */
isZero = (coords[5] == 0) ? 1 : 0;
MPI_Comm_split(MPI_COMM_WORLD, isZero, myrank, &profile_comm);
#if BGQ_DEBUG
if(isMaster) {
printf("Communicator split done, find master\n");
}
#endif
/* Every process needs to know the master rank in profile_comm to know
the root of the broadcast */
coords[0] = coords[1] = coords[2] = coords[3] = coords[4] = coords[5] = 0;
MPIX_Torus2rank(coords, &tmasterRank);
#if BGQ_DEBUG
if(isMaster) {
printf("Found master, informing master\n");
}
#endif
if(isMaster) {
MPI_Comm_rank(profile_comm, &masterRank);
}
/* Broadcast the rank of the master in profile_comm */
MPI_Bcast(&masterRank, 1, MPI_INT, tmasterRank, MPI_COMM_WORLD);
#if BGQ_DEBUG
if(isMaster) {
printf("Informed master, attaching counters\n");
}
#endif
if(isZero) {
hNWSet = Bgpm_CreateEventSet();
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_PP_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_DYN_PP_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_ESC_PP_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_SUBC_COL_SENT);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_PP_RECV);
Bgpm_AddEvent(hNWSet, PEVT_NW_USER_PP_RECV_FIFO);
numevents = 6;
if (Bgpm_Attach(hNWSet, UPC_NW_ALL_TORUS_LINKS, 0) != 0) {
printf("Error: something went wrong in attaching link counters\n");
}
curset = maxset = 0;
for(unsigned int i = 0; i < NUM_REGIONS; i++) {
for(unsigned int j = 0; j < NUM_TORUS_LINKS * numevents; j++) {
values[i].counters[j] = 0;
}
values[i].time = 0;
}
}
if(isMaster) {
printf("Init intercept complete\n");
}
}
INLINE void PROFILER_FINALIZE() {
uint64_t *allCounters;
double *times;
int nranks;
if(isMaster) {
printf("Finalize intercepted: numevents: %u, max set: %u\n",numevents, maxset);
MPI_Comm_size(profile_comm,&nranks);
allCounters = (uint64_t*) malloc(NUM_TORUS_LINKS * numevents * nranks *sizeof(uint64_t));
times = (double*) malloc(nranks * sizeof(double));
}
if(isZero) {
for(unsigned int i = 1; i <= maxset; i++) {
/* collect all counter data into allCounters */
MPI_Gather(values[i].counters, NUM_TORUS_LINKS * numevents,
MPI_UNSIGNED_LONG_LONG, allCounters, NUM_TORUS_LINKS * numevents,
MPI_UNSIGNED_LONG_LONG, masterRank, profile_comm);
MPI_Gather(&values[i].time, 1, MPI_DOUBLE, times, 1, MPI_DOUBLE, masterRank,
profile_comm);
if(isMaster) {
MPI_Group world, profile_group;
MPI_Comm_group(MPI_COMM_WORLD, &world);
MPI_Comm_group(profile_comm, &profile_group);
int *world_ranks, *profile_ranks;
profile_ranks = (int*)malloc(nranks*sizeof(int));
world_ranks = (int*)malloc(nranks*sizeof(int));
for(unsigned int j = 0; j < nranks; j++) {
profile_ranks[j] = j;
}
/* find the ranks in MPI_COMM_WORLD for all processes in profile_comm */
MPI_Group_translate_ranks(profile_group, nranks, profile_ranks, world, world_ranks);
unsigned int cnt = 0;
int coords[6];
for(unsigned int j = 0; j < nranks; j++) {
//MPIX_Rank2torus(j*Kernel_ProcessCount(), coords);
MPIX_Rank2torus(world_ranks[j], coords);
fprintf(dataFile,"%d %d ",i,world_ranks[j]);
fprintf(dataFile,"%d %d %d %d %d %d ** ",coords[0],coords[1],coords[2],coords[3],coords[4],coords[5]);
for(unsigned int k = 0; k < NUM_TORUS_LINKS * numevents; k++) {
fprintf(dataFile,"%lu ", allCounters[cnt++]);
}
fprintf(dataFile,"\n");
}
double min, max, avg = 0;
min = max = times[0];
for(unsigned j = 0; j < nranks; j++) {
avg += times[j];
if(min > times[j]) {
min = times[j];
}
if(max < times[j]) {
max = times[j];
}
}
avg /= nranks;
printf("Timing Summary: min - %.3f s, avg - %.3f s, max - %.3f s\n", min, avg, max);
free(profile_ranks); free(world_ranks);
}
}
}
if(isMaster) {
if(dataFile != stdout)
fclose(dataFile);
printf("Done profiling, exiting\n");
free(allCounters);
free(times);
}
}
