/**
* Main function
*/
int main(){
ping_pong_test();
merge_sort_test();
//multiple_channels_test();
//multiple_channels_test_v2();
//multiple_channels_test_v3();
return 0;
}
int main(int argc, char *argv[])
{
int i, j, k, lval, gval, numClasses;
int nextPart, rc, len, level, mine;
int num_groups, my_group, sender, receiver;
int me[NLEVELS], nprocs[NLEVELS];
int num_significant_levels=0;
int level_part_range[NLEVELS][2];
int level_number[NLEVELS];
int *classes=NULL;
int group_start, group_end;
char *errorstr;
comm_group level_info[NLEVELS];
MPI_Init(&argc, &argv);
for (i=0; i < NLEVELS; i++){
rc = MPI_Comm_size(comm[i], nprocs + i);
if (rc != MPI_SUCCESS){
MPI_Error_string(rc, errorstr, &len);
fprintf(stderr,"(%d) MPI_Comm_size %s : %s\n",me[0],commName[i],errorstr);
}
rc = MPI_Comm_rank(comm[i], me + i);
if (rc != MPI_SUCCESS){
MPI_Error_string(rc, errorstr, &len);
fprintf(stderr,"(%d) MPI_Comm_size %s : %s\n",me[0],commName[i],errorstr);
}
#ifdef DEBUG_ME
MPI_Barrier(MPI_COMM_WORLD);
printf("(%d) %s communicator, size %d, my rank %d\n",me[0],commName[i],nprocs[i],me[i]);
MPI_Barrier(MPI_COMM_WORLD);
#endif
}
buf = (int *)malloc(sizeof(int) * nprocs[0]);
classes = (int *)malloc(sizeof(int) * nprocs[0]);
procToPart = (int *)malloc(sizeof(int) * nprocs[0]);
partToProc = (int *)malloc(sizeof(int) * nprocs[0]);
level_part_range[0][0] = 0;
level_part_range[0][1] = nprocs[0]-1;
for (i=0; i < NLEVELS - 1; i++){
/*
* classes[k] contains the rank (in level i) of the rank 0 element of element k's subcommunicator
*/
level_down(i, classes, me, nprocs);
/*
* my sub communicator will create which parts in the final partitioning?
*/
new_part_range(classes, level_part_range[i], level_part_range[i+1], me[i], nprocs[i]);
}
#ifdef DEBUG_ME
MPI_Barrier(MPI_COMM_WORLD);
for (i=0; i < nprocs[0]; i++){
if (i == me[0]){
printf("(%d) ranges: %s (%d %d iam %d) %s (%d %d iam %d) %s (%d %d iam %d) %s (%d %d iam %d) \n",
me[0],
commName[0], level_part_range[0][0], level_part_range[0][1], level_part_range[0][0] + me[0],
commName[1], level_part_range[1][0], level_part_range[1][1], level_part_range[1][0] + me[1],
commName[2], level_part_range[2][0], level_part_range[2][1], level_part_range[2][0] + me[2],
commName[3], level_part_range[3][0], level_part_range[3][1], level_part_range[3][0] + me[3]);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
}
#endif
/* Figure out which levels are significant */
num_significant_levels = 1;
level_number[0] = 0;
gval = 0;
for (i=0; i < NLEVELS - 1; i++){
if ((nprocs[i+1] == nprocs[i]) || (nprocs[i+1] == 1)){
lval=1; /* insignificant */
}
else{
lval=0; /* meaningful level in hierarchy */
}
MPI_Allreduce(&lval, &gval, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (gval < nprocs[0]){
/* next level in hierarchy is significant for at least some processes */
level_number[num_significant_levels++] = i + 1;
}
}
#ifdef DEBUG_ME
MPI_Barrier(MPI_COMM_WORLD);
printf("(%d) %d significant levels %d %d %d %d\n",me[0], num_significant_levels,
level_number[0], level_number[1], level_number[2], level_number[3]);
MPI_Barrier(MPI_COMM_WORLD);
#endif
/* Save global info about topology */
level = level_number[num_significant_levels-1];
my_part_number = level_part_range[level][0] + me[level];
MPI_Allgather(&my_part_number, 1, MPI_INT, procToPart, 1, MPI_INT, MPI_COMM_WORLD);
for (i=0; i < nprocs[0]; i++){
partToProc[procToPart[i]] = i;
}
level_info[0].comm = MPI_COMM_WORLD;
level_info[0].name = commName[0];
level_info[0].nGroups = 1;
level_info[0].myGroup = 0;
level_info[0].offsets = (int *)malloc(sizeof(int) * 2);
level_info[0].offsets[0] = 0;
level_info[0].offsets[1] = nprocs[0];
level_info[0].myLocalPart = my_part_number;
for (i=1; i < num_significant_levels; i++){
level = level_number[i];
memset((void *)&level_info[i], 0, sizeof(comm_group));
mine = 0;
MPI_Allgather(&(level_part_range[level][0]), 1, MPI_INT, buf, 1, MPI_INT, MPI_COMM_WORLD);
memset(classes, 0, sizeof(int) * nprocs[0]);
for (j=0; j < nprocs[0]; j++){
classes[buf[j]]++;
}
memset(buf, 0, sizeof(int) * nprocs[0]);
for (j=0, numClasses=0; j < nprocs[0]; j++){
if (classes[j] > 0){
if (j < level_part_range[level][0]) mine++;
buf[numClasses++] = classes[j]; /* number of parts/procs in sub communicator */
}
}
level_info[i].comm = comm[level];
level_info[i].name = commName[level];
level_info[i].nGroups = numClasses;
level_info[i].myGroup = mine;
level_info[i].offsets = (int *)malloc(sizeof(int) * (numClasses + 1));
level_info[i].offsets[0] = 0;
for (k=0; k < numClasses; k++){
level_info[i].offsets[k+1] = level_info[i].offsets[k] + buf[k];
}
level_info[i].myLocalPart = me[level];
}
free(buf);
free(classes);
/*#ifdef DEBUG_ME*/
if (my_part_number == 0){
for (i=0; i < num_significant_levels; i++){
printf("Level %d (%s) has %d groups: \n",i, level_info[i].name, level_info[i].nGroups);
for (j=0; j < level_info[i].nGroups; j++){
printf(" Group %d has parts %d through %d\n",j, level_info[i].offsets[j], level_info[i].offsets[j+1]-1);
}
}
}
/*#endif*/
/* do some ping pong tests to compare the levels in the topology */
for (i=1; i < num_significant_levels; i++){
if (level_info[i].nGroups <= 1) continue;
/* ping pong within an entity */
sender = partToProc[0];
receiver = partToProc[level_info[i].offsets[1] - 1];
if (me[0] == sender){
printf("\nPing pong test within a %s communicator (between parts %d and %d):\n",
level_info[i].name, 0, level_info[i].offsets[1]-1);
}
ping_pong_test(MPI_COMM_WORLD, me[0], sender, receiver, 100);
/* ping pong across an entity */
if (me[0] == sender){
printf("\nPing pong test between two %s communicators (between parts %d and %d):\n",
level_info[i].name, 0, level_info[i].offsets[1]);
}
sender = partToProc[0];
receiver = partToProc[level_info[i].offsets[1]];
ping_pong_test(MPI_COMM_WORLD, me[0], sender, receiver, 100);
}
MPI_Finalize();
}