Main(CkArgMsg* m) {
#if CMK_BLUEGENEL
BGLPersonality bgl_p;
int i = rts_get_personality(&bgl_p, sizeof(BGLPersonality));
#elif CMK_BLUEGENEP
DCMF_Hardware_t bgp_hwt;
DCMF_Hardware(&bgp_hwt);
#elif XT3_TOPOLOGY
XT3TorusManager xt3tm;
#elif XT4_TOPOLOGY || XT5_TOPOLOGY
XTTorusManager xttm;
#endif
mainProxy = thishandle;
CkPrintf("Testing TopoManager .... \n");
TopoManager tmgr;
CkPrintf("Torus Size [%d] [%d] [%d] [%d]\n", tmgr.getDimNX(), tmgr.getDimNY(), tmgr.getDimNZ(), tmgr.getDimNT());
#if CMK_BLUEGENEP
CkPrintf("Torus Size [%d] [%d] [%d] [%d]\n", bgp_hwt.xSize, bgp_hwt.ySize, bgp_hwt.zSize, bgp_hwt.tSize);
#endif
int x, y, z, t;
for(int i=0; i<CkNumPes(); i++) {
tmgr.rankToCoordinates(i, x, y, z, t);
CkPrintf("---- Processor %d ---> x %d y %d z %d t %d\n", i, x, y, z, t);
#if CMK_BLUEGENEL
unsigned int tmp_t, tmp_x, tmp_y, tmp_z;
rts_coordinatesForRank(i, &tmp_x, &tmp_y, &tmp_z, &tmp_t);
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#elif CMK_BLUEGENEP
unsigned int tmp_t, tmp_x, tmp_y, tmp_z;
#if (DCMF_VERSION_MAJOR >= 3)
DCMF_NetworkCoord_t nc;
DCMF_Messager_rank2network(i, DCMF_DEFAULT_NETWORK, &nc);
tmp_x = nc.torus.x;
tmp_y = nc.torus.y;
tmp_z = nc.torus.z;
tmp_t = nc.torus.t;
#else
DCMF_Messager_rank2torus(c, &tmp_x, &tmp_y, &tmp_z, &tmp_t);
#endif
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#elif XT3_TOPOLOGY
int tmp_t, tmp_x, tmp_y, tmp_z;
xt3tm.realRankToCoordinates(i, tmp_x, tmp_y, tmp_z, tmp_t);
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#elif XT4_TOPOLOGY || XT5_TOPOLOGY
int tmp_t, tmp_x, tmp_y, tmp_z;
xttm.realRankToCoordinates(i, tmp_x, tmp_y, tmp_z, tmp_t);
CkPrintf("Real Processor %d ---> x %d y %d z %d t %d\n", i, tmp_x, tmp_y, tmp_z, tmp_t);
#endif
} // end of for loop
int size = tmgr.getDimNX() * tmgr.getDimNY() * tmgr.getDimNZ();
CkPrintf("Torus Contiguity Metric %d : %d [%f] \n", size, CkNumPes()/tmgr.getDimNT(), (float)(CkNumPes())/(tmgr.getDimNT()*size) );
CkExit();
};
void get_contention()
{
unsigned int iter, size, dst;
unsigned int i, j, k, s;
unsigned int xdim, ydim, zdim;
unsigned int xdisp, ydisp, zdisp;
DCMF_Request_t get_req[ITERATIONS];
DCMF_Callback_t get_done;
unsigned int done_count;
DCMF_NetworkCoord_t myaddr, dstaddr;
DCMF_Network ntwk;
char buf[50];
get_done.function = done;
get_done.clientdata = (void *) &done_count;
DCMF_Messager_rank2network(nranks - 1, DCMF_TORUS_NETWORK, &dstaddr);
xdim = dstaddr.torus.x + 1;
ydim = dstaddr.torus.y + 1;
zdim = dstaddr.torus.z + 1;
if (myrank == 0)
{
printf("Dimensions of Torus : %d, %d, %d \n", xdim, ydim, zdim);
fflush(stdout);
}
DCMF_Messager_rank2network(myrank, DCMF_TORUS_NETWORK, &myaddr);
dstaddr.network = myaddr.network;
dstaddr.torus.t = myaddr.torus.t;
int size_array[] = { 8, 64, 512, 4096, 32768, 262144, 1048576 };
int size_count = sizeof(size_array) / sizeof(int);
int disp_array[][3] = { { 0, 0, 1 }, { 0, 0, 3 }, { 0, 3, 3 },
{ 3, 3, 3 }, { 0, 1, 3 }, { 1, 1, 3 },
{ 0, 2, 3 }, { 1, 2, 3 }, { 2, 2, 3 },
{ 1, 3, 3 }, { 2, 3, 3 } };
int disp_count = sizeof(disp_array) / (sizeof(int) * 3);
for (s = 0; s < size_count; s++)
{
size = size_array[s];
if (myrank == 0)
{
printf("Message Size : %20d \n", size);
printf("%30s %20s \n",
"Displacement b/w Pairs",
"Avg Bandwidth (Mbps)");
fflush(stdout);
}
/*Assumes all dimensions are equal*/
for (i = 0; i < disp_count; i++)
{
xdisp = disp_array[i][0];
ydisp = disp_array[i][1];
zdisp = disp_array[i][2];
dstaddr.torus.x = (myaddr.torus.x + xdisp) % xdim;
dstaddr.torus.y = (myaddr.torus.y + ydisp) % ydim;
dstaddr.torus.z = (myaddr.torus.z + zdisp) % zdim;
DCMF_Messager_network2rank(&dstaddr, &dst, &ntwk);
barrier();
/***********************
* start timer *
***********************/
t_start = DCMF_Timebase();
done_count = ITERATIONS;
for (iter = 0; iter < ITERATIONS; iter++)
{
DCMF_Get(&get_reg,
&get_req[iter],
get_done,
DCMF_SEQUENTIAL_CONSISTENCY,
dst,
size,
memregion[dst],
memregion[myrank],
MAX_MSG_SIZE * ITERATIONS + iter * size,
iter * size);
}
while (done_count)
DCMF_Messager_advance();
t_stop = DCMF_Timebase();
/***********************
* stop timer *
***********************/
t_sec = (t_stop - t_start) / (clockMHz * 1000000);
bw = (ITERATIONS * size) / (t_sec * 1024 * 1024);
barrier();
allreduce(-1,
(char *) &bw,
(char *) &bw_avg,
1,
DCMF_DOUBLE,
DCMF_SUM);
if (myrank == 0)
{
bw_avg = bw_avg / nranks;
sprintf(buf, "(%d)(%d)(%d)", xdisp, ydisp, zdisp);
printf("%30s %20.0f \n", buf, bw_avg);
fflush(stdout);
}
}
}
}
void send_localvsremote()
{
DCMF_Request_t send_req[ITERATIONS];
DCMF_Callback_t send_done, nocallback;
int done_count;
unsigned int msgsize, i, dst;
DCMF_NetworkCoord_t myaddr, dstaddr;
DCMF_Network ntwk;
DCQuad msginfo[ITERATIONS];
DCMF_Messager_rank2network(myrank, DCMF_TORUS_NETWORK, &myaddr);
dstaddr.network = myaddr.network;
dstaddr.torus.x = (myaddr.torus.x + 3) % 8;
dstaddr.torus.y = (myaddr.torus.y + 3) % 8;
dstaddr.torus.z = (myaddr.torus.z + 3) % 8;
dstaddr.torus.t = myaddr.torus.t;
DCMF_Messager_network2rank(&dstaddr, &dst, &ntwk);
send_done.function = done;
send_done.clientdata = (void *) &done_count;
nocallback.function = NULL;
nocallback.clientdata = NULL;
if (myrank == 0)
{
printf("Send call overhead in usec\n");
fflush(stdout);
}
if (myrank == 0)
{
char buffer[100];
sprintf(buffer,
"%20s %20s %20s",
"Msg Size",
"Farthest pairs",
"Closest pairs");
printf("%s \n", buffer);
fflush(stdout);
}
for (msgsize = 1; msgsize < MAX_MSG_SIZE; msgsize *= 2)
{
/***********************
* warmup *
***********************/
snd_rcv_active += SKIP;
done_count += SKIP;
for (i = 0; i < SKIP; i++)
{
DCMF_Send(&snd_reg,
&send_req[i],
send_done,
DCMF_SEQUENTIAL_CONSISTENCY,
dst,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
}
while (done_count || snd_rcv_active)
DCMF_Messager_advance();
t_avg = 0;
t_avg1 = 0, t_avg2 = 0;
target_index = 0;
barrier();
snd_rcv_active += ITERATIONS;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
DCMF_Send(&snd_reg,
&send_req[i],
nocallback,
DCMF_SEQUENTIAL_CONSISTENCY,
dst,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
}
t_stop = DCMF_Timebase();
t_usec = (t_stop - t_start) / (clockMHz * ITERATIONS);
while (snd_rcv_active)
DCMF_Messager_advance();
barrier();
allreduce(-1,
(char *) &t_usec,
(char *) &t_avg,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
target_index = 0;
snd_rcv_active += ITERATIONS;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
DCMF_Send(&snd_reg,
&send_req[i],
nocallback,
DCMF_SEQUENTIAL_CONSISTENCY,
(myrank + 1) % nranks,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
}
t_stop = DCMF_Timebase();
t_usec1 = (t_stop - t_start) / (clockMHz * ITERATIONS);
while (snd_rcv_active)
DCMF_Messager_advance();
barrier();
allreduce(-1,
(char *) &t_usec1,
(char *) &t_avg1,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
if (myrank == 0)
{
t_avg = t_avg / nranks;
t_avg1 = t_avg1 / nranks;
printf("%20d %20.2f %20.2f \n", msgsize, t_avg, t_avg1);
fflush(stdout);
}
}
if (myrank == 0)
{
printf("Send latency in usec with local vs remote completion \n");
fflush(stdout);
}
if (myrank == 0)
{
char buffer[100];
sprintf(buffer,
"%20s %20s %20s %20s %20s %20s %20s",
"Msg Size",
"Farthest pairs-local",
"Farthest pairs-remote",
"Farthest pairs-both",
"Closest pairs-local",
"Closest pairs-remote",
"Closest pairs-both");
printf("%s \n", buffer);
fflush(stdout);
}
barrier();
for (msgsize = 1; msgsize < MAX_MSG_SIZE; msgsize *= 2)
{
/***********************
* start timer *
***********************/
snd_rcv_active += ITERATIONS;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
done_count = 1;
DCMF_Send(&snd_reg,
&send_req[i],
send_done,
DCMF_SEQUENTIAL_CONSISTENCY,
dst,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
while (done_count)
DCMF_Messager_advance();
}
t_stop = DCMF_Timebase();
t_usec = (t_stop - t_start) / (clockMHz * ITERATIONS);
while (snd_rcv_active)
DCMF_Messager_advance();
barrier();
allreduce(-1,
(char *) &t_usec,
(char *) &t_avg,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
target_index = 0;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
ack_rcv_active = 1;
DCMF_Send(&rcb_snd_reg,
&send_req[i],
nocallback,
DCMF_SEQUENTIAL_CONSISTENCY,
dst,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
while (ack_rcv_active)
DCMF_Messager_advance();
}
t_stop = DCMF_Timebase();
t_usec1 = (t_stop - t_start) / (clockMHz * ITERATIONS);
barrier();
allreduce(-1,
(char *) &t_usec1,
(char *) &t_avg1,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
target_index = 0;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
done_count = 1;
ack_rcv_active = 1;
DCMF_Send(&rcb_snd_reg,
&send_req[i],
send_done,
DCMF_SEQUENTIAL_CONSISTENCY,
dst,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
while (done_count || ack_rcv_active)
DCMF_Messager_advance();
}
t_stop = DCMF_Timebase();
t_usec2 = (t_stop - t_start) / (clockMHz * ITERATIONS);
/***********************
* stop timer *
***********************/
barrier();
allreduce(-1,
(char *) &t_usec2,
(char *) &t_avg2,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
if (myrank == 0)
{
t_avg = t_avg / nranks;
t_avg1 = t_avg1 / nranks;
t_avg2 = t_avg2 / nranks;
printf("%20d %20.2f %20.2f %20.2f", msgsize, t_avg, t_avg1, t_avg2);
fflush(stdout);
}
t_avg = 0;
t_avg1 = 0, t_avg2 = 0;
target_index = 0;
barrier();
/***********************
* start timer *
***********************/
snd_rcv_active += ITERATIONS;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
done_count = 1;
DCMF_Send(&snd_reg,
&send_req[i],
send_done,
DCMF_SEQUENTIAL_CONSISTENCY,
(myrank + 1) % nranks,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
while (done_count)
DCMF_Messager_advance();
}
t_stop = DCMF_Timebase();
t_usec = (t_stop - t_start) / (clockMHz * ITERATIONS);
while (snd_rcv_active)
DCMF_Messager_advance();
barrier();
allreduce(-1,
(char *) &t_usec,
(char *) &t_avg,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
target_index = 0;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
ack_rcv_active = 1;
DCMF_Send(&rcb_snd_reg,
&send_req[i],
nocallback,
DCMF_SEQUENTIAL_CONSISTENCY,
(myrank + 1) % nranks,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
while (ack_rcv_active)
DCMF_Messager_advance();
}
t_stop = DCMF_Timebase();
t_usec1 = (t_stop - t_start) / (clockMHz * ITERATIONS);
barrier();
allreduce(-1,
(char *) &t_usec1,
(char *) &t_avg1,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
target_index = 0;
t_start = DCMF_Timebase();
for (i = 0; i < ITERATIONS; i++)
{
done_count = 1;
ack_rcv_active = 1;
DCMF_Send(&rcb_snd_reg,
&send_req[i],
send_done,
DCMF_SEQUENTIAL_CONSISTENCY,
(myrank + 1) % nranks,
msgsize,
source + i * msgsize,
&msginfo[i],
1);
while (done_count || ack_rcv_active)
DCMF_Messager_advance();
}
t_stop = DCMF_Timebase();
t_usec2 = (t_stop - t_start) / (clockMHz * ITERATIONS);
/***********************
* stop timer *
***********************/
allreduce(-1,
(char *) &t_usec2,
(char *) &t_avg2,
1,
DCMF_DOUBLE,
DCMF_SUM);
barrier();
if (myrank == 0)
{
t_avg = t_avg / nranks;
t_avg1 = t_avg1 / nranks;
t_avg2 = t_avg2 / nranks;
printf("%20.2f %20.2f %20.2f \n", t_avg, t_avg1, t_avg2);
fflush(stdout);
}
}
}
