void
test_amlong(threaddata_t *tdata)
{
int peer = RANDOM_PEER(tdata);
int node = tt_thread_map[peer];
void *laddr = tt_addr_map[tdata->tid];
void *raddr = tt_addr_map[peer];
size_t len;
do {
len = RANDOM_SIZE();
} while ((len > gasnet_AMMaxLongRequest()) || (len > gasnet_AMMaxLongReply())
|| (len > TEST_SEGZ_PER_THREAD));
tdata->flag = -1;
gasnett_local_wmb();
ACTION_PRINTF("tid=%3d> AMLongRequest (sz=%7d) to tid=%3d", tdata->tid, (int)len, peer);
GASNET_Safe(gasnet_AMRequestLong2(node,
hidx_ping_longhandler, laddr, len, raddr,
tdata->ltid, peer));
GASNET_BLOCKUNTIL(tdata->flag == 0);
tdata->flag = -1;
ACTION_PRINTF("tid=%3d> AMLongRequest to tid=%3d complete.", tdata->tid, peer);
}
void
test_sleep(threaddata_t *tdata)
{
unsigned usecs = (unsigned) sleep_min_us +
(rand() % (sleep_max_us - sleep_min_us));
ACTION_PRINTF("tid=%3d> sleeping %.3f millisecs", tdata->tid, usecs/1000.0);
{ uint64_t goal = gasnett_ticks_to_us(gasnett_ticks_now()) + usecs;
while (gasnett_ticks_to_us(gasnett_ticks_now()) < goal)
gasnett_sched_yield();
}
ACTION_PRINTF("tid=%3d> awaking", tdata->tid);
}
void
test_amshort(threaddata_t *tdata)
{
int peer = RANDOM_PEER(tdata);
int node = tt_thread_map[peer];
ACTION_PRINTF("tid=%3d> AMShortRequest to tid=%3d", tdata->tid, peer);
tdata->flag = -1;
gasnett_local_wmb();
GASNET_Safe(gasnet_AMRequestShort1(node,
hidx_ping_shorthandler, tdata->ltid));
GASNET_BLOCKUNTIL(tdata->flag == 0);
tdata->flag = -1;
ACTION_PRINTF("tid=%3d> AMShortRequest to tid=%3d complete.", tdata->tid, peer);
}
void mpi_handler(gasnet_token_t token, harg_t tid, harg_t sz) {
gasnet_node_t node;
int mpipeer;
int tag;
char *buf;
gasnet_AMGetMsgSource(token, &node);
PRINT_AM(("node=%2d> AMShort MPI Request for tid=%i, nbytes=%i\n",
(int)gasnet_mynode(), (int)tid, (int)sz));
assert(tt_thread_map[tid] == node);
assert(sz > 0);
mpipeer = gasnetnode_to_mpirank[node];
tag = tid;
buf = (char*)test_malloc(sz);
MPI_LOCK();
assert(mpi_buf[tid] == NULL);
assert(mpi_recvhandle[tid] == MPI_REQUEST_NULL);
assert(mpi_sendhandle[tid] == MPI_REQUEST_NULL);
mpi_buf[tid] = buf;
mpi_bufsz[tid] = sz;
ACTION_PRINTF("node=%2d> setting MPI_Irecv, %i bytes\n", (int)gasnet_mynode(), (int)sz);
MPI_SAFE(MPI_Irecv(mpi_buf[tid], sz, MPI_BYTE, mpipeer, tag, MPI_COMM_WORLD, &(mpi_recvhandle[tid])));
assert(mpi_recvhandle[tid] != MPI_REQUEST_NULL);
MPI_UNLOCK();
}
void
test_ammedium(threaddata_t *tdata)
{
int peer = RANDOM_PEER(tdata);
int node = tt_thread_map[peer];
void *laddr = tt_addr_map[tdata->tid];
size_t len;
do {
len = RANDOM_SIZE();
} while (len > gasnet_AMMaxMedium());
ACTION_PRINTF("tid=%3d> AMMediumRequest (sz=%7d) to tid=%3d", tdata->tid, (int)len, peer);
tdata->flag = -1;
gasnett_local_wmb();
GASNET_Safe(gasnet_AMRequestMedium1(node,
hidx_ping_medhandler, laddr, len,
tdata->ltid));
GASNET_BLOCKUNTIL(tdata->flag == 0);
tdata->flag = -1;
ACTION_PRINTF("tid=%3d> AMMediumRequest to tid=%3d complete.", tdata->tid, peer);
}
void mpi_probehandler(gasnet_token_t token, harg_t tid) {
gasnet_node_t node;
int mpipeer;
int tag;
int reply = 0;
gasnet_AMGetMsgSource(token, &node);
assert(tt_thread_map[tid] == node);
mpipeer = gasnetnode_to_mpirank[node];
tag = tid;
MPI_LOCK();
if (mpi_recvhandle[tid] != MPI_REQUEST_NULL) {
MPI_Status status;
int flag = 0;
MPI_SAFE(MPI_Test(&mpi_recvhandle[tid],&flag,&status));
if (flag) {
int sz = mpi_bufsz[tid];
assert(mpi_recvhandle[tid] == MPI_REQUEST_NULL);
assert(mpi_sendhandle[tid] == MPI_REQUEST_NULL);
assert(mpi_buf[tid] != NULL && sz >= 0);
ACTION_PRINTF("node=%2d> sending MPI reply message, %i bytes\n", (int)gasnet_mynode(), sz);
MPI_SAFE(MPI_Isend(mpi_buf[tid], sz, MPI_BYTE, mpipeer, 10000+tag, MPI_COMM_WORLD, &(mpi_sendhandle[tid])));
assert(mpi_sendhandle[tid] != MPI_REQUEST_NULL);
}
} else if (mpi_sendhandle[tid] != MPI_REQUEST_NULL) {
MPI_Status status;
int flag = 0;
MPI_SAFE(MPI_Test(&mpi_sendhandle[tid],&flag,&status));
if (flag) {
assert(mpi_recvhandle[tid] == MPI_REQUEST_NULL);
assert(mpi_sendhandle[tid] == MPI_REQUEST_NULL);
reply = 1;
}
} else {
/* nothing to do */
}
MPI_UNLOCK();
if (reply) {
assert(mpi_buf[tid] != NULL);
test_free(mpi_buf[tid]);
mpi_buf[tid] = NULL;
PRINT_AM(("node=%2d> Sending AMShort MPI Reply for tid=%i\n",
(int)gasnet_mynode(), (int)tid));
GASNET_Safe(gasnet_AMReplyShort1(token, hidx_mpi_replyhandler, tid));
}
}
void
test_get(threaddata_t *tdata)
{
int peer = tdata->tid_peer;
int node = tt_thread_map[peer];
void *laddr = tt_addr_map[tdata->tid];
void *raddr = tt_addr_map[peer];
int len;
do {
len = RANDOM_SIZE();
} while (len > TEST_SEGZ_PER_THREAD);
ACTION_PRINTF("tid=%3d> get (%p,%8d) <- tid=%3d,node=%d,addr=%p",
tdata->tid, laddr, len, peer, node, raddr);
gasnet_get(laddr, node, raddr, len);
}
void test_mpi(threaddata_t *tdata) {
MPI_Request sendhandle = MPI_REQUEST_NULL;
MPI_Request recvhandle = MPI_REQUEST_NULL;
int peer = tdata->tid_peer;
int node = tt_thread_map[peer];
int mpipeer = gasnetnode_to_mpirank[node];
int sz;
char *sendbuf;
char *recvbuf;
int tag = tdata->tid;
int i;
do { sz = RANDOM_SIZE(); } while (sz == 0); /* some MPI's may barf on 0 byte send/recv */
sendbuf = (char*)test_malloc(sz);
recvbuf = (char*)test_malloc(sz);
for (i=0; i < MIN(sz,4096); i++) { /* randomize at least the first 4 KB */
sendbuf[i] = (char)rand();
}
ACTION_PRINTF("tid=%3d> starting MPI ping-pong with tid=%3d.\n", tdata->tid, peer);
MPI_LOCK();
ACTION_PRINTF("tid=%3d> setting MPI_Irecv, %i bytes\n", tdata->tid, sz);
MPI_SAFE(MPI_Irecv(recvbuf, sz, MPI_BYTE, mpipeer, 10000+tag, MPI_COMM_WORLD, &recvhandle));
assert(recvhandle != MPI_REQUEST_NULL);
ACTION_PRINTF("tid=%3d> sending MPI message, %i bytes\n", tdata->tid, sz);
MPI_SAFE(MPI_Isend(sendbuf, sz, MPI_BYTE, mpipeer, tag, MPI_COMM_WORLD, &sendhandle));
assert(sendhandle != MPI_REQUEST_NULL);
MPI_UNLOCK();
tdata->flag = -1;
gasnett_local_wmb();
ACTION_PRINTF("tid=%3d> MPI AMShortRequest to tid=%3d\n", tdata->tid, peer);
GASNET_Safe(gasnet_AMRequestShort2(node, hidx_mpi_handler, tdata->tid, sz));
while (tdata->flag != 0) {
ACTION_PRINTF("tid=%3d> MPI probe AMShortRequest to tid=%3d\n", tdata->tid, peer);
GASNET_Safe(gasnet_AMRequestShort1(node, hidx_mpi_probehandler, tdata->tid));
gasnett_sched_yield();
test_sleep(tdata);
GASNET_Safe(gasnet_AMPoll());
mpi_test(&sendhandle); /* occasional testing may be required for progress */
mpi_test(&recvhandle);
}
tdata->flag = -1;
mpi_complete(&sendhandle);
mpi_complete(&recvhandle);
/* verify */
for (i=0; i < sz; i++) {
if (sendbuf[i] != recvbuf[i])
FATALERR("mismatch at element %i in MPI test.", i);
}
test_free(sendbuf);
test_free(recvbuf);
ACTION_PRINTF("tid=%3d> MPI ping-pong with tid=%3d complete.\n", tdata->tid, peer);
}
