void XMP_gasnet_atomic_sync_memory()
{
for(int i=0;i<_atomic_operations;i++)
GASNET_BLOCKUNTIL(_xmp_gasnet_atomic_queue[i] == _XMP_ATOMIC_DONE);
_atomic_operations = 0;
}
void
test_amlong(threaddata_t *tdata)
{
int peer = RANDOM_PEER(tdata);
int node = tt_thread_map[peer];
void *laddr = tt_addr_map[tdata->tid];
void *raddr = tt_addr_map[peer];
size_t len;
do {
len = RANDOM_SIZE();
} while ((len > gasnet_AMMaxLongRequest()) || (len > gasnet_AMMaxLongReply())
|| (len > TEST_SEGZ_PER_THREAD));
tdata->flag = -1;
gasnett_local_wmb();
ACTION_PRINTF("tid=%3d> AMLongRequest (sz=%7d) to tid=%3d", tdata->tid, (int)len, peer);
GASNET_Safe(gasnet_AMRequestLong2(node,
hidx_ping_longhandler, laddr, len, raddr,
tdata->ltid, peer));
GASNET_BLOCKUNTIL(tdata->flag == 0);
tdata->flag = -1;
ACTION_PRINTF("tid=%3d> AMLongRequest to tid=%3d complete.", tdata->tid, peer);
}
/* ------------------------------------------------------------------------------------ */
void *doAll(void *ptr) {
if (ptr) {
if (recvr) GASNET_BLOCKUNTIL(done);
} else {
doAMShort();
doAMMed();
doAMLong();
if (recvr) gex_AM_RequestShort0(myteam, mynode, hidx_done_shorthandler, 0);
}
return NULL;
}
/* ------------------------------------------------------------------------------------ */
void *doAll(void *ptr) {
if (ptr) {
if (recvr) GASNET_BLOCKUNTIL(done);
} else {
doAMShort();
doAMMed();
doAMLong();
doAMLongAsync();
if (recvr) GASNET_Safe(gasnet_AMRequestShort0(mynode, hidx_done_shorthandler));
}
return NULL;
}
void
test_amshort(threaddata_t *tdata)
{
int peer = RANDOM_PEER(tdata);
int node = tt_thread_map[peer];
ACTION_PRINTF("tid=%3d> AMShortRequest to tid=%3d", tdata->tid, peer);
tdata->flag = -1;
gasnett_local_wmb();
GASNET_Safe(gasnet_AMRequestShort1(node,
hidx_ping_shorthandler, tdata->ltid));
GASNET_BLOCKUNTIL(tdata->flag == 0);
tdata->flag = -1;
ACTION_PRINTF("tid=%3d> AMShortRequest to tid=%3d complete.", tdata->tid, peer);
}
void
test_ammedium(threaddata_t *tdata)
{
int peer = RANDOM_PEER(tdata);
int node = tt_thread_map[peer];
void *laddr = tt_addr_map[tdata->tid];
size_t len;
do {
len = RANDOM_SIZE();
} while (len > gasnet_AMMaxMedium());
ACTION_PRINTF("tid=%3d> AMMediumRequest (sz=%7d) to tid=%3d", tdata->tid, (int)len, peer);
tdata->flag = -1;
gasnett_local_wmb();
GASNET_Safe(gasnet_AMRequestMedium1(node,
hidx_ping_medhandler, laddr, len,
tdata->ltid));
GASNET_BLOCKUNTIL(tdata->flag == 0);
tdata->flag = -1;
ACTION_PRINTF("tid=%3d> AMMediumRequest to tid=%3d complete.", tdata->tid, peer);
}
/* ------------------------------------------------------------------------------------ */
void doAMShort(void) {
GASNET_BEGIN_FUNCTION();
if (sender) { /* warm-up */
flag = 0;
for (i=0; i < iters; i++) {
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_ping_shorthandler_flood));
}
GASNET_BLOCKUNTIL(flag == iters);
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_ping_shorthandler));
GASNET_BLOCKUNTIL(flag == iters+1);
}
BARRIER();
/* ------------------------------------------------------------------------------------ */
if (TEST_SECTION_BEGIN_ENABLED() && sender) {
int64_t start = TIME();
flag = -1;
for (i=0; i < iters; i++) {
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_ping_shorthandler));
GASNET_BLOCKUNTIL(flag == i);
}
report(" AMShort ping-pong roundtrip ReqRep",TIME() - start, iters, 0, 1);
}
BARRIER();
/* ------------------------------------------------------------------------------------ */
if (TEST_SECTION_ENABLED()) {
int64_t start = TIME();
flag = -1;
BARRIER();
if (sender && recvr) {
assert(peer == mynode);
for (i=0; i < iters; i++) {
int lim = i << 1;
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_pong_shorthandler));
GASNET_BLOCKUNTIL(flag == lim);
lim++;
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_pong_shorthandler));
GASNET_BLOCKUNTIL(flag == lim);
}
} else if (sender) {
for (i=0; i < iters; i++) {
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_pong_shorthandler));
GASNET_BLOCKUNTIL(flag == i);
}
} else if (recvr) {
for (i=0; i < iters; i++) {
GASNET_BLOCKUNTIL(flag == i);
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_pong_shorthandler));
}
}
report(" AMShort ping-pong roundtrip ReqReq",TIME() - start, iters, 0, 1);
if (mynode == 0) { printf("\n"); fflush(stdout); }
}
BARRIER();
/* ------------------------------------------------------------------------------------ */
if (TEST_SECTION_ENABLED()) {
if (sender) {
int64_t start = TIME();
flag = 0;
BARRIER();
for (i=0; i < iters; i++) {
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_pong_shorthandler_flood));
}
if (recvr) GASNET_BLOCKUNTIL(flag == iters);
BARRIER();
report(" AMShort flood one-way Req",TIME() - start, iters, 0, 0);
} else {
flag = 0;
BARRIER();
GASNET_BLOCKUNTIL(flag == iters);
BARRIER();
}
if (mynode == 0) { printf("\n"); fflush(stdout); }
}
BARRIER();
/* ------------------------------------------------------------------------------------ */
if (TEST_SECTION_ENABLED() && sender) {
int64_t start = TIME();
flag = 0;
for (i=0; i < iters; i++) {
GASNET_Safe(gasnet_AMRequestShort0(peer, hidx_ping_shorthandler_flood));
}
GASNET_BLOCKUNTIL(flag == iters);
report(" AMShort flood roundtrip ReqRep",TIME() - start, iters, 0, 1);
if (mynode == 0) { printf("\n"); fflush(stdout); }
}
BARRIER();
}
void *doit(void *id) {
gasnett_threadkey_set(mythread,id);
if ((uintptr_t)id != 0) { /* additional threads polling, to encourage handler concurrency */
while (!done) {
gasnet_AMPoll();
gasnett_sched_yield();
}
return 0;
}
MSG0("Running %sAM%s%s%s%s correctness test %s%swith %i iterations, max_payload=%i, depth=%i...",
#if GASNET_PAR
(domultith?"multi-threaded ":"single-threaded "),
#else
"",
#endif
(amopt?(domed?" Medium":""):""),(amopt?(dolong?" Long":""):""),(amopt?(dolongasync?" LongAsync":""):""),
((doinseg^dooutseg)?(doinseg?" in-segment":" out-of-segment"):""),
(dosizesync?"":"loosely-synced "),
(doprime?"with priming ":""),
iters,max_payload,depth);
BARRIER();
if (doprime) { /* issue some initial puts that cover the Long regions, to try and trigger dynamic pinning */
int chunkidx;
for (chunkidx = 0; chunkidx < depth; chunkidx++) {
/* AMRequestLong primer */
gasnet_put(peerproc, peerreqseg+chunkidx*max_payload, privateseg+chunkidx*max_payload, max_payload);
gasnet_put(peerproc, peerreqseg+chunkidx*max_payload, localseg+chunkidx*max_payload, max_payload);
/* AMRequestLongAsync primer */
gasnet_put(peerproc, peerreqseg+(depth+chunkidx)*max_payload, privateseg+chunkidx*max_payload, max_payload);
gasnet_put(peerproc, peerreqseg+(depth+chunkidx)*max_payload, localseg+chunkidx*max_payload, max_payload);
/* AMReplyLong primer */
gasnet_put(peerproc, peerrepseg+chunkidx*max_payload, myseg+chunkidx*max_payload, max_payload);
gasnet_put(peerproc, peerrepseg+chunkidx*max_payload, longreplysrc+chunkidx*max_payload, max_payload);
/* AMReplyLongAsync primer */
gasnet_put(peerproc, peerrepseg+(depth+chunkidx)*max_payload, myseg+(depth+chunkidx)*max_payload, max_payload);
gasnet_put(peerproc, peerrepseg+(depth+chunkidx)*max_payload, alongreplysrc+chunkidx*max_payload, max_payload);
}
BARRIER();
}
{ int sz,iter,savesz = 1;
int max1 = gasnet_AMMaxMedium(), max2 = maxlong;
if (maxlong < gasnet_AMMaxMedium()) { max1 = maxlong; max2 = gasnet_AMMaxMedium(); }
assert_always(max1 <= max2);
for (sz = 1; sz <= max_payload; ) {
if (dosizesync) BARRIER(); /* optional barrier, to synchronize tests at each payload size across nodes */
MSG0("payload = %i",sz);
for (iter = 0; iter < iters; iter++) {
int chunkidx;
uint8_t *srcseg = ITERSEG(iter);
/* initialize local seg to known values */
for (chunkidx = 0; chunkidx < depth; chunkidx++) {
init_chunk(srcseg,sz,iter,chunkidx);
}
if (domed && sz <= gasnet_AMMaxMedium()) { /* test Medium AMs */
gasnett_atomic_set(&pong_recvd,0,0);
for (chunkidx = 0; chunkidx < depth; chunkidx++) {
GASNET_Safe(gasnet_AMRequestMedium2(peerproc, hidx_ping_medhandler, srcseg+chunkidx*sz, sz,
iter, chunkidx));
}
/* wait for completion */
GASNET_BLOCKUNTIL(gasnett_atomic_read(&pong_recvd,0) == depth);
}
if (sz <= maxlong) {
if (dolong) { /* test Long AMs */
gasnett_atomic_set(&pong_recvd,0,0);
for (chunkidx = 0; chunkidx < depth; chunkidx++) {
GASNET_Safe(gasnet_AMRequestLong2(peerproc, hidx_ping_longhandler, srcseg+chunkidx*sz, sz,
peerreqseg+chunkidx*sz, iter, chunkidx));
}
/* wait for completion */
GASNET_BLOCKUNTIL(gasnett_atomic_read(&pong_recvd,0) == depth);
}
if (dolongasync) { /* test AsyncLong AMs */
gasnett_atomic_set(&pong_recvd,0,0);
for (chunkidx = 0; chunkidx < depth; chunkidx++) {
GASNET_Safe(gasnet_AMRequestLongAsync2(peerproc, hidx_ping_alonghandler, srcseg+chunkidx*sz, sz,
peerreqseg+(depth+chunkidx)*sz, iter, chunkidx));
}
/* wait for completion */
GASNET_BLOCKUNTIL(gasnett_atomic_read(&pong_recvd,0) == depth);
}
}
}
/* double sz each time, but make sure to also exactly hit MaxMedium, MaxLong and max payload */
if (sz < max1 && savesz * 2 > max1) sz = max1;
else if (sz < max2 && savesz * 2 > max2) sz = max2;
else if (sz < max_payload && savesz * 2 > max_payload) sz = max_payload;
else { sz = savesz * 2; savesz = sz; }
}
}
BARRIER();
done = 1;
return(0);
}
static void * doTest(void *arg) {
int warmups = MIN(100, iters/100);
int64_t start,total;
int result;
int i = 0;
if (arg) {
/* I am a polling thread */
GASNET_BLOCKUNTIL(done);
return NULL;
}
/* Warmup Named */
for (i=0; i < warmups; i++) {
my_barrier_notify(i, 0);
GASNET_Safe(my_barrier_wait(i, 0));
assert_always(!gasnet_barrier_result(&result));
}
BARRIER();
start = TIME();
for (i=0; i < iters; i++) {
my_barrier_notify(i, 0);
GASNET_Safe(my_barrier_wait(i, 0));
}
total = TIME() - start;
BARRIER();
if (mynode == 0) {
printf("Total time: %8.3f sec Avg Named Barrier latency: %8.3f us\n",
((float)total)/1000000, ((float)total)/iters);
fflush(stdout);
}
BARRIER();
/* Warmup Anonymous */
for (i=0; i < warmups; i++) {
my_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS);
GASNET_Safe(my_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS));
assert_always(gasnet_barrier_result(&result));
}
BARRIER();
start = TIME();
for (i=0; i < iters; i++) {
my_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS);
GASNET_Safe(my_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS));
}
total = TIME() - start;
BARRIER();
if (mynode == 0) {
printf("Total time: %8.3f sec Avg Anon. Barrier latency: %8.3f us\n",
((float)total)/1000000, ((float)total)/iters);
fflush(stdout);
}
BARRIER();
if (!PERFORM_MIXED_NAMED_ANON_TESTS) {
if (mynode == 0) {
MSG("WARNING: skipping tests which mix named and anonymous barriers, "
"which are known to fail in this configuration");
}
} else {
int parity = (mynode & 1);
/* Warmup Mixed */
for (i=0; i < warmups; i++, parity ^= 1) {
int value = parity ? iters : 0;
int flags = parity ? 0 : GASNET_BARRIERFLAG_ANONYMOUS;
my_barrier_notify(value, flags);
GASNET_Safe(my_barrier_wait(value, flags));
assert_always(!gasnet_barrier_result(&result) || (nodes == 1));
}
BARRIER();
start = TIME();
for (i=0; i < iters; i++, parity ^= 1) {
int value = parity ? iters : 0;
int flags = parity ? 0 : GASNET_BARRIERFLAG_ANONYMOUS;
my_barrier_notify(value, flags);
GASNET_Safe(my_barrier_wait(value, flags));
}
total = TIME() - start;
BARRIER();
if (mynode == 0) {
printf("Total time: %8.3f sec Avg Mixed Barrier latency: %8.3f us\n",
((float)total)/1000000, ((float)total)/iters);
fflush(stdout);
}
}
BARRIER();
#if TEST_UNNAMED_BARRIER
/* Warmup Unnamed */
for (i=0; i < warmups; i++) {
my_barrier_notify(0, GASNET_BARRIERFLAG_UNNAMED);
GASNET_Safe(my_barrier_wait(0, GASNET_BARRIERFLAG_UNNAMED));
assert_always(gasnet_barrier_result(&result));
}
BARRIER();
start = TIME();
for (i=0; i < iters; i++) {
my_barrier_notify(0, GASNET_BARRIERFLAG_UNNAMED);
GASNET_Safe(my_barrier_wait(0, GASNET_BARRIERFLAG_UNNAMED));
}
total = TIME() - start;
BARRIER();
if (mynode == 0) {
printf("Total time: %8.3f sec Avg Uname Barrier latency: %8.3f us\n",
((float)total)/1000000, ((float)total)/iters);
fflush(stdout);
}
BARRIER();
#endif
GASNET_Safe(gasnet_AMRequestShort0(mynode, hidx_done_shorthandler));
return NULL;
}
