extern void gasnete_amref_get_nbi_bulk (void *dest, gasnet_node_t node, void *src, size_t nbytes GASNETE_THREAD_FARG) {
gasnete_threaddata_t * const mythread = GASNETE_MYTHREAD;
gasnete_iop_t * const op = mythread->current_iop;
GASNETI_CHECKPSHM_GET(UNALIGNED,V);
if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) {
op->initiated_get_cnt++;
GASNETI_SAFE(
SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_amref_get_reqh),
(gasnet_handlerarg_t)nbytes, PACK(dest), PACK(src), PACK_IOP_DONE(op,get))));
return;
} else {
int chunksz;
gasnet_handler_t reqhandler;
uint8_t *psrc = src;
uint8_t *pdest = dest;
#if GASNETE_USE_LONG_GETS
gasneti_memcheck(gasneti_seginfo);
if (gasneti_in_segment(gasneti_mynode, dest, nbytes)) {
chunksz = gasnet_AMMaxLongReply();
reqhandler = gasneti_handleridx(gasnete_amref_getlong_reqh);
}
else
#endif
{ reqhandler = gasneti_handleridx(gasnete_amref_get_reqh);
chunksz = gasnet_AMMaxMedium();
}
for (;;) {
op->initiated_get_cnt++;
if (nbytes > chunksz) {
GASNETI_SAFE(
SHORT_REQ(4,7,(node, reqhandler,
(gasnet_handlerarg_t)chunksz, PACK(pdest), PACK(psrc), PACK_IOP_DONE(op,get))));
nbytes -= chunksz;
psrc += chunksz;
pdest += chunksz;
} else {
GASNETI_SAFE(
SHORT_REQ(4,7,(node, reqhandler,
(gasnet_handlerarg_t)nbytes, PACK(pdest), PACK(psrc), PACK_IOP_DONE(op,get))));
break;
}
}
return;
}
}
void gasnete_amref_putlong_reqh_inner(gasnet_token_t token,
void *addr, size_t nbytes,
void *done) {
gasneti_sync_writes();
GASNETI_SAFE(
SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph),
PACK(done))));
}
void gasnete_amref_getlong_reqh_inner(gasnet_token_t token,
gasnet_handlerarg_t nbytes, void *dest, void *src, void *done) {
GASNETI_SAFE(
LONG_REP(1,2,(token, gasneti_handleridx(gasnete_amref_getlong_reph),
src, nbytes, dest,
PACK(done))));
}
void gasnete_amref_get_reqh_inner(gasnet_token_t token,
gasnet_handlerarg_t nbytes, void *dest, void *src, void *done) {
gasneti_assert(nbytes <= gasnet_AMMaxMedium());
GASNETI_SAFE(
MEDIUM_REP(2,4,(token, gasneti_handleridx(gasnete_amref_get_reph),
src, nbytes,
PACK(dest), PACK(done))));
}
void gasnete_amref_memset_reqh_inner(gasnet_token_t token,
gasnet_handlerarg_t val, void *nbytes_arg, void *dest, void *done) {
size_t nbytes = (uintptr_t)nbytes_arg;
memset(dest, (int)(uint32_t)val, nbytes);
gasneti_sync_writes();
GASNETI_SAFE(
SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph),
PACK(done))));
}
void gasnete_amref_put_reqh_inner(gasnet_token_t token,
void *addr, size_t nbytes,
void *dest, void *done) {
GASNETE_FAST_UNALIGNED_MEMCPY(dest, addr, nbytes);
gasneti_sync_writes();
GASNETI_SAFE(
SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph),
PACK(done))));
}
void gasnete_extref_memset_nbi (gasnet_node_t node, void *dest, int val, size_t nbytes GASNETE_THREAD_FARG) {
gasnete_threaddata_t * const mythread = GASNETE_MYTHREAD;
gasnete_iop_t *op = mythread->current_iop;
op->initiated_put_cnt++;
GASNETI_SAFE(
SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_extref_memset_reqh),
(gasnet_handlerarg_t)val, PACK(nbytes),
PACK(dest), PACK(op))));
}
/* (note this relies on the fact that our implementation of access regions allows recursion) */
gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS);
#if 0 /* No AM-based Bulk Puts in gm-conduit */
if (isbulk) gasnete_extref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS);
else gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS);
#else
gasneti_assert(!isbulk);
gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS);
#endif
return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE);
}
}
gasnet_handle_t gasnete_extref_put_nb (gasnet_node_t node, void *dest, void *src, size_t nbytes GASNETE_THREAD_FARG) {
return gasnete_extref_put_nb_inner(node, dest, src, nbytes, 0 GASNETE_THREAD_PASS);
}
#if 0 /* UNUSED: No AM-based Bulk Puts in gm-conduit */
gasnet_handle_t gasnete_extref_put_nb_bulk (gasnet_node_t node, void *dest, void *src, size_t nbytes GASNETE_THREAD_FARG) {
return gasnete_extref_put_nb_inner(node, dest, src, nbytes, 1 GASNETE_THREAD_PASS);
}
#endif /* UNUSED */
gasnet_handle_t gasnete_extref_memset_nb (gasnet_node_t node, void *dest, int val, size_t nbytes GASNETE_THREAD_FARG) {
gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD);
GASNETI_SAFE(
SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_extref_memset_reqh),
(gasnet_handlerarg_t)val, PACK(nbytes),
PACK(dest), PACK(op))));
return (gasnet_handle_t)op;
}
gasnet_handle_t gasnete_amref_put_nb_inner(gasnet_node_t node, void *dest, void *src, size_t nbytes, int isbulk GASNETE_THREAD_FARG) {
if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) {
gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD);
GASNETI_SAFE(
MEDIUM_REQ(2,4,(node, gasneti_handleridx(gasnete_amref_put_reqh),
src, nbytes,
PACK(dest), PACK_EOP_DONE(op))));
return (gasnet_handle_t)op;
} else if (nbytes <= gasnet_AMMaxLongRequest()) {
gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD);
if (isbulk) {
GASNETI_SAFE(
LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
src, nbytes, dest,
PACK_EOP_DONE(op))));
} else {
GASNETI_SAFE(
LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
src, nbytes, dest,
PACK_EOP_DONE(op))));
}
return (gasnet_handle_t)op;
} else {
/* TODO: don't need the iop for large xfers in the GASNETE_EOP_COUNTED case */
/* need many messages - use an access region to coalesce them into a single handle */
/* (note this relies on the fact that our implementation of access regions allows recursion) */
gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS);
#if GASNETE_BUILD_AMREF_PUT_BULK && GASNETE_BUILD_AMREF_PUT
if (isbulk) gasnete_amref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS);
else gasnete_amref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS);
#elif GASNETE_BUILD_AMREF_PUT_BULK
gasnete_amref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS);
#else
gasnete_amref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS);
#endif
return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE);
}
}
gasnet_handle_t gasnete_extref_put_nb_inner(gasnet_node_t node, void *dest, void *src, size_t nbytes, int isbulk GASNETE_THREAD_FARG) {
if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) {
gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD);
GASNETI_SAFE(
MEDIUM_REQ(2,4,(node, gasneti_handleridx(gasnete_extref_put_reqh),
src, nbytes,
PACK(dest), PACK(op))));
return (gasnet_handle_t)op;
} else if (nbytes <= gasnet_AMMaxLongRequest()) {
gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD);
if (isbulk) {
GASNETI_SAFE(
LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_extref_putlong_reqh),
src, nbytes, dest,
PACK(op))));
} else {
GASNETI_SAFE(
LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_extref_putlong_reqh),
src, nbytes, dest,
PACK(op))));
}
return (gasnet_handle_t)op;
} else {
/* need many messages - use an access region to coalesce them into a single handle */
/* (note this relies on the fact that our implementation of access regions allows recursion) */
gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS);
#if 0 /* No AM-based Bulk Puts in gm-conduit */
if (isbulk) gasnete_extref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS);
else gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS);
#else
gasneti_assert(!isbulk);
gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS);
#endif
return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE);
}
}
extern gasnet_handle_t gasnete_amref_memset_nb (gasnet_node_t node, void *dest, int val, size_t nbytes GASNETE_THREAD_FARG) {
GASNETI_CHECKPSHM_MEMSET(H);
{
gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD);
GASNETI_SAFE(
SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_amref_memset_reqh),
(gasnet_handlerarg_t)val, PACK(nbytes),
PACK(dest), PACK_EOP_DONE(op))));
return (gasnet_handle_t)op;
}
}
/* ------------------------------------------------------------------------------------ */
#if 0 /* UNUSED: No AM-based Gets in gm-conduit */
gasnet_handle_t gasnete_extref_get_nb_bulk (void *dest, gasnet_node_t node, void *src, size_t nbytes GASNETE_THREAD_FARG) {
if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) {
gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD);
GASNETI_SAFE(
SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_extref_get_reqh),
(gasnet_handlerarg_t)nbytes, PACK(dest), PACK(src), PACK(op))));
return (gasnet_handle_t)op;
} else {
/* need many messages - use an access region to coalesce them into a single handle */
/* (note this relies on the fact that our implementation of access regions allows recursion) */
gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS);
gasnete_extref_get_nbi_bulk(dest, node, src, nbytes GASNETE_THREAD_PASS);
return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE);
}
}
/* Coordinate a global exit, returning non-zero on success */
static int gasnetc_exit_coordinate(int exitcode) {
/* Disable processing of user's AMs, to avoid reentrance if user's handler exits */
for (int i = GASNETE_HANDLER_BASE; i < GASNETC_MAX_NUMHANDLERS; ++i) {
gasnetc_handler[i] = (gasneti_handler_fn_t)&gasnetc_noop;
}
/* Coordinate using dissemination-pattern, with timeout.
* lg(N) rounds each of which sends and recvs 1 AM
*/
const uint64_t timeout_ns = gasnetc_exittimeout * 1000000000L;
const gasneti_tick_t t_start = gasneti_ticks_now();
for (int distance = 1; distance < gasneti_nodes; distance *= 2) {
int peer = (gasneti_mynode + distance) % gasneti_nodes;
int ret = gasnetc_AMRequestShortM(peer, gasneti_handleridx(gasnetc_exit_reqh),
2, exitcode, distance);
if (ret != GASNET_OK) return 0;
do { /* wait for completion of the proper receive, which might arrive out of order */
if (timeout_ns < gasneti_ticks_to_ns(gasneti_ticks_now() - t_start)) return 0;
gasnetc_AMPoll();
} while (!(distance & gasneti_atomic_read(&gasnetc_exit_dist, 0)));
}
return 1;
}
void gasnete_amref_put_nbi_inner(gasnet_node_t node, void *dest, void *src, size_t nbytes, int isbulk GASNETE_THREAD_FARG) {
gasnete_threaddata_t * const mythread = GASNETE_MYTHREAD;
gasnete_iop_t * const op = mythread->current_iop;
if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) {
op->initiated_put_cnt++;
GASNETI_SAFE(
MEDIUM_REQ(2,4,(node, gasneti_handleridx(gasnete_amref_put_reqh),
src, nbytes,
PACK(dest), PACK_IOP_DONE(op,put))));
return;
} else if (nbytes <= gasnet_AMMaxLongRequest()) {
op->initiated_put_cnt++;
if (isbulk) {
GASNETI_SAFE(
LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
src, nbytes, dest,
PACK_IOP_DONE(op,put))));
} else {
GASNETI_SAFE(
LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
src, nbytes, dest,
PACK_IOP_DONE(op,put))));
}
return;
} else {
int chunksz = gasnet_AMMaxLongRequest();
uint8_t *psrc = src;
uint8_t *pdest = dest;
for (;;) {
op->initiated_put_cnt++;
if (nbytes > chunksz) {
if (isbulk) {
GASNETI_SAFE(
LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
psrc, chunksz, pdest,
PACK_IOP_DONE(op,put))));
} else {
GASNETI_SAFE(
LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
psrc, chunksz, pdest,
PACK_IOP_DONE(op,put))));
}
nbytes -= chunksz;
psrc += chunksz;
pdest += chunksz;
} else {
if (isbulk) {
GASNETI_SAFE(
LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
psrc, nbytes, pdest,
PACK_IOP_DONE(op,put))));
} else {
GASNETI_SAFE(
LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh),
psrc, nbytes, pdest,
PACK_IOP_DONE(op,put))));
}
break;
}
}
return;
}
}