extern void gasnete_amref_get_nbi_bulk (void *dest, gasnet_node_t node, void *src, size_t nbytes GASNETE_THREAD_FARG) { gasnete_threaddata_t * const mythread = GASNETE_MYTHREAD; gasnete_iop_t * const op = mythread->current_iop; GASNETI_CHECKPSHM_GET(UNALIGNED,V); if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) { op->initiated_get_cnt++; GASNETI_SAFE( SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_amref_get_reqh), (gasnet_handlerarg_t)nbytes, PACK(dest), PACK(src), PACK_IOP_DONE(op,get)))); return; } else { int chunksz; gasnet_handler_t reqhandler; uint8_t *psrc = src; uint8_t *pdest = dest; #if GASNETE_USE_LONG_GETS gasneti_memcheck(gasneti_seginfo); if (gasneti_in_segment(gasneti_mynode, dest, nbytes)) { chunksz = gasnet_AMMaxLongReply(); reqhandler = gasneti_handleridx(gasnete_amref_getlong_reqh); } else #endif { reqhandler = gasneti_handleridx(gasnete_amref_get_reqh); chunksz = gasnet_AMMaxMedium(); } for (;;) { op->initiated_get_cnt++; if (nbytes > chunksz) { GASNETI_SAFE( SHORT_REQ(4,7,(node, reqhandler, (gasnet_handlerarg_t)chunksz, PACK(pdest), PACK(psrc), PACK_IOP_DONE(op,get)))); nbytes -= chunksz; psrc += chunksz; pdest += chunksz; } else { GASNETI_SAFE( SHORT_REQ(4,7,(node, reqhandler, (gasnet_handlerarg_t)nbytes, PACK(pdest), PACK(psrc), PACK_IOP_DONE(op,get)))); break; } } return; } }
void gasnete_amref_putlong_reqh_inner(gasnet_token_t token, void *addr, size_t nbytes, void *done) { gasneti_sync_writes(); GASNETI_SAFE( SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph), PACK(done)))); }
void gasnete_amref_getlong_reqh_inner(gasnet_token_t token, gasnet_handlerarg_t nbytes, void *dest, void *src, void *done) { GASNETI_SAFE( LONG_REP(1,2,(token, gasneti_handleridx(gasnete_amref_getlong_reph), src, nbytes, dest, PACK(done)))); }
void gasnete_amref_get_reqh_inner(gasnet_token_t token, gasnet_handlerarg_t nbytes, void *dest, void *src, void *done) { gasneti_assert(nbytes <= gasnet_AMMaxMedium()); GASNETI_SAFE( MEDIUM_REP(2,4,(token, gasneti_handleridx(gasnete_amref_get_reph), src, nbytes, PACK(dest), PACK(done)))); }
void gasnete_amref_memset_reqh_inner(gasnet_token_t token, gasnet_handlerarg_t val, void *nbytes_arg, void *dest, void *done) { size_t nbytes = (uintptr_t)nbytes_arg; memset(dest, (int)(uint32_t)val, nbytes); gasneti_sync_writes(); GASNETI_SAFE( SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph), PACK(done)))); }
void gasnete_amref_put_reqh_inner(gasnet_token_t token, void *addr, size_t nbytes, void *dest, void *done) { GASNETE_FAST_UNALIGNED_MEMCPY(dest, addr, nbytes); gasneti_sync_writes(); GASNETI_SAFE( SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph), PACK(done)))); }
void gasnete_extref_memset_nbi (gasnet_node_t node, void *dest, int val, size_t nbytes GASNETE_THREAD_FARG) { gasnete_threaddata_t * const mythread = GASNETE_MYTHREAD; gasnete_iop_t *op = mythread->current_iop; op->initiated_put_cnt++; GASNETI_SAFE( SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_extref_memset_reqh), (gasnet_handlerarg_t)val, PACK(nbytes), PACK(dest), PACK(op)))); }
/* (note this relies on the fact that our implementation of access regions allows recursion) */ gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS); #if 0 /* No AM-based Bulk Puts in gm-conduit */ if (isbulk) gasnete_extref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS); else gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS); #else gasneti_assert(!isbulk); gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS); #endif return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE); } } gasnet_handle_t gasnete_extref_put_nb (gasnet_node_t node, void *dest, void *src, size_t nbytes GASNETE_THREAD_FARG) { return gasnete_extref_put_nb_inner(node, dest, src, nbytes, 0 GASNETE_THREAD_PASS); } #if 0 /* UNUSED: No AM-based Bulk Puts in gm-conduit */ gasnet_handle_t gasnete_extref_put_nb_bulk (gasnet_node_t node, void *dest, void *src, size_t nbytes GASNETE_THREAD_FARG) { return gasnete_extref_put_nb_inner(node, dest, src, nbytes, 1 GASNETE_THREAD_PASS); } #endif /* UNUSED */ gasnet_handle_t gasnete_extref_memset_nb (gasnet_node_t node, void *dest, int val, size_t nbytes GASNETE_THREAD_FARG) { gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD); GASNETI_SAFE( SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_extref_memset_reqh), (gasnet_handlerarg_t)val, PACK(nbytes), PACK(dest), PACK(op)))); return (gasnet_handle_t)op; }
gasnet_handle_t gasnete_amref_put_nb_inner(gasnet_node_t node, void *dest, void *src, size_t nbytes, int isbulk GASNETE_THREAD_FARG) { if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) { gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD); GASNETI_SAFE( MEDIUM_REQ(2,4,(node, gasneti_handleridx(gasnete_amref_put_reqh), src, nbytes, PACK(dest), PACK_EOP_DONE(op)))); return (gasnet_handle_t)op; } else if (nbytes <= gasnet_AMMaxLongRequest()) { gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD); if (isbulk) { GASNETI_SAFE( LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), src, nbytes, dest, PACK_EOP_DONE(op)))); } else { GASNETI_SAFE( LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), src, nbytes, dest, PACK_EOP_DONE(op)))); } return (gasnet_handle_t)op; } else { /* TODO: don't need the iop for large xfers in the GASNETE_EOP_COUNTED case */ /* need many messages - use an access region to coalesce them into a single handle */ /* (note this relies on the fact that our implementation of access regions allows recursion) */ gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS); #if GASNETE_BUILD_AMREF_PUT_BULK && GASNETE_BUILD_AMREF_PUT if (isbulk) gasnete_amref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS); else gasnete_amref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS); #elif GASNETE_BUILD_AMREF_PUT_BULK gasnete_amref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS); #else gasnete_amref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS); #endif return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE); } }
gasnet_handle_t gasnete_extref_put_nb_inner(gasnet_node_t node, void *dest, void *src, size_t nbytes, int isbulk GASNETE_THREAD_FARG) { if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) { gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD); GASNETI_SAFE( MEDIUM_REQ(2,4,(node, gasneti_handleridx(gasnete_extref_put_reqh), src, nbytes, PACK(dest), PACK(op)))); return (gasnet_handle_t)op; } else if (nbytes <= gasnet_AMMaxLongRequest()) { gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD); if (isbulk) { GASNETI_SAFE( LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_extref_putlong_reqh), src, nbytes, dest, PACK(op)))); } else { GASNETI_SAFE( LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_extref_putlong_reqh), src, nbytes, dest, PACK(op)))); } return (gasnet_handle_t)op; } else { /* need many messages - use an access region to coalesce them into a single handle */ /* (note this relies on the fact that our implementation of access regions allows recursion) */ gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS); #if 0 /* No AM-based Bulk Puts in gm-conduit */ if (isbulk) gasnete_extref_put_nbi_bulk(node, dest, src, nbytes GASNETE_THREAD_PASS); else gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS); #else gasneti_assert(!isbulk); gasnete_extref_put_nbi (node, dest, src, nbytes GASNETE_THREAD_PASS); #endif return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE); } }
extern gasnet_handle_t gasnete_amref_memset_nb (gasnet_node_t node, void *dest, int val, size_t nbytes GASNETE_THREAD_FARG) { GASNETI_CHECKPSHM_MEMSET(H); { gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD); GASNETI_SAFE( SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_amref_memset_reqh), (gasnet_handlerarg_t)val, PACK(nbytes), PACK(dest), PACK_EOP_DONE(op)))); return (gasnet_handle_t)op; } }
/* ------------------------------------------------------------------------------------ */ #if 0 /* UNUSED: No AM-based Gets in gm-conduit */ gasnet_handle_t gasnete_extref_get_nb_bulk (void *dest, gasnet_node_t node, void *src, size_t nbytes GASNETE_THREAD_FARG) { if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) { gasnete_eop_t *op = gasnete_eop_new(GASNETE_MYTHREAD); GASNETI_SAFE( SHORT_REQ(4,7,(node, gasneti_handleridx(gasnete_extref_get_reqh), (gasnet_handlerarg_t)nbytes, PACK(dest), PACK(src), PACK(op)))); return (gasnet_handle_t)op; } else { /* need many messages - use an access region to coalesce them into a single handle */ /* (note this relies on the fact that our implementation of access regions allows recursion) */ gasnete_begin_nbi_accessregion(1 /* enable recursion */ GASNETE_THREAD_PASS); gasnete_extref_get_nbi_bulk(dest, node, src, nbytes GASNETE_THREAD_PASS); return gasnete_end_nbi_accessregion(GASNETE_THREAD_PASS_ALONE); } }
/* Coordinate a global exit, returning non-zero on success */ static int gasnetc_exit_coordinate(int exitcode) { /* Disable processing of user's AMs, to avoid reentrance if user's handler exits */ for (int i = GASNETE_HANDLER_BASE; i < GASNETC_MAX_NUMHANDLERS; ++i) { gasnetc_handler[i] = (gasneti_handler_fn_t)&gasnetc_noop; } /* Coordinate using dissemination-pattern, with timeout. * lg(N) rounds each of which sends and recvs 1 AM */ const uint64_t timeout_ns = gasnetc_exittimeout * 1000000000L; const gasneti_tick_t t_start = gasneti_ticks_now(); for (int distance = 1; distance < gasneti_nodes; distance *= 2) { int peer = (gasneti_mynode + distance) % gasneti_nodes; int ret = gasnetc_AMRequestShortM(peer, gasneti_handleridx(gasnetc_exit_reqh), 2, exitcode, distance); if (ret != GASNET_OK) return 0; do { /* wait for completion of the proper receive, which might arrive out of order */ if (timeout_ns < gasneti_ticks_to_ns(gasneti_ticks_now() - t_start)) return 0; gasnetc_AMPoll(); } while (!(distance & gasneti_atomic_read(&gasnetc_exit_dist, 0))); } return 1; }
void gasnete_amref_put_nbi_inner(gasnet_node_t node, void *dest, void *src, size_t nbytes, int isbulk GASNETE_THREAD_FARG) { gasnete_threaddata_t * const mythread = GASNETE_MYTHREAD; gasnete_iop_t * const op = mythread->current_iop; if (nbytes <= GASNETE_GETPUT_MEDIUM_LONG_THRESHOLD) { op->initiated_put_cnt++; GASNETI_SAFE( MEDIUM_REQ(2,4,(node, gasneti_handleridx(gasnete_amref_put_reqh), src, nbytes, PACK(dest), PACK_IOP_DONE(op,put)))); return; } else if (nbytes <= gasnet_AMMaxLongRequest()) { op->initiated_put_cnt++; if (isbulk) { GASNETI_SAFE( LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), src, nbytes, dest, PACK_IOP_DONE(op,put)))); } else { GASNETI_SAFE( LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), src, nbytes, dest, PACK_IOP_DONE(op,put)))); } return; } else { int chunksz = gasnet_AMMaxLongRequest(); uint8_t *psrc = src; uint8_t *pdest = dest; for (;;) { op->initiated_put_cnt++; if (nbytes > chunksz) { if (isbulk) { GASNETI_SAFE( LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), psrc, chunksz, pdest, PACK_IOP_DONE(op,put)))); } else { GASNETI_SAFE( LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), psrc, chunksz, pdest, PACK_IOP_DONE(op,put)))); } nbytes -= chunksz; psrc += chunksz; pdest += chunksz; } else { if (isbulk) { GASNETI_SAFE( LONGASYNC_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), psrc, nbytes, pdest, PACK_IOP_DONE(op,put)))); } else { GASNETI_SAFE( LONG_REQ(1,2,(node, gasneti_handleridx(gasnete_amref_putlong_reqh), psrc, nbytes, pdest, PACK_IOP_DONE(op,put)))); } break; } } return; } }