void gasnete_extref_get_reph_inner(gasnet_token_t token,
void *addr, size_t nbytes,
void *dest, void *op) {
GASNETE_FAST_UNALIGNED_MEMCPY(dest, addr, nbytes);
gasneti_sync_writes();
gasnete_op_markdone((gasnete_op_t *)op, 1);
}
void gasnete_amref_putlong_reqh_inner(gasnet_token_t token,
void *addr, size_t nbytes,
void *done) {
gasneti_sync_writes();
GASNETI_SAFE(
SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph),
PACK(done))));
}
void gasnete_amref_memset_reqh_inner(gasnet_token_t token,
gasnet_handlerarg_t val, void *nbytes_arg, void *dest, void *done) {
size_t nbytes = (uintptr_t)nbytes_arg;
memset(dest, (int)(uint32_t)val, nbytes);
gasneti_sync_writes();
GASNETI_SAFE(
SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph),
PACK(done))));
}
void gasnete_amref_put_reqh_inner(gasnet_token_t token,
void *addr, size_t nbytes,
void *dest, void *done) {
GASNETE_FAST_UNALIGNED_MEMCPY(dest, addr, nbytes);
gasneti_sync_writes();
GASNETI_SAFE(
SHORT_REP(1,2,(token, gasneti_handleridx(gasnete_amref_markdone_reph),
PACK(done))));
}
extern void gasnetc_exit(int exitcode) {
gasnetc_exit_in_progress = 1;
gasneti_sync_writes();
/* once we start a shutdown, ignore all future SIGQUIT signals or we risk reentrancy */
gasneti_reghandler(SIGQUIT, SIG_IGN);
{ /* ensure only one thread ever continues past this point */
static gasneti_mutex_t exit_lock = GASNETI_MUTEX_INITIALIZER;
gasneti_mutex_lock(&exit_lock);
}
GASNETI_TRACE_PRINTF(C,("gasnet_exit(%i)\n", exitcode));
/* Establish a last-ditch signal handler in case of failure. */
gasneti_reghandler(SIGALRM, gasnetc_exit_sighandler);
#if GASNET_DEBUG
gasneti_reghandler(SIGABRT, SIG_DFL);
#else
gasneti_reghandler(SIGABRT, gasnetc_exit_sighandler);
#endif
gasneti_reghandler(SIGILL, gasnetc_exit_sighandler);
gasneti_reghandler(SIGSEGV, gasnetc_exit_sighandler);
gasneti_reghandler(SIGFPE, gasnetc_exit_sighandler);
gasneti_reghandler(SIGBUS, gasnetc_exit_sighandler);
/* Prior to attach we cannot send AMs to coordinate the exit */
if (! gasneti_attach_done) {
fprintf(stderr, "WARNING: GASNet ofi-conduit may not shutdown cleanly when gasnet_exit() is called before gasnet_attach()\n");
gasneti_bootstrapAbort(exitcode);
gasneti_killmyprocess(exitcode);
}
const int timeout = (unsigned int)gasnetc_exittimeout;
alarm(2 + timeout);
if (gasnetc_exit_coordinate(exitcode)) {
alarm(timeout);
gasnetc_ofi_exit();
}
alarm(0);
gasneti_flush_streams();
gasneti_trace_finish();
gasneti_sched_yield();
alarm(timeout);
gasneti_bootstrapFini();
alarm(0);
gasneti_killmyprocess(exitcode);
gasneti_fatalerror("gasnetc_exit failed!");
}
extern int
firehose_remote_callback(gasnet_node_t node,
const firehose_region_t *pin_list, size_t num_pinned,
firehose_remotecallback_args_t *args)
{
#if GASNETC_PIN_SEGMENT
/* DO NOTHING. IF WE GET CALLED WE COMPLAIN. */
gasneti_fatalerror("invalid attempted to call firehose_remote_callback()");
return -1;
#else
/* Memcpy payload into place */
gasneti_assert(args != NULL);
gasneti_assert(args->addr != NULL);
gasneti_assert(args->len > 0);
gasneti_assert(args->len <= gasnetc_putinmove_limit);
memcpy(args->addr, args->data, args->len);
gasneti_sync_writes();
return 0;
#endif
}
static void
gasnete_coll_pami_allgavi(const gasnet_team_handle_t team,
void *dst, const void *src,
size_t nbytes, int flags GASNETE_THREAD_FARG)
{
int i_am_leader = gasnete_coll_pami_images_barrier(team); /* XXX: over-synced for IN_NO and IN_MY */
const gasnete_coll_threaddata_t * const td = GASNETE_COLL_MYTHREAD_NOALLOC;
if (flags & GASNET_COLL_IN_ALLSYNC) {
if (i_am_leader) gasnetc_fast_barrier();
(void) gasnete_coll_pami_images_barrier(team);
}
GASNETE_FAST_UNALIGNED_MEMCPY(gasnete_coll_scale_ptr(team->pami.scratch_space,
td->my_local_image,
nbytes),
src, nbytes);
(void) gasnete_coll_pami_images_barrier(team);
if (i_am_leader) {
volatile unsigned int done = 0;
pami_result_t rc;
pami_xfer_t op;
op = gasnete_op_template_allgavi; /* allgatherv_int */
op.cookie = (void *)&done;
op.algorithm = team->pami.allgavi_alg;
op.cmd.xfer_allgatherv_int.sndbuf = team->pami.scratch_space;
op.cmd.xfer_allgatherv_int.stypecount = nbytes * team->my_images;
op.cmd.xfer_allgatherv_int.rcvbuf = dst;
op.cmd.xfer_allgatherv_int.rtypecounts = team->pami.counts;
op.cmd.xfer_allgatherv_int.rdispls = team->pami.displs;
if (team->pami.prev_nbytes != nbytes) {
int i;
for (i = 0; i < team->total_ranks; ++i) {
op.cmd.xfer_allgatherv_int.rtypecounts[i] = nbytes * team->all_images[i];
op.cmd.xfer_allgatherv_int.rdispls[i] = nbytes * team->all_offset[i];
}
team->pami.prev_nbytes = nbytes;
}
GASNETC_PAMI_LOCK(gasnetc_context);
rc = PAMI_Collective(gasnetc_context, &op);
GASNETC_PAMI_UNLOCK(gasnetc_context);
GASNETC_PAMI_CHECK(rc, "initiating blocking allgatherv_int");
gasneti_polluntil(done);
gasneti_assert(NULL == team->pami.tmp_addr);
gasneti_sync_writes(); /* XXX: is this necessary? */
team->pami.tmp_addr = dst; /* wakes pollers, below */
(void) gasnete_coll_pami_images_barrier(team); /* matches instance below vvvv */
team->pami.tmp_addr = NULL;
} else {
gasneti_waitwhile(NULL == team->pami.tmp_addr);
GASNETE_FAST_UNALIGNED_MEMCPY(dst, team->pami.tmp_addr, nbytes * team->total_images);
(void) gasnete_coll_pami_images_barrier(team); /* matches instance above ^^^^ */
}
if (flags & GASNET_COLL_OUT_ALLSYNC) {
if (i_am_leader) gasnetc_fast_barrier();
(void) gasnete_coll_pami_images_barrier(team);
}
}
static void
gasnete_coll_pami_scattvi(const gasnet_team_handle_t team, void *dst,
gasnet_image_t srcimage, const void *src,
size_t nbytes, int flags GASNETI_THREAD_FARG)
{
const int i_am_root = gasnete_coll_image_is_local(team, srcimage);
int i_am_leader = gasnete_coll_pami_images_barrier(team); /* XXX: over-synced for IN_NO and IN_MY */
const gasnete_coll_threaddata_t * const td = GASNETE_COLL_MYTHREAD_NOALLOC;
if ((flags & GASNET_COLL_LOCAL) && i_am_root) {
/* root thread must be leader for its node */
i_am_leader = (srcimage == td->my_image);
}
if (i_am_leader) {
volatile unsigned int done = 0;
pami_result_t rc;
pami_xfer_t op;
if (flags & GASNET_COLL_IN_ALLSYNC) gasnetc_fast_barrier();
op = gasnete_op_template_scattvi; /* scatterv_int */
op.cookie = (void *)&done;
op.algorithm = team->pami.scattvi_alg;
op.cmd.xfer_scatterv_int.root = gasnetc_endpoint(GASNETE_COLL_REL2ACT(team,gasnete_coll_image_node(team, srcimage)));
op.cmd.xfer_scatterv_int.rcvbuf = team->pami.scratch_space;
op.cmd.xfer_scatterv_int.rtypecount = nbytes * team->my_images;
if (i_am_root) {
op.cmd.xfer_scatterv_int.sndbuf = (/*not-const*/ void *)src;
op.cmd.xfer_scatterv_int.stypecounts = team->pami.counts;
op.cmd.xfer_scatterv_int.sdispls = team->pami.displs;
if (team->pami.prev_nbytes != nbytes) {
int i;
for (i = 0; i < team->total_ranks; ++i) {
op.cmd.xfer_scatterv_int.stypecounts[i] = nbytes * team->all_images[i];
op.cmd.xfer_scatterv_int.sdispls[i] = nbytes * team->all_offset[i];
}
team->pami.prev_nbytes = nbytes;
}
}
GASNETC_PAMI_LOCK(gasnetc_context);
rc = PAMI_Collective(gasnetc_context, &op);
GASNETC_PAMI_UNLOCK(gasnetc_context);
GASNETC_PAMI_CHECK(rc, "initiating blocking scatterv_int");
gasneti_polluntil(done);
gasneti_assert(NULL == team->pami.tmp_addr);
gasneti_sync_writes();
team->pami.tmp_addr = team->pami.scratch_space; /* wakes pollers, below */
} else {
gasneti_waitwhile(NULL == team->pami.tmp_addr);
}
GASNETI_MEMCPY (dst,
gasnete_coll_scale_ptr(team->pami.tmp_addr,
td->my_local_image,
nbytes),
nbytes);
(void) gasnete_coll_pami_images_barrier(team);
if (i_am_leader) {
team->pami.tmp_addr = NULL;
}
if (flags & GASNET_COLL_OUT_ALLSYNC) {
if (i_am_leader) gasnetc_fast_barrier();
(void) gasnete_coll_pami_images_barrier(team);
}
}
void gasnete_extref_getlong_reph_inner(gasnet_token_t token,
void *addr, size_t nbytes,
void *op) {
gasneti_sync_writes();
gasnete_op_markdone((gasnete_op_t *)op, 1);
}
