void
gasnete_fh_request_put(void *_pop, const firehose_request_t *req,
int allLocalHit)
{
gasnete_eop_t *pop = (gasnete_eop_t *) _pop;
gasnet_node_t node;
gasneti_assert(pop != NULL);
gasneti_assert(pop->src > 0 && pop->dest > 0);
node = pop->node;
gasneti_assert(node != gasneti_mynode && node < gasneti_nodes);
gasneti_assert(pop->len > 0);
gasneti_assert(req == &(pop->req_remote));
gasneti_mutex_lock(&gasnetc_lock_gm);
gasnetc_token_lo_poll();
GASNETI_TRACE_PRINTF(C,
("Firehose directed send(%p): (%d,%p) <- (%p,%d)",
(void *) pop, (unsigned) node, (void *) pop->dest,
(void *) pop->src, pop->len));
#if GASNETI_STATS_OR_TRACE
if (!allLocalHit)
pop->fh_stats = pop->len > 4096 ? fh_many : fh_one;
#endif
GASNETC_GM_PUT(
_gmc.port, (void *) pop->src, (gm_remote_ptr_t) pop->dest,
(unsigned long) pop->len, GM_LOW_PRIORITY,
gasnetc_nodeid(node), gasnetc_portid(node),
gasnete_fh_callback_put, (void *) pop);
gasneti_mutex_unlock(&gasnetc_lock_gm);
return;
}
/* ------------------------------------------------------------------------------------ */
static int gasnetc_attach_primary(void) {
int retval = GASNET_OK;
AMLOCK();
/* pause to make sure all nodes have called attach
if a node calls gasnet_exit() between init/attach, then this allows us
to process the AMUDP_SPMD control messages required for job shutdown
*/
gasnetc_bootstrapBarrier();
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
#if GASNET_TRACE || GASNET_DEBUG
#if !GASNET_DEBUG
if (GASNETI_TRACE_ENABLED(A))
#endif
GASNETI_AM_SAFE(AMUDP_SetHandlerCallbacks(gasnetc_endpoint,
gasnetc_enteringHandler_hook, gasnetc_leavingHandler_hook));
#endif
#if GASNETC_MOCK_EVERYTHING
retval = AM_SetSeg(gasnetc_endpoint, NULL, (uintptr_t)-1);
if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed");
#endif
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasnetc_bootstrapBarrier();
AMUNLOCK();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach_primary(): primary attach complete\n"));
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
AMLOCK();
gasnetc_bootstrapBarrier();
AMUNLOCK();
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
// TODO-EX: this is a candidate for factorization (once we understand the per-conduit variations)
extern int gasnetc_attach( gex_TM_t _tm,
gasnet_handlerentry_t *table,
int numentries,
uintptr_t segsize)
{
int retval = GASNET_OK;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%"PRIuPTR")",
numentries, segsize));
gasneti_TM_t tm = gasneti_import_tm(_tm);
gasneti_EP_t ep = tm->_ep;
if (!gasneti_init_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already attached");
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize too large");
#else
segsize = 0;
#endif
/* primary attach */
if (GASNET_OK != gasnetc_attach_primary())
GASNETI_RETURN_ERRR(RESOURCE,"Error in primary attach");
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
/* register client segment */
gex_Segment_t seg; // g2ex segment is automatically saved by a hook
if (GASNET_OK != gasnetc_attach_segment(&seg, _tm, segsize, gasneti_defaultExchange, GASNETI_FLAG_INIT_LEGACY))
GASNETI_RETURN_ERRR(RESOURCE,"Error attaching segment");
#endif
AMLOCK();
/* register client handlers */
if (table && gasneti_amregister_legacy(ep->_amtbl, table, numentries) != GASNET_OK)
INITERR(RESOURCE,"Error registering handlers");
AMUNLOCK();
/* ensure everything is initialized across all nodes */
gasnet_barrier(0, GASNET_BARRIERFLAG_UNNAMED);
return GASNET_OK;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
extern void gasnetc_exit(int exitcode) {
gasnetc_exit_in_progress = 1;
gasneti_sync_writes();
/* once we start a shutdown, ignore all future SIGQUIT signals or we risk reentrancy */
gasneti_reghandler(SIGQUIT, SIG_IGN);
{ /* ensure only one thread ever continues past this point */
static gasneti_mutex_t exit_lock = GASNETI_MUTEX_INITIALIZER;
gasneti_mutex_lock(&exit_lock);
}
GASNETI_TRACE_PRINTF(C,("gasnet_exit(%i)\n", exitcode));
/* Establish a last-ditch signal handler in case of failure. */
gasneti_reghandler(SIGALRM, gasnetc_exit_sighandler);
#if GASNET_DEBUG
gasneti_reghandler(SIGABRT, SIG_DFL);
#else
gasneti_reghandler(SIGABRT, gasnetc_exit_sighandler);
#endif
gasneti_reghandler(SIGILL, gasnetc_exit_sighandler);
gasneti_reghandler(SIGSEGV, gasnetc_exit_sighandler);
gasneti_reghandler(SIGFPE, gasnetc_exit_sighandler);
gasneti_reghandler(SIGBUS, gasnetc_exit_sighandler);
/* Prior to attach we cannot send AMs to coordinate the exit */
if (! gasneti_attach_done) {
fprintf(stderr, "WARNING: GASNet ofi-conduit may not shutdown cleanly when gasnet_exit() is called before gasnet_attach()\n");
gasneti_bootstrapAbort(exitcode);
gasneti_killmyprocess(exitcode);
}
const int timeout = (unsigned int)gasnetc_exittimeout;
alarm(2 + timeout);
if (gasnetc_exit_coordinate(exitcode)) {
alarm(timeout);
gasnetc_ofi_exit();
}
alarm(0);
gasneti_flush_streams();
gasneti_trace_finish();
gasneti_sched_yield();
alarm(timeout);
gasneti_bootstrapFini();
alarm(0);
gasneti_killmyprocess(exitcode);
gasneti_fatalerror("gasnetc_exit failed!");
}
extern void gasnete_VIS_SetPeerCompletionHandler(gex_AM_Index_t handler,
const void *source_addr, size_t nbytes, gex_Flags_t flags GASNETI_THREAD_FARG) {
GASNETI_TRACE_PRINTF(A,("VIS_SetPeerCompletionHandler: handler=%i, source_addr="GASNETI_LADDRFMT", nbytes=%"PRIuSZ,
(int)handler, GASNETI_LADDRSTR(source_addr),nbytes));
gasnete_vis_threaddata_t * const td = GASNETE_VIS_MYTHREAD;
gasnete_vis_pcinfo_t * const pcinfo = &(td->pcinfo);
if (!handler) { // disarm
pcinfo->_handler = 0;
} else { // arm
gasneti_assert(nbytes <= GEX_VIS_MAX_PEERCOMPLETION);
gasneti_assert(!nbytes || source_addr);
pcinfo->_handler = handler;
pcinfo->_nbytes = nbytes;
pcinfo->_srcaddr = source_addr;
}
}
extern void gasnete_vis_init(void) {
gasneti_assert(!gasnete_vis_isinit);
gasnete_vis_isinit = 1;
GASNETI_TRACE_PRINTF(C,("gasnete_vis_init()"));
#define GASNETE_VIS_ENV_YN(varname, envname, enabler) do { \
if (enabler) { \
varname = gasneti_getenv_yesno_withdefault(#envname, enabler##_DEFAULT); \
} else if (!gasneti_mynode && gasneti_getenv(#envname) && gasneti_getenv_yesno_withdefault(#envname, 0)) { \
fprintf(stderr, "WARNING: %s is set in environment, but %s support is compiled out - setting ignored\n", \
#envname, #enabler); \
} \
} while (0)
#if !GASNETE_USE_AMPIPELINE
int gasnete_vis_use_ampipe = 0; // dummy
#endif
GASNETE_VIS_ENV_YN(gasnete_vis_use_ampipe,GASNET_VIS_AMPIPE, GASNETE_USE_AMPIPELINE);
#if GASNETE_USE_AMPIPELINE
if (gasnete_vis_use_ampipe) {
#ifndef GASNETE_VIS_MAXCHUNK_DEFAULT
#define GASNETE_VIS_MAXCHUNK_DEFAULT MIN(gex_AM_LUBRequestMedium(),gex_AM_LUBReplyMedium())-2*sizeof(void*)
#endif
#ifndef GASNETE_VIS_PUT_MAXCHUNK_DEFAULT
#define GASNETE_VIS_PUT_MAXCHUNK_DEFAULT GASNETE_VIS_MAXCHUNK_DEFAULT
#endif
#ifndef GASNETE_VIS_GET_MAXCHUNK_DEFAULT
#define GASNETE_VIS_GET_MAXCHUNK_DEFAULT GASNETE_VIS_MAXCHUNK_DEFAULT
#endif
int gasnete_vis_maxchunk_set = !!gasneti_getenv("GASNET_VIS_MAXCHUNK");
size_t gasnete_vis_maxchunk = GASNETE_VIS_MAXCHUNK_DEFAULT;
gasnete_vis_maxchunk = gasneti_getenv_int_withdefault("GASNET_VIS_MAXCHUNK", gasnete_vis_maxchunk, 1);
gasnete_vis_put_maxchunk = GASNETE_VIS_PUT_MAXCHUNK_DEFAULT;
gasnete_vis_put_maxchunk = gasneti_getenv_int_withdefault("GASNET_VIS_PUT_MAXCHUNK",
(gasnete_vis_maxchunk_set ? gasnete_vis_maxchunk : gasnete_vis_put_maxchunk), 1);
gasnete_vis_get_maxchunk = GASNETE_VIS_GET_MAXCHUNK_DEFAULT;
gasnete_vis_get_maxchunk = gasneti_getenv_int_withdefault("GASNET_VIS_GET_MAXCHUNK",
(gasnete_vis_maxchunk_set ? gasnete_vis_maxchunk : gasnete_vis_get_maxchunk), 1);
} else { // !gasnete_vis_use_ampipe
gasnete_vis_put_maxchunk = 0;
gasnete_vis_get_maxchunk = 0;
}
#endif
#if !GASNETE_USE_REMOTECONTIG_GATHER_SCATTER
int gasnete_vis_use_remotecontig = 0; // dummy
#endif
GASNETE_VIS_ENV_YN(gasnete_vis_use_remotecontig,GASNET_VIS_REMOTECONTIG, GASNETE_USE_REMOTECONTIG_GATHER_SCATTER);
}
extern void gasnetc_exit(int exitcode) {
/* once we start a shutdown, ignore all future SIGQUIT signals or we risk reentrancy */
gasneti_reghandler(SIGQUIT, SIG_IGN);
{ /* ensure only one thread ever continues past this point */
static gasneti_mutex_t exit_lock = GASNETI_MUTEX_INITIALIZER;
gasneti_mutex_lock(&exit_lock);
}
GASNETI_TRACE_PRINTF(C,("gasnet_exit(%i)\n", exitcode));
gasnetc_p4_exit();
gasneti_flush_streams();
gasneti_trace_finish();
gasneti_sched_yield();
gasneti_killmyprocess(exitcode);
gasneti_fatalerror("gasnetc_exit failed!");
}
extern void gasnetc_exit(int exitcode) {
/* once we start a shutdown, ignore all future SIGQUIT signals or we risk reentrancy */
gasneti_reghandler(SIGQUIT, SIG_IGN);
gasnetc_exitcalled = 1;
{ /* ensure only one thread ever continues past this point */
static gasneti_mutex_t exit_lock = GASNETI_MUTEX_INITIALIZER;
gasneti_mutex_lock(&exit_lock);
}
GASNETI_TRACE_PRINTF(C,("gasnet_exit(%i)\n", exitcode));
#ifdef GASNETE_EXIT_CALLBACK
/* callback for native conduits using an mpi-conduit core
this should cleanup extended API resources (only)
and then return so that MPI can be shutdown properly
*/
GASNETE_EXIT_CALLBACK(exitcode);
#endif
gasneti_flush_streams();
gasneti_trace_finish();
gasneti_sched_yield();
{ int i;
/* try to prevent races where we exit while other local pthreads are in MPI
can't use a blocking lock here, because may be in a signal context
*/
for (i=0; i < 5; i++) {
#if GASNET_DEBUG
/* ignore recursive lock attempts */
if (gasnetc_AMlock.owner == GASNETI_THREADIDQUERY()) break;
#endif
if (!gasneti_mutex_trylock(&gasnetc_AMlock)) break;
gasneti_sched_yield();
}
}
AMMPI_SPMDExit(exitcode);
gasneti_fatalerror("AMMPI_SPMDExit failed");
}
extern void gasnete_vis_init(void) {
gasneti_assert(!gasnete_vis_isinit);
gasnete_vis_isinit = 1;
GASNETI_TRACE_PRINTF(C,("gasnete_vis_init()"));
#define GASNETE_VIS_ENV_YN(varname, envname, enabler) do { \
if (enabler) { \
varname = gasneti_getenv_yesno_withdefault(#envname, enabler##_DEFAULT); \
} else if (!gasnet_mynode() && gasneti_getenv(#envname) && gasneti_getenv_yesno_withdefault(#envname, 0)) { \
fprintf(stderr, "WARNING: %s is set in environment, but %s support is compiled out - setting ignored", \
#envname, #enabler); \
} \
} while (0)
#if GASNETE_USE_AMPIPELINE
GASNETE_VIS_ENV_YN(gasnete_vis_use_ampipe,GASNET_VIS_AMPIPE, GASNETE_USE_AMPIPELINE);
gasnete_vis_maxchunk = gasneti_getenv_int_withdefault("GASNET_VIS_MAXCHUNK", gasnet_AMMaxMedium()-2*sizeof(void*),1);
#endif
#if GASNETE_USE_REMOTECONTIG_GATHER_SCATTER
GASNETE_VIS_ENV_YN(gasnete_vis_use_remotecontig,GASNET_VIS_REMOTECONTIG, GASNETE_USE_REMOTECONTIG_GATHER_SCATTER);
#endif
}
extern void gasnetc_exit(int exitcode) {
/* once we start a shutdown, ignore all future SIGQUIT signals or we risk reentrancy */
gasneti_reghandler(SIGQUIT, SIG_IGN);
gasnetc_exitcalled = 1;
{ /* ensure only one thread ever continues past this point */
static gasneti_mutex_t exit_lock = GASNETI_MUTEX_INITIALIZER;
gasneti_mutex_lock(&exit_lock);
}
GASNETI_TRACE_PRINTF(C,("gasnet_exit(%i)\n", exitcode));
gasneti_flush_streams();
gasneti_trace_finish();
gasneti_sched_yield();
/* bug2181: try to prevent races where we exit while other local pthreads are in AMUDP
can't use a blocking lock here, because may be in a signal context
*/
AMLOCK_CAUTIOUS();
AMUDP_SPMDExit(exitcode);
gasneti_fatalerror("AMUDP_SPMDExit failed!");
}
/* ------------------------------------------------------------------------------------ */
extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries,
uintptr_t segsize, uintptr_t minheapoffset) {
int retval = GASNET_OK;
void *segbase = NULL;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%lu, minheapoffset=%lu)",
numentries, (unsigned long)segsize, (unsigned long)minheapoffset));
AMLOCK();
if (!gasneti_init_done)
INITERR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
INITERR(NOT_INIT, "GASNet already attached");
/* pause to make sure all nodes have called attach
if a node calls gasnet_exit() between init/attach, then this allows us
to process the AMUDP_SPMD control messages required for job shutdown
*/
gasnetc_bootstrapBarrier();
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
INITERR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
INITERR(BAD_ARG, "segsize too large");
if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */
minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE;
#else
segsize = 0;
minheapoffset = 0;
#endif
segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */
/* ------------------------------------------------------------------------------------ */
/* register handlers */
#ifdef GASNETC_MAX_NUMHANDLERS
/* Initialize shadow handler table */
{ int i;
for (i=0; i<GASNETC_MAX_NUMHANDLERS; i++)
gasnetc_handler[i]=(gasneti_handler_fn_t)&gasneti_defaultAMHandler;
}
#endif
{ /* core API handlers */
gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(ctable);
while (ctable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(ctable, len, 1, 63, 0, &numreg) != GASNET_OK)
INITERR(RESOURCE,"Error registering core API handlers");
gasneti_assert(numreg == len);
}
{ /* extended API handlers */
gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(etable);
while (etable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(etable, len, 64, 127, 0, &numreg) != GASNET_OK)
INITERR(RESOURCE,"Error registering extended API handlers");
gasneti_assert(numreg == len);
}
if (table) { /* client handlers */
int numreg1 = 0;
int numreg2 = 0;
/* first pass - assign all fixed-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK)
INITERR(RESOURCE,"Error registering fixed-index client handlers");
/* second pass - fill in dontcare-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK)
INITERR(RESOURCE,"Error registering fixed-index client handlers");
gasneti_assert(numreg1 + numreg2 == numentries);
}
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
/* ------------------------------------------------------------------------------------ */
/* register segment */
gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t));
gasneti_leak(gasneti_seginfo);
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, &gasnetc_bootstrapExchange);
#else /* GASNET_SEGMENT_EVERYTHING */
{ int i;
for (i=0;i<gasneti_nodes;i++) {
gasneti_seginfo[i].addr = (void *)0;
gasneti_seginfo[i].size = (uintptr_t)-1;
}
}
#endif
segbase = gasneti_seginfo[gasneti_mynode].addr;
segsize = gasneti_seginfo[gasneti_mynode].size;
/* After local segment is attached, call optional client-provided hook
(###) should call BEFORE any conduit-specific pinning/registration of the segment
*/
if (gasnet_client_attach_hook) {
gasnet_client_attach_hook(segbase, segsize);
}
/* AMUDP allows arbitrary registration with no further action */
if (segsize) {
retval = AM_SetSeg(gasnetc_endpoint, segbase, segsize);
if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed");
}
#if GASNET_TRACE
if (GASNETI_TRACE_ENABLED(A))
GASNETI_AM_SAFE(AMUDP_SetHandlerCallbacks(gasnetc_endpoint,
gasnetc_enteringHandler_hook, gasnetc_leavingHandler_hook));
#endif
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasnetc_bootstrapBarrier();
AMUNLOCK();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete\n"));
gasneti_auxseg_attach(); /* provide auxseg */
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
AMLOCK();
gasnetc_bootstrapBarrier();
AMUNLOCK();
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
static int gasnetc_init(int *argc, char ***argv) {
int retval = GASNET_OK;
/* check system sanity */
gasnetc_check_config();
/* --------- begin Master code ------------ */
if (!AMUDP_SPMDIsWorker(argv?*argv:NULL)) {
/* assume we're an implicit master
(we don't currently support explicit workers spawned
without using the AMUDP SPMD API)
*/
int num_nodes;
int i;
char spawnfn;
amudp_spawnfn_t fp = (amudp_spawnfn_t)NULL;
if (!argv) {
gasneti_fatalerror("implicit-master without argv not supported - use amudprun");
}
/* pretend we're node 0, for purposes of verbose env reporting */
gasneti_init_done = 1;
gasneti_mynode = 0;
#if defined(GASNET_CSPAWN_CMD)
{ /* set configure default cspawn cmd */
const char *cmd = gasneti_getenv_withdefault("GASNET_CSPAWN_CMD",GASNET_CSPAWN_CMD);
gasneti_setenv("GASNET_CSPAWN_CMD",cmd);
}
#endif
/* parse node count from command line */
if (*argc < 2) {
fprintf(stderr, "GASNet: Missing parallel node count\n");
fprintf(stderr, "GASNet: Specify node count as first argument, or use upcrun/tcrun spawner script to start job\n");
fprintf(stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit(-1);
}
/*
* argv[1] is number of nodes; argv[0] is program name; argv is
* list of arguments including program name and number of nodes.
* We need to remove argv[1] when the argument array is passed
* to the tic_main().
*/
num_nodes = atoi((*argv)[1]);
if (num_nodes < 1) {
fprintf (stderr, "GASNet: Invalid number of nodes: %s\n", (*argv)[1]);
fprintf (stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit (1);
}
/* remove the num_nodes argument */
for (i = 1; i < (*argc)-1; i++) {
(*argv)[i] = (*argv)[i+1];
}
(*argv)[(*argc)-1] = NULL;
(*argc)--;
/* get spawnfn */
spawnfn = *gasneti_getenv_withdefault("GASNET_SPAWNFN", _STRINGIFY(GASNETC_DEFAULT_SPAWNFN));
{ /* ensure we pass the effective spawnfn to worker env */
char spawnstr[2];
spawnstr[0] = toupper(spawnfn);
spawnstr[1] = '\0';
gasneti_setenv("GASNET_SPAWNFN",spawnstr);
}
/* ensure reliable localhost operation by forcing use of 127.0.0.1
* setting GASNET_MASTERIP to the empty string will prevent this */
if (('L' == toupper(spawnfn)) && !gasneti_getenv("GASNET_MASTERIP")) {
gasneti_setenv("GASNET_MASTERIP","127.0.0.1");
}
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
if (toupper(spawnfn) == toupper(AMUDP_Spawnfn_Desc[i].abbrev)) {
fp = AMUDP_Spawnfn_Desc[i].fnptr;
break;
}
}
if (!fp) {
fprintf (stderr, "GASNet: Invalid spawn function specified in GASNET_SPAWNFN\n");
fprintf (stderr, "GASNet: The following mechanisms are available:\n");
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
fprintf(stderr, " '%c' %s\n",
toupper(AMUDP_Spawnfn_Desc[i].abbrev), AMUDP_Spawnfn_Desc[i].desc);
}
exit(1);
}
#if GASNET_DEBUG_VERBOSE
/* note - can't call trace macros during gasnet_init because trace system not yet initialized */
fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
#endif
retval = AMUDP_SPMDStartup(argc, argv,
num_nodes, 0, fp,
NULL, &gasnetc_bundle, &gasnetc_endpoint);
/* master startup should never return */
gasneti_fatalerror("master AMUDP_SPMDStartup() failed");
}
/* --------- begin Worker code ------------ */
AMLOCK();
if (gasneti_init_done)
INITERR(NOT_INIT, "GASNet already initialized");
gasneti_freezeForDebugger();
AMUDP_VerboseErrors = gasneti_VerboseErrors;
AMUDP_SPMDkillmyprocess = gasneti_killmyprocess;
/* perform job spawn */
retval = AMUDP_SPMDStartup(argc, argv,
0, 0, NULL, /* dummies */
&gasnetc_networkpid, &gasnetc_bundle, &gasnetc_endpoint);
if (retval != AM_OK) INITERR(RESOURCE, "slave AMUDP_SPMDStartup() failed");
gasneti_init_done = 1; /* enable early to allow tracing */
gasneti_conduit_getenv = (/* cast drops const */ gasneti_getenv_fn_t*)&AMUDP_SPMDgetenvMaster;
gasneti_mynode = AMUDP_SPMDMyProc();
gasneti_nodes = AMUDP_SPMDNumProcs();
/* enable tracing */
gasneti_trace_init(argc, argv);
GASNETI_AM_SAFE(AMUDP_SPMDSetExitCallback(gasnetc_traceoutput));
/* for local spawn, assume we want to wait-block */
if (gasneti_getenv("GASNET_SPAWNFN") && *gasneti_getenv("GASNET_SPAWNFN") == 'L') {
GASNETI_TRACE_PRINTF(C,("setting gasnet_set_waitmode(GASNET_WAIT_BLOCK) for localhost spawn"));
gasnet_set_waitmode(GASNET_WAIT_BLOCK);
}
#if GASNET_DEBUG_VERBOSE
fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n",
gasneti_mynode, gasneti_nodes); fflush(stderr);
#endif
gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);
#if GASNET_PSHM
gasneti_pshm_init(&gasnetc_bootstrapSNodeBroadcast, 0);
#endif
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
{ uintptr_t limit;
#if HAVE_MMAP
limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
&gasnetc_bootstrapExchange,
&gasnetc_bootstrapBarrier);
#else
limit = (intptr_t)-1;
#endif
gasneti_segmentInit(limit, &gasnetc_bootstrapExchange);
}
#elif GASNET_SEGMENT_EVERYTHING
/* segment is everything - nothing to do */
#else
#error Bad segment config
#endif
#if GASNET_BLCR
gasneti_checkpoint_guid = gasnetc_networkpid;
gasneti_checkpoint_init(NULL);
#endif
AMUNLOCK();
gasneti_auxseg_init(); /* adjust max seg values based on auxseg */
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
static int gasnetc_init(int *argc, char ***argv) {
int retval = GASNET_OK;
int networkdepth = 0;
const char *pstr = NULL;
const char *tmsgstr = NULL;
AMLOCK();
if (gasneti_init_done)
INITERR(NOT_INIT, "GASNet already initialized");
gasneti_init_done = 1; /* enable early to allow tracing */
/* check system sanity */
gasnetc_check_config();
gasneti_freezeForDebugger();
#if GASNET_DEBUG_VERBOSE
/* note - can't call trace macros during gasnet_init because trace system not yet initialized */
fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
#endif
/* choose network depth */
networkdepth = gasnett_getenv_int_withdefault("GASNET_NETWORKDEPTH", GASNETC_DEFAULT_NETWORKDEPTH, 0);
if (networkdepth <= 1) networkdepth = GASNETC_DEFAULT_NETWORKDEPTH;
AMMPI_VerboseErrors = gasneti_VerboseErrors;
AMMPI_SPMDkillmyprocess = gasneti_killmyprocess;
#if !defined(GASNETI_DISABLE_MPI_INIT_THREAD)
{ int res;
#if GASNETI_THREADS
/* tell MPI to be thread-safe */
res = AMMPI_SPMDSetThreadMode(1, &pstr, argc, argv);
#else
res = AMMPI_SPMDSetThreadMode(0, &pstr, argc, argv);
#endif
if (!res) {
#if GASNETI_THREADS
{ static char tmsg[255];
snprintf(tmsg, sizeof(tmsg),
"*** WARNING: The pthreaded version of mpi-conduit requires an MPI implementation "
"which supports threading mode MPI_THREAD_SERIALIZED, "
"but this implementation reports it can only support %s\n", pstr);
#if GASNET_DEBUG_VERBOSE
/* only show this in verbose mode, because some versions of MPICH (eg Quadrics version)
lie and report THREAD_SINGLE, when in actuality MPI_THREAD_SERIALIZED works just fine */
if (!gasneti_getenv_yesno_withdefault("GASNET_QUIET",0)) fprintf(stderr, "%s", tmsg);
#else
tmsgstr = tmsg;
#endif
}
#else
fprintf(stderr,"unknown failure in AMMPI_SPMDSetThreadMode() => %s\n",pstr);
#endif
}
}
#endif
/* perform job spawn */
retval = AMMPI_SPMDStartup(argc, argv, networkdepth, NULL, &gasnetc_bundle, &gasnetc_endpoint);
if (retval != AM_OK) INITERR(RESOURCE, "AMMPI_SPMDStartup() failed");
gasneti_mynode = AMMPI_SPMDMyProc();
gasneti_nodes = AMMPI_SPMDNumProcs();
/* a number of MPI job spawners fail to propagate the environment to all compute nodes */
/* do this before trace_init to make sure it gets right environment */
gasneti_setupGlobalEnvironment(gasneti_nodes, gasneti_mynode,
gasnetc_bootstrapExchange, gasnetc_bootstrapBroadcast);
/* enable tracing */
gasneti_trace_init(argc, argv);
GASNETI_AM_SAFE(AMMPI_SPMDSetExitCallback(gasnetc_traceoutput));
if (pstr) GASNETI_TRACE_PRINTF(C,("AMMPI_SPMDSetThreadMode/MPI_Init_thread()=>%s",pstr));
if (tmsgstr) GASNETI_TRACE_PRINTF(I,("%s",tmsgstr));
#if GASNET_DEBUG_VERBOSE
fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n",
gasneti_mynode, gasneti_nodes); fflush(stderr);
#endif
gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);
#if GASNET_PSHM
gasneti_pshm_init(&gasnetc_bootstrapExchange, 0);
#endif
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
{ uintptr_t limit;
#if HAVE_MMAP
limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
&gasnetc_bootstrapExchange,
&gasnetc_bootstrapBarrier);
#else
limit = (intptr_t)-1;
#endif
gasneti_segmentInit(limit, &gasnetc_bootstrapExchange);
}
#elif GASNET_SEGMENT_EVERYTHING
/* segment is everything - nothing to do */
#else
#error Bad segment config
#endif
AMUNLOCK();
gasneti_auxseg_init(); /* adjust max seg values based on auxseg */
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
/* ##################################################################### */
void
gasnete_fh_callback_put(struct gm_port *p, void *context,
gm_status_t status) {
#ifndef GASNET_SEGMENT_FAST
GASNET_BEGIN_FUNCTION(); /* thread cache for *_IN_UNKNOWN */
#endif
gasnete_eop_t *pop = (gasnete_eop_t *) context;
#if defined(GASNET_DEBUG) || defined(GASNETI_STATS_OR_TRACE)
gasnet_node_t node = pop->node;
gasneti_tick_t starttime = GASNETI_TICKS_NOW_IFENABLED(C);
#endif
const firehose_request_t *fhreqs[2];
int numreqs = 1;
gasneti_mutex_assertlocked(&gasnetc_lock_gm);
gasneti_assert(pop != NULL);
gasneti_assert(node != gasneti_mynode && node < gasneti_nodes);
if_pf (status != GM_SUCCESS)
gasnetc_callback_error(status, NULL);
gasnetc_token_lo_release();
GASNETI_TRACE_PRINTF(C,
("Firehose decrement remote refcount for (%p,%d) on node %d (op=%p,%p,%d)\n",
(void *) pop->dest, pop->len, (unsigned) node, (void *) pop,
(void *)pop->req_remote.addr, (int)pop->req_remote.len));
fhreqs[0] = &(pop->req_remote);
/* Puts use an ambuffer, while bulk puts send from a pinned location */
if (OPMISC(pop) == OPMISC_AMBUF) {
gasnetc_bufdesc_t *bufd;
bufd = (gasnetc_bufdesc_t *) GASNETC_BUFDESC_PTR(pop->src);
GASNETC_ASSERT_BUFDESC_PTR(bufd, pop->src);
gasnetc_callback_ambuffer(p, (void *) bufd, status);
}
else {
fhreqs[1] = pop->req_local;
numreqs++;
#ifdef GASNET_SEGMENT_FAST
/* Only release locally for "fast" segment */
firehose_release(fhreqs+1, 1);
#endif
}
#ifndef GASNET_SEGMENT_FAST
GASNETE_GM_SET_IN_UNKNOWN();
firehose_release(fhreqs, numreqs);
GASNETE_GM_UNSET_IN_UNKNOWN();
#endif
GASNETE_FIREHOSE_TRACE_PUTGET(pop, PUT);
/* If this was associated to an iop, increment put completed count */
if (pop->iop != NULL) {
gasneti_weakatomic_increment(&(pop->iop->completed_put_cnt),0/*Rel?*/);
gasneti_free(pop); /* free a "dummy" eop */
} else {
gasnete_op_markdone((gasnete_op_t *)pop, 0);
}
GASNETI_TRACE_EVENT_TIME(C, FIREHOSE_MOVE_LOCAL,
GASNETI_TICKS_NOW_IFENABLED(C)-starttime);
return;
}
extern
int
firehose_move_callback(gasnet_node_t node,
const firehose_region_t *unpin_list, size_t unpin_num,
firehose_region_t *pin_list, size_t pin_num)
{
int i;
int locked = GASNETE_GM_IN_UNKNOWN();
gm_status_t status;
if (!locked)
gasneti_mutex_lock(&gasnetc_lock_gm);
for (i = 0; i < unpin_num; i++) {
gasneti_assert(unpin_list[i].addr % GASNET_PAGESIZE == 0);
gasneti_assert(unpin_list[i].len % GASNET_PAGESIZE == 0);
status = gm_deregister_memory(_gmc.port,
(void *) unpin_list[i].addr, unpin_list[i].len);
if (status != GM_SUCCESS)
gasneti_fatalerror("gm_deregister_memory() failed for "
"page located at %p (%s)", (void*)unpin_list[i].addr,
gm_strerror(status));
GASNETI_TRACE_PRINTF(C,
("Firehose unpinlocal = %p, %d", (void *) unpin_list[i].addr,
(int)unpin_list[i].len));
}
GASNETI_TRACE_EVENT_VAL(C, FIREHOSE_LOCALUNPIN_PAGES, unpin_num);
for (i = 0; i < pin_num; i++) {
int perm_adj = 0;
retry_register:
gasneti_assert(pin_list[i].addr % GASNET_PAGESIZE == 0);
gasneti_assert(pin_list[i].len % GASNET_PAGESIZE == 0);
status = gm_register_memory(_gmc.port, (void *)
pin_list[i].addr, pin_list[i].len);
if (status != GM_SUCCESS) {
if (perm_adj) {
gasneti_fatalerror("gm_register_memory() failed for "
"page located at %p (%s)", (void*)pin_list[i].addr,
gm_strerror(status));
} else { /* bug 1036 - GM cannot register read-only data memory */
int j, num_pages = pin_list[i].len / GASNET_PAGESIZE;
char *p;
static char tst = 0;
/* check for readability before we mess with mprotect */
GASNETI_TRACE_PRINTF(C,("gm_register_memory() failed for "
"page located at %p len=%i pages (%s) -- checking readability",
(void*)pin_list[i].addr, num_pages, gm_strerror(status)));
p = (void *)pin_list[i].addr;
for (j = 0 ; j < num_pages; j++) {
tst += *p; /* if you get a seg fault here, it means firehose tried to register unmapped memory */
p += GASNET_PAGESIZE;
}
p = (void *)pin_list[i].addr;
for (j = 0 ; j < num_pages; j++) {
GASNETI_TRACE_PRINTF(C,("Attempting mprotect for page located at %p", (void*)p));
if (mprotect(p, GASNET_PAGESIZE, PROT_READ|PROT_WRITE))
gasneti_fatalerror("mprotect failed in firehose_move_callback: %s", strerror(errno));
p += GASNET_PAGESIZE;
}
perm_adj = 1;
goto retry_register;
}
}
GASNETI_TRACE_PRINTF(C,
("Firehose pinlocal = %p, %d", (void *) pin_list[i].addr,
(int)pin_list[i].len));
}
GASNETI_TRACE_EVENT_VAL(C, FIREHOSE_LOCALPIN_PAGES, pin_num);
if (!locked)
gasneti_mutex_unlock(&gasnetc_lock_gm);
return 0;
}
/* ------------------------------------------------------------------------------------ */
extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries,
uintptr_t segsize, uintptr_t minheapoffset) {
void *segbase = NULL;
int ret;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%lu, minheapoffset=%lu)",
numentries, (unsigned long)segsize, (unsigned long)minheapoffset));
if (!gasneti_init_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already attached");
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize too large");
if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */
minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE;
#else
segsize = 0;
minheapoffset = 0;
#endif
segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */
/* ------------------------------------------------------------------------------------ */
/* register handlers */
{ int i;
for (i = 0; i < GASNETC_MAX_NUMHANDLERS; i++)
gasnetc_handler[i] = (gasneti_handler_fn_t)&gasneti_defaultAMHandler;
}
{ /* core API handlers */
gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(ctable);
while (ctable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(ctable, len, 1, 63, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering core API handlers");
gasneti_assert(numreg == len);
}
{ /* extended API handlers */
gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(etable);
while (etable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(etable, len, 64, 127, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering extended API handlers");
gasneti_assert(numreg == len);
}
if (table) { /* client handlers */
int numreg1 = 0;
int numreg2 = 0;
/* first pass - assign all fixed-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering fixed-index client handlers");
/* second pass - fill in dontcare-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering fixed-index client handlers");
gasneti_assert(numreg1 + numreg2 == numentries);
}
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
/* ------------------------------------------------------------------------------------ */
/* register segment */
gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t));
gasneti_leak(gasneti_seginfo);
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if (segsize == 0) segbase = NULL; /* no segment */
else {
gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, &gasnetc_bootstrapExchange);
segbase = gasneti_seginfo[gasneti_mynode].addr;
segsize = gasneti_seginfo[gasneti_mynode].size;
gasneti_assert(((uintptr_t)segbase) % GASNET_PAGESIZE == 0);
gasneti_assert(segsize % GASNET_PAGESIZE == 0);
}
#else
{ /* GASNET_SEGMENT_EVERYTHING */
gasnet_node_t i;
for (i=0; i<gasneti_nodes; i++) {
gasneti_seginfo[i].addr = (void *)0;
gasneti_seginfo[i].size = (uintptr_t)-1;
}
segbase = (void *)0;
segsize = (uintptr_t)-1;
}
#endif
ret = gasnetc_p4_attach(segbase, segsize);
if (GASNET_OK != ret) {
GASNETI_RETURN_ERRR(RESOURCE,"Error attaching Portals4 resources");
}
/* ------------------------------------------------------------------------------------ */
/* gather segment information */
/* This was done by segmentAttach above */
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasnetc_bootstrapBarrier();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete"));
gasneti_assert(gasneti_seginfo[gasneti_mynode].addr == segbase &&
gasneti_seginfo[gasneti_mynode].size == segsize);
gasneti_auxseg_attach(); /* provide auxseg */
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
gasnetc_bootstrapBarrier();
return GASNET_OK;
}
static int gasnetc_init(int *argc, char ***argv, gex_Flags_t flags) {
int retval = GASNET_OK;
/* check system sanity */
gasnetc_check_config();
/* --------- begin Master code ------------ */
if (!AMUDP_SPMDIsWorker(argv?*argv:NULL)) {
/* assume we're an implicit master
(we don't currently support explicit workers spawned
without using the AMUDP SPMD API)
*/
int num_nodes;
int i;
char spawnfn;
amudp_spawnfn_t fp = (amudp_spawnfn_t)NULL;
if (!argv) {
gasneti_fatalerror("implicit-master without argv not supported - use amudprun");
}
/* pretend we're node 0, for purposes of verbose env reporting */
gasneti_init_done = 1;
gasneti_mynode = 0;
#if defined(GASNET_CSPAWN_CMD)
{ /* set configure default cspawn cmd */
const char *cmd = gasneti_getenv_withdefault("GASNET_CSPAWN_CMD",GASNET_CSPAWN_CMD);
gasneti_setenv("GASNET_CSPAWN_CMD",cmd);
}
#endif
/* parse node count from command line */
if (*argc < 2) {
fprintf(stderr, "GASNet: Missing parallel node count\n");
fprintf(stderr, "GASNet: Specify node count as first argument, or use upcrun/tcrun spawner script to start job\n");
fprintf(stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit(-1);
}
/*
* argv[1] is number of nodes; argv[0] is program name; argv is
* list of arguments including program name and number of nodes.
* We need to remove argv[1] when the argument array is passed
* to the tic_main().
*/
num_nodes = atoi((*argv)[1]);
if (num_nodes < 1) {
fprintf (stderr, "GASNet: Invalid number of nodes: %s\n", (*argv)[1]);
fprintf (stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit (1);
}
/* remove the num_nodes argument */
for (i = 1; i < (*argc)-1; i++) {
(*argv)[i] = (*argv)[i+1];
}
(*argv)[(*argc)-1] = NULL;
(*argc)--;
/* get spawnfn */
spawnfn = *gasneti_getenv_withdefault("GASNET_SPAWNFN", _STRINGIFY(GASNETC_DEFAULT_SPAWNFN));
{ /* ensure we pass the effective spawnfn to worker env */
char spawnstr[2];
spawnstr[0] = toupper(spawnfn);
spawnstr[1] = '\0';
gasneti_setenv("GASNET_SPAWNFN",spawnstr);
}
/* ensure reliable localhost operation by forcing use of 127.0.0.1
* setting GASNET_MASTERIP to the empty string will prevent this */
if (('L' == toupper(spawnfn)) && !gasneti_getenv("GASNET_MASTERIP")) {
gasneti_setenv("GASNET_MASTERIP","127.0.0.1");
}
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
if (toupper(spawnfn) == toupper(AMUDP_Spawnfn_Desc[i].abbrev)) {
fp = AMUDP_Spawnfn_Desc[i].fnptr;
break;
}
}
if (!fp) {
fprintf (stderr, "GASNet: Invalid spawn function specified in GASNET_SPAWNFN\n");
fprintf (stderr, "GASNet: The following mechanisms are available:\n");
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
fprintf(stderr, " '%c' %s\n",
toupper(AMUDP_Spawnfn_Desc[i].abbrev), AMUDP_Spawnfn_Desc[i].desc);
}
exit(1);
}
#if GASNET_DEBUG_VERBOSE
/* note - can't call trace macros during gasnet_init because trace system not yet initialized */
fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
#endif
retval = AMUDP_SPMDStartup(argc, argv,
num_nodes, 0, fp,
NULL, &gasnetc_bundle, &gasnetc_endpoint);
/* master startup should never return */
gasneti_fatalerror("master AMUDP_SPMDStartup() failed");
}
/* --------- begin Worker code ------------ */
AMLOCK();
if (gasneti_init_done)
INITERR(NOT_INIT, "GASNet already initialized");
gasneti_freezeForDebugger();
AMX_VerboseErrors = gasneti_VerboseErrors;
AMUDP_SPMDkillmyprocess = gasneti_killmyprocess;
#if GASNETI_CALIBRATE_TSC
// Early x86*/Linux timer initialization before AMUDP_SPMDStartup()
//
// udp-conduit does not support user-provided values for GASNET_TSC_RATE*
// (which fine-tune timer calibration on x86/Linux). This is partially due
// to a dependency cycle at startup with envvar propagation, but more
// importantly because the retransmission algorithm (and hence all conduit
// comms) rely on gasnet timers to be accurate (at least approximately), so
// we don't allow the user to weaken or disable their calibration.
gasneti_unsetenv("GASNET_TSC_RATE");
gasneti_unsetenv("GASNET_TSC_RATE_TOLERANCE");
gasneti_unsetenv("GASNET_TSC_RATE_HARD_TOLERANCE");
GASNETI_TICKS_INIT();
#endif
/* perform job spawn */
retval = AMUDP_SPMDStartup(argc, argv,
0, 0, NULL, /* dummies */
&gasnetc_networkpid, &gasnetc_bundle, &gasnetc_endpoint);
if (retval != AM_OK) INITERR(RESOURCE, "slave AMUDP_SPMDStartup() failed");
gasneti_init_done = 1; /* enable early to allow tracing */
gasneti_getenv_hook = (/* cast drops const */ gasneti_getenv_fn_t*)&AMUDP_SPMDgetenvMaster;
gasneti_mynode = AMUDP_SPMDMyProc();
gasneti_nodes = AMUDP_SPMDNumProcs();
#if !GASNETI_CALIBRATE_TSC
/* Must init timers after global env, and preferably before tracing */
GASNETI_TICKS_INIT();
#endif
/* enable tracing */
gasneti_trace_init(argc, argv);
GASNETI_AM_SAFE(AMUDP_SPMDSetExitCallback(gasnetc_traceoutput));
/* for local spawn, assume we want to wait-block */
if (gasneti_getenv("GASNET_SPAWNFN") && *gasneti_getenv("GASNET_SPAWNFN") == 'L') {
GASNETI_TRACE_PRINTF(C,("setting gasnet_set_waitmode(GASNET_WAIT_BLOCK) for localhost spawn"));
gasnet_set_waitmode(GASNET_WAIT_BLOCK);
}
#if GASNET_DEBUG_VERBOSE
fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n",
gasneti_mynode, gasneti_nodes); fflush(stderr);
#endif
gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);
#if GASNET_PSHM
gasneti_pshm_init(&gasnetc_bootstrapSNodeBroadcast, 0);
#endif
uintptr_t mmap_limit;
#if HAVE_MMAP
mmap_limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
&gasnetc_bootstrapExchange,
&gasnetc_bootstrapBarrier);
#else
// TODO-EX: we can at least look at rlimits but such logic belongs in conduit-indep code
mmap_limit = (intptr_t)-1;
#endif
/* allocate and attach an aux segment */
gasneti_auxsegAttach(mmap_limit, &gasnetc_bootstrapExchange);
/* determine Max{Local,GLobal}SegmentSize */
gasneti_segmentInit(mmap_limit, &gasnetc_bootstrapExchange, flags);
#if GASNET_BLCR
gasneti_checkpoint_guid = gasnetc_networkpid;
gasneti_checkpoint_init(NULL);
#endif
AMUNLOCK();
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
/* ------------------------------------------------------------------------------------ */
extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries,
uintptr_t segsize, uintptr_t minheapoffset) {
void *segbase = NULL;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%"PRIuPTR", minheapoffset=%"PRIuPTR")",
numentries, segsize, minheapoffset));
if (!gasneti_init_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already attached");
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize too large");
if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */
minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE;
#else
segsize = 0;
minheapoffset = 0;
#endif
segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */
/* ------------------------------------------------------------------------------------ */
/* register handlers */
{ int i;
for (i = 0; i < GASNETC_MAX_NUMHANDLERS; i++)
gasnetc_handler[i] = (gasneti_handler_fn_t)&gasneti_defaultAMHandler;
}
{ /* core API handlers */
gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(ctable);
while (ctable[len].fnptr) len++; /* calc len */
if (gasneti_amregister(ctable, len, 1, 63, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering core API handlers");
gasneti_assert(numreg == len);
}
{ /* extended API handlers */
gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(etable);
while (etable[len].fnptr) len++; /* calc len */
if (gasneti_amregister(etable, len, 64, 127, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering extended API handlers");
gasneti_assert(numreg == len);
}
if (table) { /* client handlers */
int numreg1 = 0;
int numreg2 = 0;
/* first pass - assign all fixed-index handlers */
if (gasneti_amregister(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering fixed-index client handlers");
/* second pass - fill in dontcare-index handlers */
if (gasneti_amregister(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering variable-index client handlers");
gasneti_assert(numreg1 + numreg2 == numentries);
}
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
/* ------------------------------------------------------------------------------------ */
/* setup fo rexit coordination */
gasnetc_exittimeout = gasneti_get_exittimeout(GASNETC_DEFAULT_EXITTIMEOUT_MAX,
GASNETC_DEFAULT_EXITTIMEOUT_MIN,
GASNETC_DEFAULT_EXITTIMEOUT_FACTOR,
GASNETC_DEFAULT_EXITTIMEOUT_MIN);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
/* ------------------------------------------------------------------------------------ */
/* register segment */
gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t));
gasneti_leak(gasneti_seginfo);
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if (segsize == 0) segbase = NULL; /* no segment */
else {
gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, gasneti_bootstrapExchange);
segbase = gasneti_seginfo[gasneti_mynode].addr;
segsize = gasneti_seginfo[gasneti_mynode].size;
gasneti_assert(((uintptr_t)segbase) % GASNET_PAGESIZE == 0);
gasneti_assert(segsize % GASNET_PAGESIZE == 0);
}
#else
{ /* GASNET_SEGMENT_EVERYTHING */
gasnet_node_t i;
for (i=0; i<gasneti_nodes; i++) {
gasneti_seginfo[i].addr = (void *)0;
gasneti_seginfo[i].size = (uintptr_t)-1;
}
segbase = (void *)0;
segsize = (uintptr_t)-1;
}
#endif
gasnetc_ofi_attach(segbase, segsize);
/* After local segment is attached, call optional client-provided hook
(###) should call BEFORE any conduit-specific pinning/registration of the segment
*/
if (gasnet_client_attach_hook) {
gasnet_client_attach_hook(segbase, segsize);
}
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasneti_bootstrapBarrier();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete"));
gasneti_assert(gasneti_seginfo[gasneti_mynode].addr == segbase &&
gasneti_seginfo[gasneti_mynode].size == segsize);
/* (###) exchange_fn is optional (may be NULL) and is only used with GASNET_SEGMENT_EVERYTHING
if your conduit has an optimized bootstrapExchange pass it in place of NULL
*/
gasneti_auxseg_attach(NULL); /* provide auxseg */
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
gasneti_bootstrapBarrier();
return GASNET_OK;
}
gex_Event_t gasnete_VIS_pcwrap(gasnete_synctype_t const synctype, // manifest constant
gex_TM_t tm, gex_Rank_t rank, gex_Flags_t flags,
gex_Event_t const evt GASNETI_THREAD_FARG) {
gasnete_vis_threaddata_t * const td = GASNETE_VIS_MYTHREAD;
gasnete_vis_pcinfo_t * const pcinfo = &(td->pcinfo);
gasneti_assert(pcinfo->_handler);
GASNETI_TRACE_PRINTF(D,("VIS_PeerCompletionHandler scheduled: handler=%i, source_addr="GASNETI_LADDRFMT", nbytes=%"PRIuSZ,
(int)pcinfo->_handler, GASNETI_LADDRSTR(pcinfo->_srcaddr), pcinfo->_nbytes));
if (evt == GEX_EVENT_INVALID || // synchronously complete
(synctype == gasnete_synctype_b && (gex_Event_Wait(evt),1))) { // blocking
gex_Event_t lc;
gex_Event_t *lc_opt;
switch (synctype) {
case gasnete_synctype_b: lc_opt = GEX_EVENT_NOW; lc = GEX_EVENT_INVALID; break;
case gasnete_synctype_nb: lc_opt = &lc; break;
case gasnete_synctype_nbi: lc_opt = GEX_EVENT_GROUP; lc = GEX_EVENT_INVALID; break;
default: gasneti_unreachable();
}
gex_AM_RequestMedium1(tm, rank, _hidx_gasnete_vis_pcthunk_reqh, (void *)(pcinfo->_srcaddr), pcinfo->_nbytes, lc_opt, 0, pcinfo->_handler);
pcinfo->_handler = 0; // reset
return lc;
} else { // schedule deferred initiator-chaining
gasneti_vispc_op_t * const vispcop = gasneti_malloc(sizeof(gasneti_vispc_op_t)+GEX_VIS_MAX_PEERCOMPLETION);
gasneti_vis_op_t * const visop = &(vispcop->visop);
vispcop->tm = tm;
vispcop->rank = rank;
visop->type = GASNETI_VIS_CAT_PUTPC_CHAIN;
visop->count = pcinfo->_handler;
pcinfo->_handler = 0; // reset
visop->len = pcinfo->_nbytes;
visop->event = evt;
if (flags & GEX_FLAG_ENABLE_LEAF_LC) { // client also requesting LC
#if GASNETE_HAVE_LC
vispcop->lc = gex_Event_QueryLeaf(evt, GEX_EC_LC);
// TODO-EX: remove this event_internal vileness
// set ALC in-flight for the client's op
#define VISOP_EXTRA do { \
if (vispcop->lc) { \
if (synctype == gasnete_synctype_nb) GASNETE_EOP_LC_START((gasnete_eop_t *)(visop->eop)); \
else GASNETE_IOP_LC_START((gasnete_iop_t *)visop->iop); \
} \
} while (0)
#else
#if GASNET_DEBUG
vispcop->lc = GEX_EVENT_INVALID; // for assertions only
#endif
#define VISOP_EXTRA ((void)0)
#endif
// bounce-buffer the PC payload if the client requested LC
// this is REQUIRED for conduits lacking ALC signalling support, to correctly implement synchronous LC
// this is an optimization for other conduits, to ensure we can report VIS payload LC to the client when it happens
void *pcpayload = vispcop+1;
GASNETI_MEMCPY_SAFE_EMPTY(pcpayload, pcinfo->_srcaddr, pcinfo->_nbytes);
visop->addr = pcpayload;
} else {
visop->addr = (void*)pcinfo->_srcaddr;
vispcop->lc = GEX_EVENT_INVALID;
}
GASNETE_PUSH_VISOP_RETURN(td, visop, synctype, 0, VISOP_EXTRA);
#undef VISOP_EXTRA
}
}
