/* ------------------------------------------------------------------------------------ */
static int gasnetc_attach_primary(void) {
int retval = GASNET_OK;
AMLOCK();
/* pause to make sure all nodes have called attach
if a node calls gasnet_exit() between init/attach, then this allows us
to process the AMUDP_SPMD control messages required for job shutdown
*/
gasnetc_bootstrapBarrier();
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
#if GASNET_TRACE || GASNET_DEBUG
#if !GASNET_DEBUG
if (GASNETI_TRACE_ENABLED(A))
#endif
GASNETI_AM_SAFE(AMUDP_SetHandlerCallbacks(gasnetc_endpoint,
gasnetc_enteringHandler_hook, gasnetc_leavingHandler_hook));
#endif
#if GASNETC_MOCK_EVERYTHING
retval = AM_SetSeg(gasnetc_endpoint, NULL, (uintptr_t)-1);
if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed");
#endif
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasnetc_bootstrapBarrier();
AMUNLOCK();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach_primary(): primary attach complete\n"));
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
AMLOCK();
gasnetc_bootstrapBarrier();
AMUNLOCK();
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
extern int gasnetc_AMPoll(void) {
int retval;
GASNETI_CHECKATTACH();
#if GASNET_PSHM
gasneti_AMPSHMPoll(0);
#endif
AMLOCK();
GASNETI_AM_SAFE_NORETURN(retval,AM_Poll(gasnetc_bundle));
AMUNLOCK();
if_pf (retval) GASNETI_RETURN_ERR(RESOURCE);
else return GASNET_OK;
// TODO-EX: this is a candidate for factorization (once we understand the per-conduit variations)
extern int gasnetc_attach( gex_TM_t _tm,
gasnet_handlerentry_t *table,
int numentries,
uintptr_t segsize)
{
int retval = GASNET_OK;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%"PRIuPTR")",
numentries, segsize));
gasneti_TM_t tm = gasneti_import_tm(_tm);
gasneti_EP_t ep = tm->_ep;
if (!gasneti_init_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already attached");
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize too large");
#else
segsize = 0;
#endif
/* primary attach */
if (GASNET_OK != gasnetc_attach_primary())
GASNETI_RETURN_ERRR(RESOURCE,"Error in primary attach");
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
/* register client segment */
gex_Segment_t seg; // g2ex segment is automatically saved by a hook
if (GASNET_OK != gasnetc_attach_segment(&seg, _tm, segsize, gasneti_defaultExchange, GASNETI_FLAG_INIT_LEGACY))
GASNETI_RETURN_ERRR(RESOURCE,"Error attaching segment");
#endif
AMLOCK();
/* register client handlers */
if (table && gasneti_amregister_legacy(ep->_amtbl, table, numentries) != GASNET_OK)
INITERR(RESOURCE,"Error registering handlers");
AMUNLOCK();
/* ensure everything is initialized across all nodes */
gasnet_barrier(0, GASNET_BARRIERFLAG_UNNAMED);
return GASNET_OK;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
extern int gasnetc_AMPoll(GASNETI_THREAD_FARG_ALONE) {
int retval;
GASNETI_CHECKATTACH();
#if GASNET_PSHM
gasneti_AMPSHMPoll(0 GASNETI_THREAD_PASS);
#endif
AMLOCK();
// In single-nbrhd case never need to poll the network for client AMs.
// However, we'll still check for control traffic for orderly exit handling.
if (gasneti_mysupernode.grp_count > 1) {
GASNETI_AM_SAFE_NORETURN(retval,AM_Poll(gasnetc_bundle));
} else {
// TODO-EX: a lock-free peek would allow elimination of a lock cycle
GASNETI_AM_SAFE_NORETURN(retval,AMUDP_SPMDHandleControlTraffic(NULL));
}
AMUNLOCK();
if_pf (retval) GASNETI_RETURN_ERR(RESOURCE);
else return GASNET_OK;
/* ------------------------------------------------------------------------------------ */
extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries,
uintptr_t segsize, uintptr_t minheapoffset) {
int retval = GASNET_OK;
void *segbase = NULL;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%lu, minheapoffset=%lu)",
numentries, (unsigned long)segsize, (unsigned long)minheapoffset));
AMLOCK();
if (!gasneti_init_done)
INITERR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
INITERR(NOT_INIT, "GASNet already attached");
/* pause to make sure all nodes have called attach
if a node calls gasnet_exit() between init/attach, then this allows us
to process the AMUDP_SPMD control messages required for job shutdown
*/
gasnetc_bootstrapBarrier();
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
INITERR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
INITERR(BAD_ARG, "segsize too large");
if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */
minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE;
#else
segsize = 0;
minheapoffset = 0;
#endif
segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */
/* ------------------------------------------------------------------------------------ */
/* register handlers */
#ifdef GASNETC_MAX_NUMHANDLERS
/* Initialize shadow handler table */
{ int i;
for (i=0; i<GASNETC_MAX_NUMHANDLERS; i++)
gasnetc_handler[i]=(gasneti_handler_fn_t)&gasneti_defaultAMHandler;
}
#endif
{ /* core API handlers */
gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(ctable);
while (ctable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(ctable, len, 1, 63, 0, &numreg) != GASNET_OK)
INITERR(RESOURCE,"Error registering core API handlers");
gasneti_assert(numreg == len);
}
{ /* extended API handlers */
gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(etable);
while (etable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(etable, len, 64, 127, 0, &numreg) != GASNET_OK)
INITERR(RESOURCE,"Error registering extended API handlers");
gasneti_assert(numreg == len);
}
if (table) { /* client handlers */
int numreg1 = 0;
int numreg2 = 0;
/* first pass - assign all fixed-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK)
INITERR(RESOURCE,"Error registering fixed-index client handlers");
/* second pass - fill in dontcare-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK)
INITERR(RESOURCE,"Error registering fixed-index client handlers");
gasneti_assert(numreg1 + numreg2 == numentries);
}
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
/* ------------------------------------------------------------------------------------ */
/* register segment */
gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t));
gasneti_leak(gasneti_seginfo);
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, &gasnetc_bootstrapExchange);
#else /* GASNET_SEGMENT_EVERYTHING */
{ int i;
for (i=0;i<gasneti_nodes;i++) {
gasneti_seginfo[i].addr = (void *)0;
gasneti_seginfo[i].size = (uintptr_t)-1;
}
}
#endif
segbase = gasneti_seginfo[gasneti_mynode].addr;
segsize = gasneti_seginfo[gasneti_mynode].size;
/* After local segment is attached, call optional client-provided hook
(###) should call BEFORE any conduit-specific pinning/registration of the segment
*/
if (gasnet_client_attach_hook) {
gasnet_client_attach_hook(segbase, segsize);
}
/* AMUDP allows arbitrary registration with no further action */
if (segsize) {
retval = AM_SetSeg(gasnetc_endpoint, segbase, segsize);
if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed");
}
#if GASNET_TRACE
if (GASNETI_TRACE_ENABLED(A))
GASNETI_AM_SAFE(AMUDP_SetHandlerCallbacks(gasnetc_endpoint,
gasnetc_enteringHandler_hook, gasnetc_leavingHandler_hook));
#endif
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasnetc_bootstrapBarrier();
AMUNLOCK();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete\n"));
gasneti_auxseg_attach(); /* provide auxseg */
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
AMLOCK();
gasnetc_bootstrapBarrier();
AMUNLOCK();
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
static int gasnetc_init(int *argc, char ***argv) {
int retval = GASNET_OK;
/* check system sanity */
gasnetc_check_config();
/* --------- begin Master code ------------ */
if (!AMUDP_SPMDIsWorker(argv?*argv:NULL)) {
/* assume we're an implicit master
(we don't currently support explicit workers spawned
without using the AMUDP SPMD API)
*/
int num_nodes;
int i;
char spawnfn;
amudp_spawnfn_t fp = (amudp_spawnfn_t)NULL;
if (!argv) {
gasneti_fatalerror("implicit-master without argv not supported - use amudprun");
}
/* pretend we're node 0, for purposes of verbose env reporting */
gasneti_init_done = 1;
gasneti_mynode = 0;
#if defined(GASNET_CSPAWN_CMD)
{ /* set configure default cspawn cmd */
const char *cmd = gasneti_getenv_withdefault("GASNET_CSPAWN_CMD",GASNET_CSPAWN_CMD);
gasneti_setenv("GASNET_CSPAWN_CMD",cmd);
}
#endif
/* parse node count from command line */
if (*argc < 2) {
fprintf(stderr, "GASNet: Missing parallel node count\n");
fprintf(stderr, "GASNet: Specify node count as first argument, or use upcrun/tcrun spawner script to start job\n");
fprintf(stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit(-1);
}
/*
* argv[1] is number of nodes; argv[0] is program name; argv is
* list of arguments including program name and number of nodes.
* We need to remove argv[1] when the argument array is passed
* to the tic_main().
*/
num_nodes = atoi((*argv)[1]);
if (num_nodes < 1) {
fprintf (stderr, "GASNet: Invalid number of nodes: %s\n", (*argv)[1]);
fprintf (stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit (1);
}
/* remove the num_nodes argument */
for (i = 1; i < (*argc)-1; i++) {
(*argv)[i] = (*argv)[i+1];
}
(*argv)[(*argc)-1] = NULL;
(*argc)--;
/* get spawnfn */
spawnfn = *gasneti_getenv_withdefault("GASNET_SPAWNFN", _STRINGIFY(GASNETC_DEFAULT_SPAWNFN));
{ /* ensure we pass the effective spawnfn to worker env */
char spawnstr[2];
spawnstr[0] = toupper(spawnfn);
spawnstr[1] = '\0';
gasneti_setenv("GASNET_SPAWNFN",spawnstr);
}
/* ensure reliable localhost operation by forcing use of 127.0.0.1
* setting GASNET_MASTERIP to the empty string will prevent this */
if (('L' == toupper(spawnfn)) && !gasneti_getenv("GASNET_MASTERIP")) {
gasneti_setenv("GASNET_MASTERIP","127.0.0.1");
}
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
if (toupper(spawnfn) == toupper(AMUDP_Spawnfn_Desc[i].abbrev)) {
fp = AMUDP_Spawnfn_Desc[i].fnptr;
break;
}
}
if (!fp) {
fprintf (stderr, "GASNet: Invalid spawn function specified in GASNET_SPAWNFN\n");
fprintf (stderr, "GASNet: The following mechanisms are available:\n");
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
fprintf(stderr, " '%c' %s\n",
toupper(AMUDP_Spawnfn_Desc[i].abbrev), AMUDP_Spawnfn_Desc[i].desc);
}
exit(1);
}
#if GASNET_DEBUG_VERBOSE
/* note - can't call trace macros during gasnet_init because trace system not yet initialized */
fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
#endif
retval = AMUDP_SPMDStartup(argc, argv,
num_nodes, 0, fp,
NULL, &gasnetc_bundle, &gasnetc_endpoint);
/* master startup should never return */
gasneti_fatalerror("master AMUDP_SPMDStartup() failed");
}
/* --------- begin Worker code ------------ */
AMLOCK();
if (gasneti_init_done)
INITERR(NOT_INIT, "GASNet already initialized");
gasneti_freezeForDebugger();
AMUDP_VerboseErrors = gasneti_VerboseErrors;
AMUDP_SPMDkillmyprocess = gasneti_killmyprocess;
/* perform job spawn */
retval = AMUDP_SPMDStartup(argc, argv,
0, 0, NULL, /* dummies */
&gasnetc_networkpid, &gasnetc_bundle, &gasnetc_endpoint);
if (retval != AM_OK) INITERR(RESOURCE, "slave AMUDP_SPMDStartup() failed");
gasneti_init_done = 1; /* enable early to allow tracing */
gasneti_conduit_getenv = (/* cast drops const */ gasneti_getenv_fn_t*)&AMUDP_SPMDgetenvMaster;
gasneti_mynode = AMUDP_SPMDMyProc();
gasneti_nodes = AMUDP_SPMDNumProcs();
/* enable tracing */
gasneti_trace_init(argc, argv);
GASNETI_AM_SAFE(AMUDP_SPMDSetExitCallback(gasnetc_traceoutput));
/* for local spawn, assume we want to wait-block */
if (gasneti_getenv("GASNET_SPAWNFN") && *gasneti_getenv("GASNET_SPAWNFN") == 'L') {
GASNETI_TRACE_PRINTF(C,("setting gasnet_set_waitmode(GASNET_WAIT_BLOCK) for localhost spawn"));
gasnet_set_waitmode(GASNET_WAIT_BLOCK);
}
#if GASNET_DEBUG_VERBOSE
fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n",
gasneti_mynode, gasneti_nodes); fflush(stderr);
#endif
gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);
#if GASNET_PSHM
gasneti_pshm_init(&gasnetc_bootstrapSNodeBroadcast, 0);
#endif
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
{ uintptr_t limit;
#if HAVE_MMAP
limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
&gasnetc_bootstrapExchange,
&gasnetc_bootstrapBarrier);
#else
limit = (intptr_t)-1;
#endif
gasneti_segmentInit(limit, &gasnetc_bootstrapExchange);
}
#elif GASNET_SEGMENT_EVERYTHING
/* segment is everything - nothing to do */
#else
#error Bad segment config
#endif
#if GASNET_BLCR
gasneti_checkpoint_guid = gasnetc_networkpid;
gasneti_checkpoint_init(NULL);
#endif
AMUNLOCK();
gasneti_auxseg_init(); /* adjust max seg values based on auxseg */
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
static int gasnetc_init(int *argc, char ***argv) {
int retval = GASNET_OK;
int networkdepth = 0;
const char *pstr = NULL;
const char *tmsgstr = NULL;
AMLOCK();
if (gasneti_init_done)
INITERR(NOT_INIT, "GASNet already initialized");
gasneti_init_done = 1; /* enable early to allow tracing */
/* check system sanity */
gasnetc_check_config();
gasneti_freezeForDebugger();
#if GASNET_DEBUG_VERBOSE
/* note - can't call trace macros during gasnet_init because trace system not yet initialized */
fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
#endif
/* choose network depth */
networkdepth = gasnett_getenv_int_withdefault("GASNET_NETWORKDEPTH", GASNETC_DEFAULT_NETWORKDEPTH, 0);
if (networkdepth <= 1) networkdepth = GASNETC_DEFAULT_NETWORKDEPTH;
AMMPI_VerboseErrors = gasneti_VerboseErrors;
AMMPI_SPMDkillmyprocess = gasneti_killmyprocess;
#if !defined(GASNETI_DISABLE_MPI_INIT_THREAD)
{ int res;
#if GASNETI_THREADS
/* tell MPI to be thread-safe */
res = AMMPI_SPMDSetThreadMode(1, &pstr, argc, argv);
#else
res = AMMPI_SPMDSetThreadMode(0, &pstr, argc, argv);
#endif
if (!res) {
#if GASNETI_THREADS
{ static char tmsg[255];
snprintf(tmsg, sizeof(tmsg),
"*** WARNING: The pthreaded version of mpi-conduit requires an MPI implementation "
"which supports threading mode MPI_THREAD_SERIALIZED, "
"but this implementation reports it can only support %s\n", pstr);
#if GASNET_DEBUG_VERBOSE
/* only show this in verbose mode, because some versions of MPICH (eg Quadrics version)
lie and report THREAD_SINGLE, when in actuality MPI_THREAD_SERIALIZED works just fine */
if (!gasneti_getenv_yesno_withdefault("GASNET_QUIET",0)) fprintf(stderr, "%s", tmsg);
#else
tmsgstr = tmsg;
#endif
}
#else
fprintf(stderr,"unknown failure in AMMPI_SPMDSetThreadMode() => %s\n",pstr);
#endif
}
}
#endif
/* perform job spawn */
retval = AMMPI_SPMDStartup(argc, argv, networkdepth, NULL, &gasnetc_bundle, &gasnetc_endpoint);
if (retval != AM_OK) INITERR(RESOURCE, "AMMPI_SPMDStartup() failed");
gasneti_mynode = AMMPI_SPMDMyProc();
gasneti_nodes = AMMPI_SPMDNumProcs();
/* a number of MPI job spawners fail to propagate the environment to all compute nodes */
/* do this before trace_init to make sure it gets right environment */
gasneti_setupGlobalEnvironment(gasneti_nodes, gasneti_mynode,
gasnetc_bootstrapExchange, gasnetc_bootstrapBroadcast);
/* enable tracing */
gasneti_trace_init(argc, argv);
GASNETI_AM_SAFE(AMMPI_SPMDSetExitCallback(gasnetc_traceoutput));
if (pstr) GASNETI_TRACE_PRINTF(C,("AMMPI_SPMDSetThreadMode/MPI_Init_thread()=>%s",pstr));
if (tmsgstr) GASNETI_TRACE_PRINTF(I,("%s",tmsgstr));
#if GASNET_DEBUG_VERBOSE
fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n",
gasneti_mynode, gasneti_nodes); fflush(stderr);
#endif
gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);
#if GASNET_PSHM
gasneti_pshm_init(&gasnetc_bootstrapExchange, 0);
#endif
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
{ uintptr_t limit;
#if HAVE_MMAP
limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
&gasnetc_bootstrapExchange,
&gasnetc_bootstrapBarrier);
#else
limit = (intptr_t)-1;
#endif
gasneti_segmentInit(limit, &gasnetc_bootstrapExchange);
}
#elif GASNET_SEGMENT_EVERYTHING
/* segment is everything - nothing to do */
#else
#error Bad segment config
#endif
AMUNLOCK();
gasneti_auxseg_init(); /* adjust max seg values based on auxseg */
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
static int gasnetc_init(int *argc, char ***argv, gex_Flags_t flags) {
int retval = GASNET_OK;
/* check system sanity */
gasnetc_check_config();
/* --------- begin Master code ------------ */
if (!AMUDP_SPMDIsWorker(argv?*argv:NULL)) {
/* assume we're an implicit master
(we don't currently support explicit workers spawned
without using the AMUDP SPMD API)
*/
int num_nodes;
int i;
char spawnfn;
amudp_spawnfn_t fp = (amudp_spawnfn_t)NULL;
if (!argv) {
gasneti_fatalerror("implicit-master without argv not supported - use amudprun");
}
/* pretend we're node 0, for purposes of verbose env reporting */
gasneti_init_done = 1;
gasneti_mynode = 0;
#if defined(GASNET_CSPAWN_CMD)
{ /* set configure default cspawn cmd */
const char *cmd = gasneti_getenv_withdefault("GASNET_CSPAWN_CMD",GASNET_CSPAWN_CMD);
gasneti_setenv("GASNET_CSPAWN_CMD",cmd);
}
#endif
/* parse node count from command line */
if (*argc < 2) {
fprintf(stderr, "GASNet: Missing parallel node count\n");
fprintf(stderr, "GASNet: Specify node count as first argument, or use upcrun/tcrun spawner script to start job\n");
fprintf(stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit(-1);
}
/*
* argv[1] is number of nodes; argv[0] is program name; argv is
* list of arguments including program name and number of nodes.
* We need to remove argv[1] when the argument array is passed
* to the tic_main().
*/
num_nodes = atoi((*argv)[1]);
if (num_nodes < 1) {
fprintf (stderr, "GASNet: Invalid number of nodes: %s\n", (*argv)[1]);
fprintf (stderr, "GASNet: Usage '%s <num_nodes> {program arguments}'\n", (*argv)[0]);
exit (1);
}
/* remove the num_nodes argument */
for (i = 1; i < (*argc)-1; i++) {
(*argv)[i] = (*argv)[i+1];
}
(*argv)[(*argc)-1] = NULL;
(*argc)--;
/* get spawnfn */
spawnfn = *gasneti_getenv_withdefault("GASNET_SPAWNFN", _STRINGIFY(GASNETC_DEFAULT_SPAWNFN));
{ /* ensure we pass the effective spawnfn to worker env */
char spawnstr[2];
spawnstr[0] = toupper(spawnfn);
spawnstr[1] = '\0';
gasneti_setenv("GASNET_SPAWNFN",spawnstr);
}
/* ensure reliable localhost operation by forcing use of 127.0.0.1
* setting GASNET_MASTERIP to the empty string will prevent this */
if (('L' == toupper(spawnfn)) && !gasneti_getenv("GASNET_MASTERIP")) {
gasneti_setenv("GASNET_MASTERIP","127.0.0.1");
}
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
if (toupper(spawnfn) == toupper(AMUDP_Spawnfn_Desc[i].abbrev)) {
fp = AMUDP_Spawnfn_Desc[i].fnptr;
break;
}
}
if (!fp) {
fprintf (stderr, "GASNet: Invalid spawn function specified in GASNET_SPAWNFN\n");
fprintf (stderr, "GASNet: The following mechanisms are available:\n");
for (i=0; AMUDP_Spawnfn_Desc[i].abbrev; i++) {
fprintf(stderr, " '%c' %s\n",
toupper(AMUDP_Spawnfn_Desc[i].abbrev), AMUDP_Spawnfn_Desc[i].desc);
}
exit(1);
}
#if GASNET_DEBUG_VERBOSE
/* note - can't call trace macros during gasnet_init because trace system not yet initialized */
fprintf(stderr,"gasnetc_init(): about to spawn...\n"); fflush(stderr);
#endif
retval = AMUDP_SPMDStartup(argc, argv,
num_nodes, 0, fp,
NULL, &gasnetc_bundle, &gasnetc_endpoint);
/* master startup should never return */
gasneti_fatalerror("master AMUDP_SPMDStartup() failed");
}
/* --------- begin Worker code ------------ */
AMLOCK();
if (gasneti_init_done)
INITERR(NOT_INIT, "GASNet already initialized");
gasneti_freezeForDebugger();
AMX_VerboseErrors = gasneti_VerboseErrors;
AMUDP_SPMDkillmyprocess = gasneti_killmyprocess;
#if GASNETI_CALIBRATE_TSC
// Early x86*/Linux timer initialization before AMUDP_SPMDStartup()
//
// udp-conduit does not support user-provided values for GASNET_TSC_RATE*
// (which fine-tune timer calibration on x86/Linux). This is partially due
// to a dependency cycle at startup with envvar propagation, but more
// importantly because the retransmission algorithm (and hence all conduit
// comms) rely on gasnet timers to be accurate (at least approximately), so
// we don't allow the user to weaken or disable their calibration.
gasneti_unsetenv("GASNET_TSC_RATE");
gasneti_unsetenv("GASNET_TSC_RATE_TOLERANCE");
gasneti_unsetenv("GASNET_TSC_RATE_HARD_TOLERANCE");
GASNETI_TICKS_INIT();
#endif
/* perform job spawn */
retval = AMUDP_SPMDStartup(argc, argv,
0, 0, NULL, /* dummies */
&gasnetc_networkpid, &gasnetc_bundle, &gasnetc_endpoint);
if (retval != AM_OK) INITERR(RESOURCE, "slave AMUDP_SPMDStartup() failed");
gasneti_init_done = 1; /* enable early to allow tracing */
gasneti_getenv_hook = (/* cast drops const */ gasneti_getenv_fn_t*)&AMUDP_SPMDgetenvMaster;
gasneti_mynode = AMUDP_SPMDMyProc();
gasneti_nodes = AMUDP_SPMDNumProcs();
#if !GASNETI_CALIBRATE_TSC
/* Must init timers after global env, and preferably before tracing */
GASNETI_TICKS_INIT();
#endif
/* enable tracing */
gasneti_trace_init(argc, argv);
GASNETI_AM_SAFE(AMUDP_SPMDSetExitCallback(gasnetc_traceoutput));
/* for local spawn, assume we want to wait-block */
if (gasneti_getenv("GASNET_SPAWNFN") && *gasneti_getenv("GASNET_SPAWNFN") == 'L') {
GASNETI_TRACE_PRINTF(C,("setting gasnet_set_waitmode(GASNET_WAIT_BLOCK) for localhost spawn"));
gasnet_set_waitmode(GASNET_WAIT_BLOCK);
}
#if GASNET_DEBUG_VERBOSE
fprintf(stderr,"gasnetc_init(): spawn successful - node %i/%i starting...\n",
gasneti_mynode, gasneti_nodes); fflush(stderr);
#endif
gasneti_nodemapInit(&gasnetc_bootstrapExchange, NULL, 0, 0);
#if GASNET_PSHM
gasneti_pshm_init(&gasnetc_bootstrapSNodeBroadcast, 0);
#endif
uintptr_t mmap_limit;
#if HAVE_MMAP
mmap_limit = gasneti_mmapLimit((uintptr_t)-1, (uint64_t)-1,
&gasnetc_bootstrapExchange,
&gasnetc_bootstrapBarrier);
#else
// TODO-EX: we can at least look at rlimits but such logic belongs in conduit-indep code
mmap_limit = (intptr_t)-1;
#endif
/* allocate and attach an aux segment */
gasneti_auxsegAttach(mmap_limit, &gasnetc_bootstrapExchange);
/* determine Max{Local,GLobal}SegmentSize */
gasneti_segmentInit(mmap_limit, &gasnetc_bootstrapExchange, flags);
#if GASNET_BLCR
gasneti_checkpoint_guid = gasnetc_networkpid;
gasneti_checkpoint_init(NULL);
#endif
AMUNLOCK();
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
