extern void gasneti_checkpoint_init(gasneti_bootstrapBroadcastfn_t bcast_fn) {
/* Initialize the GUID if the conduit has not already set a non-zero value */
if (!gasneti_checkpoint_guid) {
if (! gasneti_mynode) {
gasneti_checkpoint_guid = gasneti_checkpoint_mkguid();
}
bcast_fn(&gasneti_checkpoint_guid, sizeof(uint64_t), &gasneti_checkpoint_guid, 0);
}
/* Enforce use of absolute paths */
{
char *val;
if (NULL != (val = gasneti_getenv("GASNET_CHECKPOINT_JOBDIR"))) {
if ('/' != val[0]) {
gasneti_fatalerror("Environment variable GASNET_CHECKPOINT_JOBDIR='%s' is not an absolute path", val);
} else {
gasneti_checkpoint_jobdir = val;
}
} else {
char *dir;
size_t len;
if (NULL != (val = gasneti_getenv("GASNET_CHECKPOINT_BASEDIR"))) {
if ('/' != val[0]) {
gasneti_fatalerror("Environment variable GASNET_CHECKPOINT_BASEDIR='%s' is not an absolute path", val);
} else {
dir = val;
}
} else {
if (NULL != (val = gasneti_getenv("HOME"))) {
if ('/' != val[0]) {
gasneti_fatalerror("Environment variable HOME='%s' is not an absolute path", val);
} else {
const char *rest = "/gasnet-checkpoint";
len = strlen(val) + strlen(rest) + 1;
dir = gasneti_malloc(len);
strcpy(dir, val);
strcat(dir, rest);
}
} else {
gasneti_fatalerror("Environment variable HOME is not set");
}
}
len = strlen(dir) + 19; /* 19 = 16 digits, '/' , '.' and '\0' */
gasneti_checkpoint_jobdir = gasneti_malloc(len);
gasneti_leak(gasneti_checkpoint_jobdir);
snprintf(gasneti_checkpoint_jobdir, len, "%s/%08x.%08x", dir,
GASNETI_HIWORD(gasneti_checkpoint_guid),
GASNETI_LOWORD(gasneti_checkpoint_guid));
if (dir != val) gasneti_free(dir);
}
}
}
/* allocate a new iop */
GASNETI_NEVER_INLINE(gasnete_iop_alloc,
static gasnete_iop_t *gasnete_iop_alloc(gasneti_threaddata_t * const thread)) {
gasnete_iop_t *iop = (gasnete_iop_t *)gasneti_malloc(sizeof(gasnete_iop_t));
gasneti_leak(iop);
#if GASNET_DEBUG
memset(iop, 0, sizeof(gasnete_iop_t)); /* set pad to known value */
#endif
SET_OPTYPE((gasnete_op_t *)iop, OPTYPE_IMPLICIT);
iop->threadidx = thread->threadidx;
iop->initiated_get_cnt = 0;
iop->initiated_put_cnt = 0;
gasnetc_atomic_set(&(iop->completed_get_cnt), 0, 0);
gasnetc_atomic_set(&(iop->completed_put_cnt), 0, 0);
return iop;
}
/* allocate more eops */
GASNETI_NEVER_INLINE(gasnete_eop_alloc,
static void gasnete_eop_alloc(gasneti_threaddata_t * const thread)) {
gasnete_eopaddr_t addr;
int bufidx = thread->eop_num_bufs;
gasnete_eop_t *buf;
int i;
gasnete_threadidx_t threadidx = thread->threadidx;
if (bufidx == 256) gasneti_fatalerror("GASNet Extended API: Ran out of explicit handles (limit=65535)");
thread->eop_num_bufs++;
buf = (gasnete_eop_t *)gasneti_calloc(256,sizeof(gasnete_eop_t));
gasneti_leak(buf);
for (i=0; i < 256; i++) {
addr.bufferidx = bufidx;
#if GASNETE_SCATTER_EOPS_ACROSS_CACHELINES
#ifdef GASNETE_EOP_MOD
addr.eopidx = (i+32) % 255;
#else
{ int k = i+32;
addr.eopidx = k > 255 ? k - 255 : k;
}
#endif
#else
addr.eopidx = i+1;
#endif
buf[i].threadidx = threadidx;
buf[i].addr = addr;
#if 0 /* these can safely be skipped when the values are zero */
SET_OPSTATE(&(buf[i]),OPSTATE_FREE);
SET_OPTYPE(&(buf[i]),OPTYPE_EXPLICIT);
#if GASNETE_EOP_COUNTED
buff[i].initiated_cnt = 0;
#endif
#endif
#if GASNETE_EOP_COUNTED
gasnetc_atomic_set(&(buf[i].completed_cnt), 0, 0);
#endif
}
/* add a list terminator */
#if GASNETE_SCATTER_EOPS_ACROSS_CACHELINES
#ifdef GASNETE_EOP_MOD
buf[223].addr.eopidx = 255; /* modular arithmetic messes up this one */
#endif
buf[255].addr = EOPADDR_NIL;
#else
buf[255].addr = EOPADDR_NIL;
#endif
thread->eop_bufs[bufidx] = buf;
addr.bufferidx = bufidx;
addr.eopidx = 0;
thread->eop_free = addr;
#if GASNET_DEBUG
{ /* verify new free list got built correctly */
int i;
int seen[256];
gasnete_eopaddr_t addr = thread->eop_free;
#if 0
if (gasneti_mynode == 0)
for (i=0;i<256;i++) {
fprintf(stderr,"%i: %i: next=%i\n",gasneti_mynode,i,buf[i].addr.eopidx);
fflush(stderr);
}
sleep(5);
#endif
gasneti_memcheck(thread->eop_bufs[bufidx]);
memset(seen, 0, 256*sizeof(int));
for (i=0;i<(bufidx==255?255:256);i++) {
gasnete_eop_t *eop;
gasneti_assert(!gasnete_eopaddr_isnil(addr));
eop = GASNETE_EOPADDR_TO_PTR(thread,addr);
gasneti_assert(OPTYPE(eop) == OPTYPE_EXPLICIT);
gasneti_assert(OPSTATE(eop) == OPSTATE_FREE);
gasneti_assert(eop->threadidx == threadidx);
gasneti_assert(addr.bufferidx == bufidx);
gasneti_assert(!seen[addr.eopidx]);/* see if we hit a cycle */
seen[addr.eopidx] = 1;
addr = eop->addr;
}
gasneti_assert(gasnete_eopaddr_isnil(addr));
}
#endif
}
/* ------------------------------------------------------------------------------------ */
extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries,
uintptr_t segsize, uintptr_t minheapoffset) {
void *segbase = NULL;
int ret;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%lu, minheapoffset=%lu)",
numentries, (unsigned long)segsize, (unsigned long)minheapoffset));
if (!gasneti_init_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already attached");
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize too large");
if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */
minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE;
#else
segsize = 0;
minheapoffset = 0;
#endif
segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */
/* ------------------------------------------------------------------------------------ */
/* register handlers */
{ int i;
for (i = 0; i < GASNETC_MAX_NUMHANDLERS; i++)
gasnetc_handler[i] = (gasneti_handler_fn_t)&gasneti_defaultAMHandler;
}
{ /* core API handlers */
gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(ctable);
while (ctable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(ctable, len, 1, 63, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering core API handlers");
gasneti_assert(numreg == len);
}
{ /* extended API handlers */
gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(etable);
while (etable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(etable, len, 64, 127, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering extended API handlers");
gasneti_assert(numreg == len);
}
if (table) { /* client handlers */
int numreg1 = 0;
int numreg2 = 0;
/* first pass - assign all fixed-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering fixed-index client handlers");
/* second pass - fill in dontcare-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering fixed-index client handlers");
gasneti_assert(numreg1 + numreg2 == numentries);
}
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
/* ------------------------------------------------------------------------------------ */
/* register segment */
gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t));
gasneti_leak(gasneti_seginfo);
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if (segsize == 0) segbase = NULL; /* no segment */
else {
gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, &gasnetc_bootstrapExchange);
segbase = gasneti_seginfo[gasneti_mynode].addr;
segsize = gasneti_seginfo[gasneti_mynode].size;
gasneti_assert(((uintptr_t)segbase) % GASNET_PAGESIZE == 0);
gasneti_assert(segsize % GASNET_PAGESIZE == 0);
}
#else
{ /* GASNET_SEGMENT_EVERYTHING */
gasnet_node_t i;
for (i=0; i<gasneti_nodes; i++) {
gasneti_seginfo[i].addr = (void *)0;
gasneti_seginfo[i].size = (uintptr_t)-1;
}
segbase = (void *)0;
segsize = (uintptr_t)-1;
}
#endif
ret = gasnetc_p4_attach(segbase, segsize);
if (GASNET_OK != ret) {
GASNETI_RETURN_ERRR(RESOURCE,"Error attaching Portals4 resources");
}
/* ------------------------------------------------------------------------------------ */
/* gather segment information */
/* This was done by segmentAttach above */
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasnetc_bootstrapBarrier();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete"));
gasneti_assert(gasneti_seginfo[gasneti_mynode].addr == segbase &&
gasneti_seginfo[gasneti_mynode].size == segsize);
gasneti_auxseg_attach(); /* provide auxseg */
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
gasnetc_bootstrapBarrier();
return GASNET_OK;
}
/* ------------------------------------------------------------------------------------ */
extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries,
uintptr_t segsize, uintptr_t minheapoffset) {
int retval = GASNET_OK;
void *segbase = NULL;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%lu, minheapoffset=%lu)",
numentries, (unsigned long)segsize, (unsigned long)minheapoffset));
AMLOCK();
if (!gasneti_init_done)
INITERR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
INITERR(NOT_INIT, "GASNet already attached");
/* pause to make sure all nodes have called attach
if a node calls gasnet_exit() between init/attach, then this allows us
to process the AMUDP_SPMD control messages required for job shutdown
*/
gasnetc_bootstrapBarrier();
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
INITERR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
INITERR(BAD_ARG, "segsize too large");
if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */
minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE;
#else
segsize = 0;
minheapoffset = 0;
#endif
segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */
/* ------------------------------------------------------------------------------------ */
/* register handlers */
#ifdef GASNETC_MAX_NUMHANDLERS
/* Initialize shadow handler table */
{ int i;
for (i=0; i<GASNETC_MAX_NUMHANDLERS; i++)
gasnetc_handler[i]=(gasneti_handler_fn_t)&gasneti_defaultAMHandler;
}
#endif
{ /* core API handlers */
gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(ctable);
while (ctable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(ctable, len, 1, 63, 0, &numreg) != GASNET_OK)
INITERR(RESOURCE,"Error registering core API handlers");
gasneti_assert(numreg == len);
}
{ /* extended API handlers */
gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(etable);
while (etable[len].fnptr) len++; /* calc len */
if (gasnetc_reghandlers(etable, len, 64, 127, 0, &numreg) != GASNET_OK)
INITERR(RESOURCE,"Error registering extended API handlers");
gasneti_assert(numreg == len);
}
if (table) { /* client handlers */
int numreg1 = 0;
int numreg2 = 0;
/* first pass - assign all fixed-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK)
INITERR(RESOURCE,"Error registering fixed-index client handlers");
/* second pass - fill in dontcare-index handlers */
if (gasnetc_reghandlers(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK)
INITERR(RESOURCE,"Error registering fixed-index client handlers");
gasneti_assert(numreg1 + numreg2 == numentries);
}
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
/* ------------------------------------------------------------------------------------ */
/* register segment */
gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t));
gasneti_leak(gasneti_seginfo);
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, &gasnetc_bootstrapExchange);
#else /* GASNET_SEGMENT_EVERYTHING */
{ int i;
for (i=0;i<gasneti_nodes;i++) {
gasneti_seginfo[i].addr = (void *)0;
gasneti_seginfo[i].size = (uintptr_t)-1;
}
}
#endif
segbase = gasneti_seginfo[gasneti_mynode].addr;
segsize = gasneti_seginfo[gasneti_mynode].size;
/* After local segment is attached, call optional client-provided hook
(###) should call BEFORE any conduit-specific pinning/registration of the segment
*/
if (gasnet_client_attach_hook) {
gasnet_client_attach_hook(segbase, segsize);
}
/* AMUDP allows arbitrary registration with no further action */
if (segsize) {
retval = AM_SetSeg(gasnetc_endpoint, segbase, segsize);
if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed");
}
#if GASNET_TRACE
if (GASNETI_TRACE_ENABLED(A))
GASNETI_AM_SAFE(AMUDP_SetHandlerCallbacks(gasnetc_endpoint,
gasnetc_enteringHandler_hook, gasnetc_leavingHandler_hook));
#endif
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasnetc_bootstrapBarrier();
AMUNLOCK();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete\n"));
gasneti_auxseg_attach(); /* provide auxseg */
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
AMLOCK();
gasnetc_bootstrapBarrier();
AMUNLOCK();
gasneti_assert(retval == GASNET_OK);
return retval;
done: /* error return while locked */
AMUNLOCK();
GASNETI_RETURN(retval);
}
/* ------------------------------------------------------------------------------------ */
extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries,
uintptr_t segsize, uintptr_t minheapoffset) {
void *segbase = NULL;
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%"PRIuPTR", minheapoffset=%"PRIuPTR")",
numentries, segsize, minheapoffset));
if (!gasneti_init_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet attach called before init");
if (gasneti_attach_done)
GASNETI_RETURN_ERRR(NOT_INIT, "GASNet already attached");
/* check argument sanity */
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if ((segsize % GASNET_PAGESIZE) != 0)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize not page-aligned");
if (segsize > gasneti_MaxLocalSegmentSize)
GASNETI_RETURN_ERRR(BAD_ARG, "segsize too large");
if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */
minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE;
#else
segsize = 0;
minheapoffset = 0;
#endif
segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */
/* ------------------------------------------------------------------------------------ */
/* register handlers */
{ int i;
for (i = 0; i < GASNETC_MAX_NUMHANDLERS; i++)
gasnetc_handler[i] = (gasneti_handler_fn_t)&gasneti_defaultAMHandler;
}
{ /* core API handlers */
gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(ctable);
while (ctable[len].fnptr) len++; /* calc len */
if (gasneti_amregister(ctable, len, 1, 63, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering core API handlers");
gasneti_assert(numreg == len);
}
{ /* extended API handlers */
gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable();
int len = 0;
int numreg = 0;
gasneti_assert(etable);
while (etable[len].fnptr) len++; /* calc len */
if (gasneti_amregister(etable, len, 64, 127, 0, &numreg) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering extended API handlers");
gasneti_assert(numreg == len);
}
if (table) { /* client handlers */
int numreg1 = 0;
int numreg2 = 0;
/* first pass - assign all fixed-index handlers */
if (gasneti_amregister(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering fixed-index client handlers");
/* second pass - fill in dontcare-index handlers */
if (gasneti_amregister(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK)
GASNETI_RETURN_ERRR(RESOURCE,"Error registering variable-index client handlers");
gasneti_assert(numreg1 + numreg2 == numentries);
}
/* ------------------------------------------------------------------------------------ */
/* register fatal signal handlers */
/* catch fatal signals and convert to SIGQUIT */
gasneti_registerSignalHandlers(gasneti_defaultSignalHandler);
/* ------------------------------------------------------------------------------------ */
/* setup fo rexit coordination */
gasnetc_exittimeout = gasneti_get_exittimeout(GASNETC_DEFAULT_EXITTIMEOUT_MAX,
GASNETC_DEFAULT_EXITTIMEOUT_MIN,
GASNETC_DEFAULT_EXITTIMEOUT_FACTOR,
GASNETC_DEFAULT_EXITTIMEOUT_MIN);
#if HAVE_ON_EXIT
on_exit(gasnetc_on_exit, NULL);
#else
atexit(gasnetc_atexit);
#endif
/* ------------------------------------------------------------------------------------ */
/* register segment */
gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t));
gasneti_leak(gasneti_seginfo);
#if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE
if (segsize == 0) segbase = NULL; /* no segment */
else {
gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, gasneti_bootstrapExchange);
segbase = gasneti_seginfo[gasneti_mynode].addr;
segsize = gasneti_seginfo[gasneti_mynode].size;
gasneti_assert(((uintptr_t)segbase) % GASNET_PAGESIZE == 0);
gasneti_assert(segsize % GASNET_PAGESIZE == 0);
}
#else
{ /* GASNET_SEGMENT_EVERYTHING */
gasnet_node_t i;
for (i=0; i<gasneti_nodes; i++) {
gasneti_seginfo[i].addr = (void *)0;
gasneti_seginfo[i].size = (uintptr_t)-1;
}
segbase = (void *)0;
segsize = (uintptr_t)-1;
}
#endif
gasnetc_ofi_attach(segbase, segsize);
/* After local segment is attached, call optional client-provided hook
(###) should call BEFORE any conduit-specific pinning/registration of the segment
*/
if (gasnet_client_attach_hook) {
gasnet_client_attach_hook(segbase, segsize);
}
/* ------------------------------------------------------------------------------------ */
/* primary attach complete */
gasneti_attach_done = 1;
gasneti_bootstrapBarrier();
GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete"));
gasneti_assert(gasneti_seginfo[gasneti_mynode].addr == segbase &&
gasneti_seginfo[gasneti_mynode].size == segsize);
/* (###) exchange_fn is optional (may be NULL) and is only used with GASNET_SEGMENT_EVERYTHING
if your conduit has an optimized bootstrapExchange pass it in place of NULL
*/
gasneti_auxseg_attach(NULL); /* provide auxseg */
gasnete_init(); /* init the extended API */
gasneti_nodemapFini();
/* ensure extended API is initialized across nodes */
gasneti_bootstrapBarrier();
return GASNET_OK;
}