/* ------------------------------------------------------------------------------------ */ static int gasnetc_attach_primary(void) { int retval = GASNET_OK; AMLOCK(); /* pause to make sure all nodes have called attach if a node calls gasnet_exit() between init/attach, then this allows us to process the AMUDP_SPMD control messages required for job shutdown */ gasnetc_bootstrapBarrier(); /* ------------------------------------------------------------------------------------ */ /* register fatal signal handlers */ /* catch fatal signals and convert to SIGQUIT */ gasneti_registerSignalHandlers(gasneti_defaultSignalHandler); #if HAVE_ON_EXIT on_exit(gasnetc_on_exit, NULL); #else atexit(gasnetc_atexit); #endif #if GASNET_TRACE || GASNET_DEBUG #if !GASNET_DEBUG if (GASNETI_TRACE_ENABLED(A)) #endif GASNETI_AM_SAFE(AMUDP_SetHandlerCallbacks(gasnetc_endpoint, gasnetc_enteringHandler_hook, gasnetc_leavingHandler_hook)); #endif #if GASNETC_MOCK_EVERYTHING retval = AM_SetSeg(gasnetc_endpoint, NULL, (uintptr_t)-1); if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed"); #endif /* ------------------------------------------------------------------------------------ */ /* primary attach complete */ gasneti_attach_done = 1; gasnetc_bootstrapBarrier(); AMUNLOCK(); GASNETI_TRACE_PRINTF(C,("gasnetc_attach_primary(): primary attach complete\n")); gasnete_init(); /* init the extended API */ gasneti_nodemapFini(); /* ensure extended API is initialized across nodes */ AMLOCK(); gasnetc_bootstrapBarrier(); AMUNLOCK(); gasneti_assert(retval == GASNET_OK); return retval; done: /* error return while locked */ AMUNLOCK(); GASNETI_RETURN(retval); }
/* ------------------------------------------------------------------------------------ */ static int gasnetc_attach_segment(gex_Segment_t *segment_p, gex_TM_t tm, uintptr_t segsize, gasneti_bootstrapExchangefn_t exchangefn, gex_Flags_t flags) { int retval = GASNET_OK; // TODO-EX: crude detection of multiple calls until we support them gasneti_assert(NULL == gasneti_seginfo[0].addr); /* ------------------------------------------------------------------------------------ */ /* register segment */ gasneti_segmentAttach(segsize, gasneti_seginfo, exchangefn, flags); void *segbase = gasneti_seginfo[gasneti_mynode].addr; segsize = gasneti_seginfo[gasneti_mynode].size; gasneti_assert(((uintptr_t)segbase) % GASNET_PAGESIZE == 0); gasneti_assert(segsize % GASNET_PAGESIZE == 0); gasneti_EP_t ep = gasneti_import_tm(tm)->_ep; ep->_segment = gasneti_alloc_segment(ep->_client, segbase, segsize, flags, 0); gasneti_legacy_segment_attach_hook(ep); *segment_p = gasneti_export_segment(ep->_segment); /* After local segment is attached, call optional client-provided hook (###) should call BEFORE any conduit-specific pinning/registration of the segment */ if (gasnet_client_attach_hook) { gasnet_client_attach_hook(segbase, segsize); } #if !GASNETC_MOCK_EVERYTHING /* AMUDP allows arbitrary registration with no further action */ if (segsize) { retval = AM_SetSeg(gasnetc_endpoint, segbase, segsize); if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed"); } #endif /* ------------------------------------------------------------------------------------ */ /* gather segment information */ /* (###) add code here to gather the segment assignment info into gasneti_seginfo on each node (may be possible to use AMShortRequest here) If gasneti_segmentAttach() was used above, this is already done. Done in gasneti_segmentAttach(), above. */ gasneti_assert(gasneti_seginfo[gasneti_mynode].addr == segbase && gasneti_seginfo[gasneti_mynode].size == segsize); done: GASNETI_RETURN(retval); }
int main(int argc, char **argv) { eb_t eb; ep_t ep; uint64_t networkpid; int64_t begin, end, total; int polling = 1; int iters = 0; TEST_STARTUP(argc, argv, networkpid, eb, ep, 1, 2, "iters (Poll/Block)"); /* setup handlers */ AM_Safe(AM_SetHandler(ep, LARGE_REQ_HANDLER, large_request_handler)); AM_Safe(AM_SetHandler(ep, LARGE_REP_HANDLER, large_reply_handler)); setupUtilHandlers(ep, eb); /* get SPMD info */ myproc = AMX_SPMDMyProc(); numprocs = AMX_SPMDNumProcs(); if (argc > 1) iters = atoi(argv[1]); if (!iters) iters = 1; if (argc > 2) { switch(argv[2][0]) { case 'p': case 'P': polling = 1; break; case 'b': case 'B': polling = 0; break; default: printf("polling must be 'P' or 'B'..\n"); AMX_SPMDExit(1); } } if (numprocs % 2 != 0 && numprocs > 1) { printf("requires an even or unary number of processors\n"); AMX_SPMDExit(1); } VMseg = (uint32_t *)malloc(AM_MaxLong()+100); memset(VMseg, 0, AM_MaxLong()+100); AM_Safe(AM_SetSeg(ep, VMseg, AM_MaxLong()+100)); if (myproc % 2 == 0) partner = (myproc + 1) % numprocs; else partner = (myproc - 1); AM_Safe(AMX_SPMDBarrier()); if (myproc == 0) printf("Running %i iterations of bulk bounce test...\n", iters); begin = getCurrentTimeMicrosec(); if (myproc % 2 == 1 || numprocs == 1) { int q; for (q=0; q<iters; q++) { /* init my source mem */ int i; uint32_t *srcmem = (uint32_t *)(((uint8_t*)VMseg)+100); for (i = 0; i < AM_MaxLong()/4; i++) { srcmem[i] = (uint32_t)((count << 16) + i); } #if VERBOSE printf("%i: sending request...", myproc); fflush(stdout); #endif done = 0; AM_Safe(AM_RequestXfer1(ep, partner, 100, LARGE_REQ_HANDLER, srcmem, AM_MaxLong(), 666)); if (polling) { /* poll until everyone done */ while (!done) { AM_Safe(AM_Poll(eb)); } } else { while (!done) { AM_Safe(AM_SetEventMask(eb, AM_NOTEMPTY)); AM_Safe(AM_WaitSema(eb)); AM_Safe(AM_Poll(eb)); } } } } else { if (polling) { /* poll until everyone done */ while (count<iters*2) { AM_Safe(AM_Poll(eb)); } } else { while (count<iters*2) { AM_Safe(AM_SetEventMask(eb, AM_NOTEMPTY)); AM_Safe(AM_WaitSema(eb)); AM_Safe(AM_Poll(eb)); } } } end = getCurrentTimeMicrosec(); total = end - begin; printf("Slave %i: %i microseconds total, throughput: %8.3f KB/sec\n", myproc, (int)total, (float)(((float)1000000)*AM_MaxLong()*iters/((int)total))/1024.0); fflush(stdout); /* dump stats */ AM_Safe(AMX_SPMDBarrier()); printGlobalStats(); AM_Safe(AMX_SPMDBarrier()); /* exit */ AM_Safe(AMX_SPMDExit(0)); return 0; }
/* ------------------------------------------------------------------------------------ */ extern int gasnetc_attach(gasnet_handlerentry_t *table, int numentries, uintptr_t segsize, uintptr_t minheapoffset) { int retval = GASNET_OK; void *segbase = NULL; GASNETI_TRACE_PRINTF(C,("gasnetc_attach(table (%i entries), segsize=%lu, minheapoffset=%lu)", numentries, (unsigned long)segsize, (unsigned long)minheapoffset)); AMLOCK(); if (!gasneti_init_done) INITERR(NOT_INIT, "GASNet attach called before init"); if (gasneti_attach_done) INITERR(NOT_INIT, "GASNet already attached"); /* pause to make sure all nodes have called attach if a node calls gasnet_exit() between init/attach, then this allows us to process the AMUDP_SPMD control messages required for job shutdown */ gasnetc_bootstrapBarrier(); /* check argument sanity */ #if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE if ((segsize % GASNET_PAGESIZE) != 0) INITERR(BAD_ARG, "segsize not page-aligned"); if (segsize > gasneti_MaxLocalSegmentSize) INITERR(BAD_ARG, "segsize too large"); if ((minheapoffset % GASNET_PAGESIZE) != 0) /* round up the minheapoffset to page sz */ minheapoffset = ((minheapoffset / GASNET_PAGESIZE) + 1) * GASNET_PAGESIZE; #else segsize = 0; minheapoffset = 0; #endif segsize = gasneti_auxseg_preattach(segsize); /* adjust segsize for auxseg reqts */ /* ------------------------------------------------------------------------------------ */ /* register handlers */ #ifdef GASNETC_MAX_NUMHANDLERS /* Initialize shadow handler table */ { int i; for (i=0; i<GASNETC_MAX_NUMHANDLERS; i++) gasnetc_handler[i]=(gasneti_handler_fn_t)&gasneti_defaultAMHandler; } #endif { /* core API handlers */ gasnet_handlerentry_t *ctable = (gasnet_handlerentry_t *)gasnetc_get_handlertable(); int len = 0; int numreg = 0; gasneti_assert(ctable); while (ctable[len].fnptr) len++; /* calc len */ if (gasnetc_reghandlers(ctable, len, 1, 63, 0, &numreg) != GASNET_OK) INITERR(RESOURCE,"Error registering core API handlers"); gasneti_assert(numreg == len); } { /* extended API handlers */ gasnet_handlerentry_t *etable = (gasnet_handlerentry_t *)gasnete_get_handlertable(); int len = 0; int numreg = 0; gasneti_assert(etable); while (etable[len].fnptr) len++; /* calc len */ if (gasnetc_reghandlers(etable, len, 64, 127, 0, &numreg) != GASNET_OK) INITERR(RESOURCE,"Error registering extended API handlers"); gasneti_assert(numreg == len); } if (table) { /* client handlers */ int numreg1 = 0; int numreg2 = 0; /* first pass - assign all fixed-index handlers */ if (gasnetc_reghandlers(table, numentries, 128, 255, 0, &numreg1) != GASNET_OK) INITERR(RESOURCE,"Error registering fixed-index client handlers"); /* second pass - fill in dontcare-index handlers */ if (gasnetc_reghandlers(table, numentries, 128, 255, 1, &numreg2) != GASNET_OK) INITERR(RESOURCE,"Error registering fixed-index client handlers"); gasneti_assert(numreg1 + numreg2 == numentries); } /* ------------------------------------------------------------------------------------ */ /* register fatal signal handlers */ /* catch fatal signals and convert to SIGQUIT */ gasneti_registerSignalHandlers(gasneti_defaultSignalHandler); #if HAVE_ON_EXIT on_exit(gasnetc_on_exit, NULL); #else atexit(gasnetc_atexit); #endif /* ------------------------------------------------------------------------------------ */ /* register segment */ gasneti_seginfo = (gasnet_seginfo_t *)gasneti_malloc(gasneti_nodes*sizeof(gasnet_seginfo_t)); gasneti_leak(gasneti_seginfo); #if GASNET_SEGMENT_FAST || GASNET_SEGMENT_LARGE gasneti_segmentAttach(segsize, minheapoffset, gasneti_seginfo, &gasnetc_bootstrapExchange); #else /* GASNET_SEGMENT_EVERYTHING */ { int i; for (i=0;i<gasneti_nodes;i++) { gasneti_seginfo[i].addr = (void *)0; gasneti_seginfo[i].size = (uintptr_t)-1; } } #endif segbase = gasneti_seginfo[gasneti_mynode].addr; segsize = gasneti_seginfo[gasneti_mynode].size; /* After local segment is attached, call optional client-provided hook (###) should call BEFORE any conduit-specific pinning/registration of the segment */ if (gasnet_client_attach_hook) { gasnet_client_attach_hook(segbase, segsize); } /* AMUDP allows arbitrary registration with no further action */ if (segsize) { retval = AM_SetSeg(gasnetc_endpoint, segbase, segsize); if (retval != AM_OK) INITERR(RESOURCE, "AM_SetSeg() failed"); } #if GASNET_TRACE if (GASNETI_TRACE_ENABLED(A)) GASNETI_AM_SAFE(AMUDP_SetHandlerCallbacks(gasnetc_endpoint, gasnetc_enteringHandler_hook, gasnetc_leavingHandler_hook)); #endif /* ------------------------------------------------------------------------------------ */ /* primary attach complete */ gasneti_attach_done = 1; gasnetc_bootstrapBarrier(); AMUNLOCK(); GASNETI_TRACE_PRINTF(C,("gasnetc_attach(): primary attach complete\n")); gasneti_auxseg_attach(); /* provide auxseg */ gasnete_init(); /* init the extended API */ gasneti_nodemapFini(); /* ensure extended API is initialized across nodes */ AMLOCK(); gasnetc_bootstrapBarrier(); AMUNLOCK(); gasneti_assert(retval == GASNET_OK); return retval; done: /* error return while locked */ AMUNLOCK(); GASNETI_RETURN(retval); }
int main(int argc, char **argv) { eb_t eb; ep_t ep; uint64_t networkpid; int64_t begin, end, total; int polling = 1; int fullduplex = 0; int rightguy; uint32_t *srcmem; int iters = 0; TEST_STARTUP(argc, argv, networkpid, eb, ep, 1, 4, "iters (bulkmsgsize) (Poll/Block) (Half/Full)"); /* setup handlers */ AM_Safe(AM_SetHandler(ep, BULK_REQ_HANDLER, bulk_request_handler)); AM_Safe(AM_SetHandler(ep, BULK_REP_HANDLER, bulk_reply_handler)); setupUtilHandlers(ep, eb); /* get SPMD info */ myproc = AMX_SPMDMyProc(); numprocs = AMX_SPMDNumProcs(); if (argc > 1) iters = atoi(argv[1]); if (!iters) iters = 1; if (argc > 2) size = atoi(argv[2]); if (size == -1) size = 512; if (argc > 3) { switch(argv[3][0]) { case 'p': case 'P': polling = 1; break; case 'b': case 'B': polling = 0; break; default: printf("polling must be 'P' or 'B'..\n"); AMX_SPMDExit(1); } } if (argc > 4) { switch(argv[4][0]) { case 'h': case 'H': fullduplex = 0; break; case 'f': case 'F': fullduplex = 1; break; default: printf("duplex must be H or F..\n"); AMX_SPMDExit(1); } } if (!fullduplex && numprocs > 1 && numprocs % 2 != 0) { printf("half duplex requires an even number of processors\n"); AMX_SPMDExit(1); } msg_size = (size > AM_MaxLong() ? AM_MaxLong() : size); nummsgs = (size % AM_MaxLong() == 0 ? size / AM_MaxLong() : (size / AM_MaxLong())+1); srcmem = (uint32_t *)malloc(msg_size); memset(srcmem, 0, msg_size); VMseg = (uint32_t *)malloc(msg_size+100); memset(VMseg, 0, msg_size+100); AM_Safe(AM_SetSeg(ep, VMseg, msg_size+100)); rightguy = (myproc + 1) % numprocs; { /* init my source mem */ int i; int numints = msg_size/4; for (i=0; i < numints; i++) srcmem[i] = i; } AM_Safe(AMX_SPMDBarrier()); if (myproc == 0) printf("Running %s bulk test sz=%i...\n", (fullduplex?"full-duplex":"half-duplex"), size); begin = getCurrentTimeMicrosec(); if (fullduplex || myproc % 2 == 1 || numprocs == 1) { int q; for (q=0; q<iters; q++) { int j; msg_size = AM_MaxLong(); for (j = 0; j < nummsgs; j++) { if (j == nummsgs-1 && size % AM_MaxLong() != 0) msg_size = size % AM_MaxLong(); /* last one */ #if VERBOSE_PING printf("%i: sending request...", myproc); fflush(stdout); #endif AM_Safe(AM_RequestXfer1(ep, rightguy, 100, BULK_REQ_HANDLER, srcmem, msg_size, 666)); } } } { int expectedmsgs = nummsgs*iters; if (numprocs == 1 || fullduplex) expectedmsgs *= 2; if (polling) { /* poll until everyone done */ while (done<expectedmsgs) { AM_Safe(AM_Poll(eb)); } } else { while (done<expectedmsgs) { AM_Safe(AM_SetEventMask(eb, AM_NOTEMPTY)); AM_Safe(AM_WaitSema(eb)); AM_Safe(AM_Poll(eb)); } } } end = getCurrentTimeMicrosec(); total = end - begin; if (fullduplex || myproc % 2 == 1 || numprocs == 1) printf("Slave %i: %i microseconds total, throughput: %8.3f KB/sec\n", myproc, (int)total, (float)(((float)1000000)*size*iters/((int)total))/1024.0); fflush(stdout); /* dump stats */ AM_Safe(AMX_SPMDBarrier()); printGlobalStats(); AM_Safe(AMX_SPMDBarrier()); /* exit */ AM_Safe(AMX_SPMDExit(0)); return 0; }