static void native_collective(pami_xfer_t *op_p, int need_lock) { pami_result_t rc; volatile unsigned int counter = 0; op_p->cb_done = &gasnetc_cb_inc_uint; op_p->cookie = (void *)&counter; op_p->options.multicontext = PAMI_HINT_DISABLE; if (need_lock) GASNETC_PAMI_LOCK(gasnetc_context); rc = PAMI_Collective(gasnetc_context, op_p); GASNETC_PAMI_CHECK(rc, "initiating a native collective"); if (need_lock) GASNETC_PAMI_UNLOCK(gasnetc_context); if (gasneti_attach_done) { gasneti_polluntil(counter); } else { rc = gasnetc_wait_uint(gasnetc_context, &counter, 1); GASNETC_PAMI_CHECK(rc, "polling a native collective"); } }
static void gasnete_coll_pami_scatt(const gasnet_team_handle_t team, void *dst, gasnet_image_t srcimage, const void *src, size_t nbytes, int flags GASNETI_THREAD_FARG) { const int i_am_root = gasnete_coll_image_is_local(team, srcimage); #if GASNET_PAR int i_am_leader = gasnete_coll_pami_images_barrier(team); /* XXX: over-synced for IN_NO and IN_MY */ if ((flags & GASNET_COLL_LOCAL) && i_am_root) { /* root thread must be leader for its node */ const gasnete_coll_threaddata_t * const td = GASNETE_COLL_MYTHREAD_NOALLOC; i_am_leader = (srcimage == td->my_image); } #else const int i_am_leader = 1; #endif if (i_am_leader) { volatile unsigned int done = 0; pami_result_t rc; pami_xfer_t op; if (flags & GASNET_COLL_IN_ALLSYNC) gasnetc_fast_barrier(); op = gasnete_op_template_scatt; op.cookie = (void *)&done; op.algorithm = team->pami.scatt_alg; op.cmd.xfer_scatter.root = gasnetc_endpoint(GASNETE_COLL_REL2ACT(team,gasnete_coll_image_node(team, srcimage))); op.cmd.xfer_scatter.sndbuf = (/*not-const*/ void *)src; op.cmd.xfer_scatter.stypecount = nbytes; op.cmd.xfer_scatter.rcvbuf = dst; op.cmd.xfer_scatter.rtypecount = nbytes; GASNETC_PAMI_LOCK(gasnetc_context); rc = PAMI_Collective(gasnetc_context, &op); GASNETC_PAMI_UNLOCK(gasnetc_context); GASNETC_PAMI_CHECK(rc, "initiating blocking scatter"); gasneti_polluntil(done); } if (flags & GASNET_COLL_OUT_ALLSYNC) { if (i_am_leader) gasnetc_fast_barrier(); (void) gasnete_coll_pami_images_barrier(team); } }
static void gasnete_coll_pami_allga(const gasnet_team_handle_t team, void *dst, const void *src, size_t nbytes, int flags GASNETE_THREAD_FARG) { #if GASNET_PAR int i_am_leader = gasnete_coll_pami_images_barrier(team); /* XXX: over-synced for IN_NO and IN_MY */ #else const int i_am_leader = 1; #endif if (i_am_leader) { volatile unsigned int done = 0; pami_result_t rc; pami_xfer_t op; if (flags & GASNET_COLL_IN_ALLSYNC) gasnetc_fast_barrier(); op = gasnete_op_template_allga; op.cookie = (void *)&done; op.algorithm = team->pami.allga_alg; op.cmd.xfer_allgather.sndbuf = (/*not-const*/ void *)src; op.cmd.xfer_allgather.stypecount = nbytes; op.cmd.xfer_allgather.rcvbuf = dst; op.cmd.xfer_allgather.rtypecount = nbytes; GASNETC_PAMI_LOCK(gasnetc_context); rc = PAMI_Collective(gasnetc_context, &op); GASNETC_PAMI_UNLOCK(gasnetc_context); GASNETC_PAMI_CHECK(rc, "initiating blocking allgather"); gasneti_polluntil(done); } if (flags & GASNET_COLL_OUT_ALLSYNC) { if (i_am_leader) gasnetc_fast_barrier(); (void) gasnete_coll_pami_images_barrier(team); } }
/* Get the default algorithm for a given (geometery, collective) pair. This will be the first "always works" algorithm unless user provides an override. */ extern void gasnetc_dflt_coll_alg(pami_geometry_t geom, pami_xfer_type_t op, pami_algorithm_t *alg_p) { static int print_once[PAMI_XFER_COUNT]; /* static var must initially be all zeros */ pami_result_t rc; size_t counts[2]; pami_algorithm_t *algorithms; pami_metadata_t *metadata; const char *envvar, *envval, *dfltval; int alg, fullcount; gasneti_assert(op >= 0); gasneti_assert(op < PAMI_XFER_COUNT); rc = PAMI_Geometry_algorithms_num(geom, op, counts); GASNETC_PAMI_CHECK(rc, "calling PAMI_Geometry_algorithms_num()"); fullcount = counts[0] + counts[1]; /* Space for algorithms and metadata */ algorithms = alloca(fullcount * sizeof(pami_algorithm_t)); metadata = alloca(fullcount * sizeof(pami_metadata_t)); rc = PAMI_Geometry_algorithms_query(geom, op, algorithms, metadata, counts[0], algorithms+counts[0], metadata+counts[0], counts[1]); GASNETC_PAMI_CHECK(rc, "calling PAMI_Geometry_algorithms_query()"); /* Process environment and defaults: */ switch(op) { /* please keep alphabetical */ /* Used for blocking gasnet exchange: */ case PAMI_XFER_ALLTOALL: envvar = "GASNET_PAMI_ALLTOALL_ALG"; #if GASNETI_ARCH_BGQ dfltval = "I0:M2MComposite:MU:MU"; /* Best on BG/Q by a large margin */ #else dfltval = "I0:Ring:"; /* Uniformly 2nd place (out of 3) on PERCS */ #endif break; /* Used for blocking gasnet exchange w/ multiple images: */ case PAMI_XFER_ALLTOALLV_INT: envvar = "GASNET_PAMI_ALLTOALLV_INT_ALG"; #if GASNETI_ARCH_BGQ dfltval = "I0:M2MComposite:MU:MU"; /* Best on BG/Q by a large margin */ #else dfltval = "I0:M2MComposite:"; /* Best on PERCS for all but smallest len (where it is close) */ #endif break; /* Used for blocking gasnet gatherall and gasnetc_bootstrapExchange(): */ case PAMI_XFER_ALLGATHER: envvar = "GASNET_PAMI_ALLGATHER_ALG"; #if GASNETI_ARCH_BGQ && 0 /* TODO: split choice based on size */ dfltval = "I0:RectangleDput:"; /* Uniformly best (or very near) for LARGE case only... */ /* .. but only available for "rectangular" jobs. */ alg = gasnetc_find_alg(dfltval, metadata, counts[0]); if (alg < counts[0]) break; /* Otherwise fall through */ #endif dfltval = "I0:Binomial:"; /* Uniformly 2nd place (out of 3) on PERCS, uniformly "OK" on BG/Q */ break; /* Used for blocking gasnet gatherall w/ multiple images: */ case PAMI_XFER_ALLGATHERV_INT: envvar = "GASNET_PAMI_ALLGATHERV_INT_ALG"; #if GASNETI_ARCH_BGQ && 0 /* TODO: split choice based on size */ dfltval = "I0:RectangleDput:"; /* Uniformly best for small to moderate cases only ... */ /* .. but only available for "rectangular" jobs. */ alg = gasnetc_find_alg(dfltval, metadata, counts[0]); if (alg < counts[0]) break; /* Otherwise fall through */ #endif dfltval = NULL; /* Only one other option available on systems I've tested -PHH */ break; /* Used for exitcode reduction and "PAMIALLREDUCE" barrier: */ case PAMI_XFER_ALLREDUCE: envvar = "GASNET_PAMI_ALLREDUCE_ALG"; #if GASNETI_ARCH_BGQ dfltval = "I0:Binomial:-:ShortMU"; /* great when available */ alg = gasnetc_find_alg(dfltval, metadata, fullcount); if (alg < fullcount) { /* make sure PAMI-allreduce barrier and exitcode reduction will "fit" */ pami_metadata_t *md = &metadata[alg]; if ((!md->check_correct.values.rangeminmax || md->range_lo <= sizeof(int)) && (!md->check_correct.values.rangeminmax || md->range_hi >= 2*sizeof(uint64_t)) && (!md->check_correct.values.sendminalign || md->send_min_align >= sizeof(char)) && (!md->check_correct.values.recvminalign || md->recv_min_align >= sizeof(char))) { break; /* Otherwise fall through */ } } dfltval = "I1:ShortAllreduce:"; /* excellent second choice on BG/Q */ alg = gasnetc_find_alg(dfltval, metadata, fullcount); if (alg < fullcount) { /* make sure PAMI-allreduce barrier and exitcode reduction will "fit" */ pami_metadata_t *md = &metadata[alg]; if ((!md->check_correct.values.rangeminmax || md->range_lo <= sizeof(int)) && (!md->check_correct.values.rangeminmax || md->range_hi >= 2*sizeof(uint64_t)) && (!md->check_correct.values.sendminalign || md->send_min_align >= sizeof(char)) && (!md->check_correct.values.recvminalign || md->recv_min_align >= sizeof(char))) { break; /* Otherwise fall through */ } } #endif dfltval = "I0:Binomial:"; /* uniformly "good" on BG/Q and PERCS */ break; /* Used for gasnetc_fast_barrier() and GASNET_BARRIERFLAG_UNNAMED */ case PAMI_XFER_BARRIER: envvar = "GASNET_PAMI_BARRIER_ALG"; #if GASNETI_ARCH_BGQ /* Note: this could be any of * "I0:MultiSync2Device:SHMEM:GI" * "I0:MultiSync:SHMEM:-", * "I0:MultiSync:-:GI", * depending on job layout, and may not be available on team != ALL. */ dfltval = "I0:MultiSync"; alg = gasnetc_find_alg(dfltval, metadata, counts[0]); if (alg < counts[0]) break; /* Otherwise fall through */ #endif dfltval = NULL; /* TODO: tune a better default than alg[0]? */ break; /* Used for blocking gasnet broadcast: */ case PAMI_XFER_BROADCAST: envvar = "GASNET_PAMI_BROADCAST_ALG"; #if GASNETI_ARCH_BGQ dfltval = "I0:2-nary:"; /* uniformly "near-best" on BG/Q */ #else #if 0 /* Seen to deadlock when using multiple procs/node */ dfltval = "I0:4-nary:"; /* uniformly "near-best" on PERSC */ #else dfltval = NULL; /* TODO: tune for better default or wait for bug fix */ #endif #endif break; /* Used for blocking gasnet gather: */ case PAMI_XFER_GATHER: envvar = "GASNET_PAMI_GATHER_ALG"; dfltval = NULL; /* TODO: tune for better default */ break; /* Used for blocking gasnet gather w/ multiple images: */ case PAMI_XFER_GATHERV_INT: envvar = "GASNET_PAMI_GATHERV_INT_ALG"; dfltval = NULL; /* TODO: tune for better default */ break; /* Used for blocking gasnet scatter: */ case PAMI_XFER_SCATTER: envvar = "GASNET_PAMI_SCATTER_ALG"; dfltval = "I0:Binomial:"; /* uniformly "good" on BG/Q and PERSC */ break; /* Used for blocking gasnet scatter w/ multiple images: */ case PAMI_XFER_SCATTERV_INT: envvar = "GASNET_PAMI_SCATTERV_INT_ALG"; dfltval = NULL; /* TODO: tune for better default */ break; default: gasneti_fatalerror("Unknown 'op' value %d in %s", (int)op, __FUNCTION__); envvar = dfltval = NULL; /* for warning suppression only */ } /* Override the defaults above for the single-task case: */ if (gasneti_nodes == 1) { const char *onetask = "I0:OneTask"; if (gasnetc_find_alg(onetask, metadata, counts[0]) < counts[0]) { dfltval = onetask; } } /* Now the user's environment value if any: */ envval = gasneti_getenv_withdefault(envvar, dfltval); alg = 0; /* failsafe */ if (NULL != envval) { while (envval[0] && isspace(envval[0])) ++envval; /* leading whitespace */ if (!envval[0]) { /* empty - treat as zero */ } else if (0 == strcmp("LIST", envval)) { if (!gasneti_mynode && !print_once[(int)op]) { int i; fprintf(stderr, "Listing available values for environment variable %s:\n", envvar); for (i=0; i<fullcount; ++i) { fprintf(stderr, " %c %3d %s\n", ((i<counts[0])?' ':'*'), i, metadata[i].name); } if (counts[1]) { fprintf(stderr, "Note: Lines marked with '*' may not be valid for all inputs and/or job layouts.\n" " The user is responsible for ensuring only valid algorithms are requested.\n" ); } print_once[(int)op] = 1; } } else if (isdigit(envval[0])) { /* integer is used just as given */ alg = atoi(envval); if (alg < 0 || alg >= fullcount) { if (!gasneti_mynode && !print_once[(int)op]) { fprintf(stderr, "WARNING: Ignoring value '%d' for environment variable %s,\n" " because it is outside the range of available algorithms.\n" " Set this variable to LIST for a list of all algorithms.\n", alg, envvar); print_once[(int)op] = 1; } alg = 0; } } else { /* string is used for PREFIX match */ alg = gasnetc_find_alg(envval, metadata, fullcount); if (alg == fullcount) { if (!gasneti_mynode && !print_once[(int)op] && (envval != dfltval)) { fprintf(stderr, "WARNING: Ignoring value '%s' for environment variable %s,\n" " because it does not match any available algorithm.\n" " Set this variable to LIST for a list of all algorithms.\n", envval, envvar); print_once[(int)op] = 1; } alg = 0; } } } *alg_p = algorithms[alg]; }
static void gasnete_coll_pami_allgavi(const gasnet_team_handle_t team, void *dst, const void *src, size_t nbytes, int flags GASNETE_THREAD_FARG) { int i_am_leader = gasnete_coll_pami_images_barrier(team); /* XXX: over-synced for IN_NO and IN_MY */ const gasnete_coll_threaddata_t * const td = GASNETE_COLL_MYTHREAD_NOALLOC; if (flags & GASNET_COLL_IN_ALLSYNC) { if (i_am_leader) gasnetc_fast_barrier(); (void) gasnete_coll_pami_images_barrier(team); } GASNETE_FAST_UNALIGNED_MEMCPY(gasnete_coll_scale_ptr(team->pami.scratch_space, td->my_local_image, nbytes), src, nbytes); (void) gasnete_coll_pami_images_barrier(team); if (i_am_leader) { volatile unsigned int done = 0; pami_result_t rc; pami_xfer_t op; op = gasnete_op_template_allgavi; /* allgatherv_int */ op.cookie = (void *)&done; op.algorithm = team->pami.allgavi_alg; op.cmd.xfer_allgatherv_int.sndbuf = team->pami.scratch_space; op.cmd.xfer_allgatherv_int.stypecount = nbytes * team->my_images; op.cmd.xfer_allgatherv_int.rcvbuf = dst; op.cmd.xfer_allgatherv_int.rtypecounts = team->pami.counts; op.cmd.xfer_allgatherv_int.rdispls = team->pami.displs; if (team->pami.prev_nbytes != nbytes) { int i; for (i = 0; i < team->total_ranks; ++i) { op.cmd.xfer_allgatherv_int.rtypecounts[i] = nbytes * team->all_images[i]; op.cmd.xfer_allgatherv_int.rdispls[i] = nbytes * team->all_offset[i]; } team->pami.prev_nbytes = nbytes; } GASNETC_PAMI_LOCK(gasnetc_context); rc = PAMI_Collective(gasnetc_context, &op); GASNETC_PAMI_UNLOCK(gasnetc_context); GASNETC_PAMI_CHECK(rc, "initiating blocking allgatherv_int"); gasneti_polluntil(done); gasneti_assert(NULL == team->pami.tmp_addr); gasneti_sync_writes(); /* XXX: is this necessary? */ team->pami.tmp_addr = dst; /* wakes pollers, below */ (void) gasnete_coll_pami_images_barrier(team); /* matches instance below vvvv */ team->pami.tmp_addr = NULL; } else { gasneti_waitwhile(NULL == team->pami.tmp_addr); GASNETE_FAST_UNALIGNED_MEMCPY(dst, team->pami.tmp_addr, nbytes * team->total_images); (void) gasnete_coll_pami_images_barrier(team); /* matches instance above ^^^^ */ } if (flags & GASNET_COLL_OUT_ALLSYNC) { if (i_am_leader) gasnetc_fast_barrier(); (void) gasnete_coll_pami_images_barrier(team); } }
static void gasnete_coll_pami_scattvi(const gasnet_team_handle_t team, void *dst, gasnet_image_t srcimage, const void *src, size_t nbytes, int flags GASNETI_THREAD_FARG) { const int i_am_root = gasnete_coll_image_is_local(team, srcimage); int i_am_leader = gasnete_coll_pami_images_barrier(team); /* XXX: over-synced for IN_NO and IN_MY */ const gasnete_coll_threaddata_t * const td = GASNETE_COLL_MYTHREAD_NOALLOC; if ((flags & GASNET_COLL_LOCAL) && i_am_root) { /* root thread must be leader for its node */ i_am_leader = (srcimage == td->my_image); } if (i_am_leader) { volatile unsigned int done = 0; pami_result_t rc; pami_xfer_t op; if (flags & GASNET_COLL_IN_ALLSYNC) gasnetc_fast_barrier(); op = gasnete_op_template_scattvi; /* scatterv_int */ op.cookie = (void *)&done; op.algorithm = team->pami.scattvi_alg; op.cmd.xfer_scatterv_int.root = gasnetc_endpoint(GASNETE_COLL_REL2ACT(team,gasnete_coll_image_node(team, srcimage))); op.cmd.xfer_scatterv_int.rcvbuf = team->pami.scratch_space; op.cmd.xfer_scatterv_int.rtypecount = nbytes * team->my_images; if (i_am_root) { op.cmd.xfer_scatterv_int.sndbuf = (/*not-const*/ void *)src; op.cmd.xfer_scatterv_int.stypecounts = team->pami.counts; op.cmd.xfer_scatterv_int.sdispls = team->pami.displs; if (team->pami.prev_nbytes != nbytes) { int i; for (i = 0; i < team->total_ranks; ++i) { op.cmd.xfer_scatterv_int.stypecounts[i] = nbytes * team->all_images[i]; op.cmd.xfer_scatterv_int.sdispls[i] = nbytes * team->all_offset[i]; } team->pami.prev_nbytes = nbytes; } } GASNETC_PAMI_LOCK(gasnetc_context); rc = PAMI_Collective(gasnetc_context, &op); GASNETC_PAMI_UNLOCK(gasnetc_context); GASNETC_PAMI_CHECK(rc, "initiating blocking scatterv_int"); gasneti_polluntil(done); gasneti_assert(NULL == team->pami.tmp_addr); gasneti_sync_writes(); team->pami.tmp_addr = team->pami.scratch_space; /* wakes pollers, below */ } else { gasneti_waitwhile(NULL == team->pami.tmp_addr); } GASNETI_MEMCPY (dst, gasnete_coll_scale_ptr(team->pami.tmp_addr, td->my_local_image, nbytes), nbytes); (void) gasnete_coll_pami_images_barrier(team); if (i_am_leader) { team->pami.tmp_addr = NULL; } if (flags & GASNET_COLL_OUT_ALLSYNC) { if (i_am_leader) gasnetc_fast_barrier(); (void) gasnete_coll_pami_images_barrier(team); } }