/**
* \brief Blocking 'world geometry' barrier
*
* This function is provided for illustrative purposes only. One would never
* include the retrieval of the world geometry and the query of the barrier
* algorithm in a performance critical code.
*
* \param[in] client The PAMI client; needed to obtain the geometry
* \param[in] context The PAMI context; used for the barrier communication
*/
void simple_barrier (pami_client_t client, pami_context_t context)
{
pami_result_t result;
pami_geometry_t world_geometry;
pami_xfer_t xfer;
pami_algorithm_t algorithm;
pami_metadata_t metadata;
/* Retrieve the PAMI 'world' geometry */
result = PAMI_ERROR;
result = PAMI_Geometry_world (client, &world_geometry);
assert (result == PAMI_SUCCESS);
/* Query the 'always works' barrier algorithm in the geometry */
result = PAMI_ERROR;
result = PAMI_Geometry_algorithms_query (world_geometry, PAMI_XFER_BARRIER,
&algorithm, &metadata, 1,
NULL, NULL, 0);
assert (result == PAMI_SUCCESS);
/* Set up the barrier */
volatile unsigned active = 1;
xfer.cb_done = simple_barrier_decrement;
xfer.cookie = (void *) & active;
xfer.algorithm = algorithm;
/* Issue the barrier collective */
result = PAMI_ERROR;
result = PAMI_Collective (context, &xfer);
assert (result == PAMI_SUCCESS);
/* Advance until the barrier has completed */
while (active)
{
result = PAMI_ERROR;
result = PAMI_Context_advance (context, 1);
assert (result == PAMI_SUCCESS);
}
return;
}
int main(int argc, char* argv[])
{
pami_result_t result = PAMI_ERROR;
if (Kernel_GetRank()==0)
print_meminfo(stdout, "before PAMI_Client_create");
/* initialize the client */
char * clientname = "";
pami_client_t client;
result = PAMI_Client_create( clientname, &client, NULL, 0 );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_create");
if (Kernel_GetRank()==0)
print_meminfo(stdout, "after PAMI_Client_create");
/* query properties of the client */
pami_configuration_t config;
size_t num_contexts;
config.name = PAMI_CLIENT_TASK_ID;
result = PAMI_Client_query( client, &config, 1);
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query");
world_rank = config.value.intval;
config.name = PAMI_CLIENT_NUM_TASKS;
result = PAMI_Client_query( client, &config, 1);
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query");
world_size = config.value.intval;
if ( world_rank == 0 )
{
printf("starting test on %ld ranks \n", world_size);
fflush(stdout);
}
config.name = PAMI_CLIENT_PROCESSOR_NAME;
result = PAMI_Client_query( client, &config, 1);
assert(result == PAMI_SUCCESS);
//printf("rank %ld is processor %s \n", world_rank, config.value.chararray);
//fflush(stdout);
config.name = PAMI_CLIENT_NUM_CONTEXTS;
result = PAMI_Client_query( client, &config, 1);
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_query");
num_contexts = config.value.intval;
/* initialize the contexts */
pami_context_t * contexts = NULL;
contexts = (pami_context_t *) malloc( num_contexts * sizeof(pami_context_t) ); assert(contexts!=NULL);
if (Kernel_GetRank()==0)
fprintf(stdout, "num_contexts = %ld \n", (long)num_contexts);
result = PAMI_Context_createv( client, &config, 0, contexts, num_contexts );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_createv");
if (Kernel_GetRank()==0)
print_meminfo(stdout, "after PAMI_Context_createv");
/* setup the world geometry */
pami_geometry_t world_geometry;
pami_xfer_type_t barrier_xfer = PAMI_XFER_BARRIER;
size_t num_alg[2];
pami_algorithm_t * safe_barrier_algs = NULL;
pami_metadata_t * safe_barrier_meta = NULL;
pami_algorithm_t * fast_barrier_algs = NULL;
pami_metadata_t * fast_barrier_meta = NULL;
result = PAMI_Geometry_world( client, &world_geometry );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Geometry_world");
if (Kernel_GetRank()==0)
print_meminfo(stdout, "after PAMI_Geometry_world");
result = PAMI_Geometry_algorithms_num( world_geometry, barrier_xfer, num_alg );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Geometry_algorithms_num");
if ( world_rank == 0 ) printf("number of barrier algorithms = {%ld,%ld} \n", num_alg[0], num_alg[1] );
if (Kernel_GetRank()==0)
print_meminfo(stdout, "after PAMI_Geometry_algorithms_num");
safe_barrier_algs = (pami_algorithm_t *) malloc( num_alg[0] * sizeof(pami_algorithm_t) ); assert(safe_barrier_algs!=NULL);
safe_barrier_meta = (pami_metadata_t *) malloc( num_alg[0] * sizeof(pami_metadata_t) ); assert(safe_barrier_meta!=NULL);
fast_barrier_algs = (pami_algorithm_t *) malloc( num_alg[1] * sizeof(pami_algorithm_t) ); assert(fast_barrier_algs!=NULL);
fast_barrier_meta = (pami_metadata_t *) malloc( num_alg[1] * sizeof(pami_metadata_t) ); assert(fast_barrier_meta!=NULL);
result = PAMI_Geometry_algorithms_query( world_geometry, barrier_xfer,
safe_barrier_algs, safe_barrier_meta, num_alg[0],
fast_barrier_algs, fast_barrier_meta, num_alg[1] );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Geometry_algorithms_query");
if (Kernel_GetRank()==0)
print_meminfo(stdout, "after PAMI_Geometry_algorithms_query");
/* perform a barrier */
size_t b;
pami_xfer_t barrier;
volatile int active = 0;
for ( b = 0 ; b < num_alg[0] ; b++ )
{
barrier.cb_done = cb_done;
barrier.cookie = (void*) &active;
barrier.algorithm = safe_barrier_algs[b];
uint64_t t0 = GetTimeBase();
active = 1;
result = PAMI_Collective( contexts[0], &barrier );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Collective - barrier");
while (active)
result = PAMI_Context_advance( contexts[0], 1 );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_advance - barrier");
uint64_t t1 = GetTimeBase();
if ( world_rank == 0 ) printf("safe barrier algorithm %ld (%s) - took %llu cycles \n", b, safe_barrier_meta[b].name, (long long unsigned int)t1-t0 );
fflush(stdout);
}
for ( b = 0 ; b < num_alg[1] ; b++ )
{
barrier.cb_done = cb_done;
barrier.cookie = (void*) &active;
barrier.algorithm = fast_barrier_algs[b];
uint64_t t0 = GetTimeBase();
active = 1;
result = PAMI_Collective( contexts[0], &barrier );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Collective - barrier");
while (active)
result = PAMI_Context_advance( contexts[0], 1 );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_advance - barrier");
uint64_t t1 = GetTimeBase();
if ( world_rank == 0 ) printf("fast barrier algorithm %ld (%s) - took %llu cycles \n", b, fast_barrier_meta[b].name, (long long unsigned int)t1-t0 );
fflush(stdout);
}
if (Kernel_GetRank()==0)
print_meminfo(stdout, "after barrier tests");
/* finalize the contexts */
result = PAMI_Context_destroyv( contexts, num_contexts );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Context_destroyv");
free(contexts);
if (Kernel_GetRank()==0)
print_meminfo(stdout, "before PAMI_Client_destroy");
/* finalize the client */
result = PAMI_Client_destroy( &client );
TEST_ASSERT(result == PAMI_SUCCESS,"PAMI_Client_destroy");
if (Kernel_GetRank()==0)
print_meminfo(stdout, "after PAMI_Client_destroy");
if ( world_rank == 0 )
{
printf("end of test \n");
fflush(stdout);
}
return 0;
}
/* Determine how many of each collective type this communicator supports */
void MPIDI_Comm_coll_query(MPID_Comm *comm)
{
TRACE_ERR("MPIDI_Comm_coll_query enter\n");
int rc = 0, i, j;
size_t num_algorithms[2];
TRACE_ERR("Getting geometry from comm %p\n", comm);
pami_geometry_t *geom = comm->mpid.geometry;;
for(i = 0; i < PAMI_XFER_COUNT; i++)
{
if(i == PAMI_XFER_AMBROADCAST || i == PAMI_XFER_AMSCATTER ||
i == PAMI_XFER_AMGATHER || i == PAMI_XFER_AMREDUCE)
continue;
rc = PAMI_Geometry_algorithms_num(geom,
i,
num_algorithms);
TRACE_ERR("Num algorithms of type %d: %zd %zd\n", i, num_algorithms[0], num_algorithms[1]);
if(rc != PAMI_SUCCESS)
{
fprintf(stderr,"PAMI_Geometry_algorithms_num returned %d for type %d\n", rc, i);
continue;
}
comm->mpid.coll_count[i][0] = 0;
comm->mpid.coll_count[i][1] = 0;
if(num_algorithms[0] || num_algorithms[1])
{
comm->mpid.coll_algorithm[i][0] = (pami_algorithm_t *)
MPIU_Malloc(sizeof(pami_algorithm_t) * num_algorithms[0]);
comm->mpid.coll_metadata[i][0] = (pami_metadata_t *)
MPIU_Malloc(sizeof(pami_metadata_t) * num_algorithms[0]);
comm->mpid.coll_algorithm[i][1] = (pami_algorithm_t *)
MPIU_Malloc(sizeof(pami_algorithm_t) * num_algorithms[1]);
comm->mpid.coll_metadata[i][1] = (pami_metadata_t *)
MPIU_Malloc(sizeof(pami_metadata_t) * num_algorithms[1]);
comm->mpid.coll_count[i][0] = num_algorithms[0];
comm->mpid.coll_count[i][1] = num_algorithms[1];
/* Despite the bad name, this looks at algorithms associated with
* the geometry, NOT inherent physical properties of the geometry*/
/* BES TODO I am assuming all contexts have the same algorithms. Probably
* need to investigate that assumption
*/
rc = PAMI_Geometry_algorithms_query(geom,
i,
comm->mpid.coll_algorithm[i][0],
comm->mpid.coll_metadata[i][0],
num_algorithms[0],
comm->mpid.coll_algorithm[i][1],
comm->mpid.coll_metadata[i][1],
num_algorithms[1]);
if(rc != PAMI_SUCCESS)
{
fprintf(stderr,"PAMI_Geometry_algorithms_query returned %d for type %d\n", rc, i);
continue;
}
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
{
for(j = 0; j < num_algorithms[0]; j++)
fprintf(stderr,"comm[%p] coll type %d (%s), algorithm %d 0: %s\n", comm, i, MPIDI_Coll_type_name(i), j, comm->mpid.coll_metadata[i][0][j].name);
for(j = 0; j < num_algorithms[1]; j++)
fprintf(stderr,"comm[%p] coll type %d (%s), algorithm %d 1: %s\n", comm, i, MPIDI_Coll_type_name(i), j, comm->mpid.coll_metadata[i][1][j].name);
if(i == PAMI_XFER_ALLGATHERV_INT || i == PAMI_XFER_ALLGATHER)
{
fprintf(stderr,"comm[%p] coll type %d (%s), \"glue\" algorithm: GLUE_ALLREDUCE\n", comm, i, MPIDI_Coll_type_name(i));
fprintf(stderr,"comm[%p] coll type %d (%s), \"glue\" algorithm: GLUE_BCAST\n", comm, i, MPIDI_Coll_type_name(i));
fprintf(stderr,"comm[%p] coll type %d (%s), \"glue\" algorithm: GLUE_ALLTOALL\n", comm, i, MPIDI_Coll_type_name(i));
}
if(i == PAMI_XFER_SCATTERV_INT)
{
fprintf(stderr,"comm[%p] coll type %d (%s), \"glue\" algorithm: GLUE_BCAST\n", comm, i, MPIDI_Coll_type_name(i));
fprintf(stderr,"comm[%p] coll type %d (%s), \"glue\" algorithm: GLUE_ALLTOALLV\n", comm, i, MPIDI_Coll_type_name(i));
}
if(i == PAMI_XFER_SCATTER)
{
fprintf(stderr,"comm[%p] coll type %d (%s), \"glue\" algorithm: GLUE_BCAST\n", comm, i, MPIDI_Coll_type_name(i));
}
}
}
}
/* Determine if we have protocols for these maybe, rather than just setting them? */
comm->coll_fns->Barrier = MPIDO_Barrier;
comm->coll_fns->Bcast = MPIDO_Bcast;
comm->coll_fns->Allreduce = MPIDO_Allreduce;
comm->coll_fns->Allgather = MPIDO_Allgather;
comm->coll_fns->Allgatherv = MPIDO_Allgatherv;
comm->coll_fns->Scatterv = MPIDO_Scatterv;
comm->coll_fns->Scatter = MPIDO_Scatter;
comm->coll_fns->Gather = MPIDO_Gather;
comm->coll_fns->Alltoallv = MPIDO_Alltoallv;
comm->coll_fns->Alltoall = MPIDO_Alltoall;
comm->coll_fns->Gatherv = MPIDO_Gatherv;
comm->coll_fns->Reduce = MPIDO_Reduce;
comm->coll_fns->Scan = MPIDO_Scan;
comm->coll_fns->Exscan = MPIDO_Exscan;
TRACE_ERR("MPIDI_Comm_coll_query exit\n");
}
/* Get the default algorithm for a given (geometery, collective) pair.
This will be the first "always works" algorithm unless user provides an override.
*/
extern void
gasnetc_dflt_coll_alg(pami_geometry_t geom, pami_xfer_type_t op, pami_algorithm_t *alg_p) {
static int print_once[PAMI_XFER_COUNT]; /* static var must initially be all zeros */
pami_result_t rc;
size_t counts[2];
pami_algorithm_t *algorithms;
pami_metadata_t *metadata;
const char *envvar, *envval, *dfltval;
int alg, fullcount;
gasneti_assert(op >= 0);
gasneti_assert(op < PAMI_XFER_COUNT);
rc = PAMI_Geometry_algorithms_num(geom, op, counts);
GASNETC_PAMI_CHECK(rc, "calling PAMI_Geometry_algorithms_num()");
fullcount = counts[0] + counts[1];
/* Space for algorithms and metadata */
algorithms = alloca(fullcount * sizeof(pami_algorithm_t));
metadata = alloca(fullcount * sizeof(pami_metadata_t));
rc = PAMI_Geometry_algorithms_query(geom, op,
algorithms, metadata, counts[0],
algorithms+counts[0], metadata+counts[0], counts[1]);
GASNETC_PAMI_CHECK(rc, "calling PAMI_Geometry_algorithms_query()");
/* Process environment and defaults: */
switch(op) { /* please keep alphabetical */
/* Used for blocking gasnet exchange: */
case PAMI_XFER_ALLTOALL:
envvar = "GASNET_PAMI_ALLTOALL_ALG";
#if GASNETI_ARCH_BGQ
dfltval = "I0:M2MComposite:MU:MU"; /* Best on BG/Q by a large margin */
#else
dfltval = "I0:Ring:"; /* Uniformly 2nd place (out of 3) on PERCS */
#endif
break;
/* Used for blocking gasnet exchange w/ multiple images: */
case PAMI_XFER_ALLTOALLV_INT:
envvar = "GASNET_PAMI_ALLTOALLV_INT_ALG";
#if GASNETI_ARCH_BGQ
dfltval = "I0:M2MComposite:MU:MU"; /* Best on BG/Q by a large margin */
#else
dfltval = "I0:M2MComposite:"; /* Best on PERCS for all but smallest len (where it is close) */
#endif
break;
/* Used for blocking gasnet gatherall and gasnetc_bootstrapExchange(): */
case PAMI_XFER_ALLGATHER:
envvar = "GASNET_PAMI_ALLGATHER_ALG";
#if GASNETI_ARCH_BGQ && 0 /* TODO: split choice based on size */
dfltval = "I0:RectangleDput:"; /* Uniformly best (or very near) for LARGE case only... */
/* .. but only available for "rectangular" jobs. */
alg = gasnetc_find_alg(dfltval, metadata, counts[0]);
if (alg < counts[0]) break; /* Otherwise fall through */
#endif
dfltval = "I0:Binomial:"; /* Uniformly 2nd place (out of 3) on PERCS, uniformly "OK" on BG/Q */
break;
/* Used for blocking gasnet gatherall w/ multiple images: */
case PAMI_XFER_ALLGATHERV_INT:
envvar = "GASNET_PAMI_ALLGATHERV_INT_ALG";
#if GASNETI_ARCH_BGQ && 0 /* TODO: split choice based on size */
dfltval = "I0:RectangleDput:"; /* Uniformly best for small to moderate cases only ... */
/* .. but only available for "rectangular" jobs. */
alg = gasnetc_find_alg(dfltval, metadata, counts[0]);
if (alg < counts[0]) break; /* Otherwise fall through */
#endif
dfltval = NULL; /* Only one other option available on systems I've tested -PHH */
break;
/* Used for exitcode reduction and "PAMIALLREDUCE" barrier: */
case PAMI_XFER_ALLREDUCE:
envvar = "GASNET_PAMI_ALLREDUCE_ALG";
#if GASNETI_ARCH_BGQ
dfltval = "I0:Binomial:-:ShortMU"; /* great when available */
alg = gasnetc_find_alg(dfltval, metadata, fullcount);
if (alg < fullcount) {
/* make sure PAMI-allreduce barrier and exitcode reduction will "fit" */
pami_metadata_t *md = &metadata[alg];
if ((!md->check_correct.values.rangeminmax || md->range_lo <= sizeof(int)) &&
(!md->check_correct.values.rangeminmax || md->range_hi >= 2*sizeof(uint64_t)) &&
(!md->check_correct.values.sendminalign || md->send_min_align >= sizeof(char)) &&
(!md->check_correct.values.recvminalign || md->recv_min_align >= sizeof(char))) {
break; /* Otherwise fall through */
}
}
dfltval = "I1:ShortAllreduce:"; /* excellent second choice on BG/Q */
alg = gasnetc_find_alg(dfltval, metadata, fullcount);
if (alg < fullcount) {
/* make sure PAMI-allreduce barrier and exitcode reduction will "fit" */
pami_metadata_t *md = &metadata[alg];
if ((!md->check_correct.values.rangeminmax || md->range_lo <= sizeof(int)) &&
(!md->check_correct.values.rangeminmax || md->range_hi >= 2*sizeof(uint64_t)) &&
(!md->check_correct.values.sendminalign || md->send_min_align >= sizeof(char)) &&
(!md->check_correct.values.recvminalign || md->recv_min_align >= sizeof(char))) {
break; /* Otherwise fall through */
}
}
#endif
dfltval = "I0:Binomial:"; /* uniformly "good" on BG/Q and PERCS */
break;
/* Used for gasnetc_fast_barrier() and GASNET_BARRIERFLAG_UNNAMED */
case PAMI_XFER_BARRIER:
envvar = "GASNET_PAMI_BARRIER_ALG";
#if GASNETI_ARCH_BGQ
/* Note: this could be any of
* "I0:MultiSync2Device:SHMEM:GI"
* "I0:MultiSync:SHMEM:-",
* "I0:MultiSync:-:GI",
* depending on job layout, and may not be available on team != ALL.
*/
dfltval = "I0:MultiSync";
alg = gasnetc_find_alg(dfltval, metadata, counts[0]);
if (alg < counts[0]) break; /* Otherwise fall through */
#endif
dfltval = NULL; /* TODO: tune a better default than alg[0]? */
break;
/* Used for blocking gasnet broadcast: */
case PAMI_XFER_BROADCAST:
envvar = "GASNET_PAMI_BROADCAST_ALG";
#if GASNETI_ARCH_BGQ
dfltval = "I0:2-nary:"; /* uniformly "near-best" on BG/Q */
#else
#if 0 /* Seen to deadlock when using multiple procs/node */
dfltval = "I0:4-nary:"; /* uniformly "near-best" on PERSC */
#else
dfltval = NULL; /* TODO: tune for better default or wait for bug fix */
#endif
#endif
break;
/* Used for blocking gasnet gather: */
case PAMI_XFER_GATHER:
envvar = "GASNET_PAMI_GATHER_ALG";
dfltval = NULL; /* TODO: tune for better default */
break;
/* Used for blocking gasnet gather w/ multiple images: */
case PAMI_XFER_GATHERV_INT:
envvar = "GASNET_PAMI_GATHERV_INT_ALG";
dfltval = NULL; /* TODO: tune for better default */
break;
/* Used for blocking gasnet scatter: */
case PAMI_XFER_SCATTER:
envvar = "GASNET_PAMI_SCATTER_ALG";
dfltval = "I0:Binomial:"; /* uniformly "good" on BG/Q and PERSC */
break;
/* Used for blocking gasnet scatter w/ multiple images: */
case PAMI_XFER_SCATTERV_INT:
envvar = "GASNET_PAMI_SCATTERV_INT_ALG";
dfltval = NULL; /* TODO: tune for better default */
break;
default:
gasneti_fatalerror("Unknown 'op' value %d in %s", (int)op, __FUNCTION__);
envvar = dfltval = NULL; /* for warning suppression only */
}
/* Override the defaults above for the single-task case: */
if (gasneti_nodes == 1) {
const char *onetask = "I0:OneTask";
if (gasnetc_find_alg(onetask, metadata, counts[0]) < counts[0]) {
dfltval = onetask;
}
}
/* Now the user's environment value if any: */
envval = gasneti_getenv_withdefault(envvar, dfltval);
alg = 0; /* failsafe */
if (NULL != envval) {
while (envval[0] && isspace(envval[0])) ++envval; /* leading whitespace */
if (!envval[0]) {
/* empty - treat as zero */
} else if (0 == strcmp("LIST", envval)) {
if (!gasneti_mynode && !print_once[(int)op]) {
int i;
fprintf(stderr, "Listing available values for environment variable %s:\n", envvar);
for (i=0; i<fullcount; ++i) {
fprintf(stderr, " %c %3d %s\n", ((i<counts[0])?' ':'*'), i, metadata[i].name);
}
if (counts[1]) {
fprintf(stderr,
"Note: Lines marked with '*' may not be valid for all inputs and/or job layouts.\n"
" The user is responsible for ensuring only valid algorithms are requested.\n"
);
}
print_once[(int)op] = 1;
}
} else if (isdigit(envval[0])) {
/* integer is used just as given */
alg = atoi(envval);
if (alg < 0 || alg >= fullcount) {
if (!gasneti_mynode && !print_once[(int)op]) {
fprintf(stderr, "WARNING: Ignoring value '%d' for environment variable %s,\n"
" because it is outside the range of available algorithms.\n"
" Set this variable to LIST for a list of all algorithms.\n",
alg, envvar);
print_once[(int)op] = 1;
}
alg = 0;
}
} else {
/* string is used for PREFIX match */
alg = gasnetc_find_alg(envval, metadata, fullcount);
if (alg == fullcount) {
if (!gasneti_mynode && !print_once[(int)op] && (envval != dfltval)) {
fprintf(stderr, "WARNING: Ignoring value '%s' for environment variable %s,\n"
" because it does not match any available algorithm.\n"
" Set this variable to LIST for a list of all algorithms.\n",
envval, envvar);
print_once[(int)op] = 1;
}
alg = 0;
}
}
}
*alg_p = algorithms[alg];
}
int main(int argc, char ** argv) {
pami_client_t client;
pami_context_t context;
size_t num_contexts = 1;
pami_task_t task_id;
size_t num_tasks;
pami_geometry_t world_geometry;
/* Barrier variables */
size_t num_algorithms[2];
pami_xfer_type_t barrier_xfer = PAMI_XFER_BARRIER;
pami_xfer_t barrier;
#if !defined(__bgq__)
pami_extension_t hfi_extension;
hfi_remote_update_fn hfi_update;
#endif
int numAlgorithms = 6;
pami_xfer_type_t algorithms[] = {PAMI_XFER_BROADCAST, PAMI_XFER_BARRIER,
PAMI_XFER_SCATTER, PAMI_XFER_ALLTOALL, PAMI_XFER_ALLREDUCE, PAMI_XFER_ALLGATHER};
const char* algorithmNames[] = {"PAMI_XFER_BROADCAST", "PAMI_XFER_BARRIER",
"PAMI_XFER_SCATTER", "PAMI_XFER_ALLTOALL", "PAMI_XFER_ALLREDUCE", "PAMI_XFER_ALLGATHER"};
const char *name = "X10";
setenv("MP_MSG_API", name, 1);
pami_configuration_t config;
config.name = PAMI_GEOMETRY_OPTIMIZE;
config.value.intval = 1;
pami_result_t status = PAMI_Client_create(name, &client, &config, 1);
if (status != PAMI_SUCCESS)
error("Unable to initialize PAMI client\n");
if ((status = PAMI_Context_createv(client, &config, 1, &context, 1)) != PAMI_SUCCESS)
error("Unable to initialize the PAMI context: %i\n", status);
#if !defined(__bgq__)
status = PAMI_Extension_open (client, "EXT_hfi_extension", &hfi_extension);
if (status == PAMI_SUCCESS)
{
#ifdef __GNUC__
__extension__
#endif
hfi_update = (hfi_remote_update_fn) PAMI_Extension_symbol(hfi_extension, "hfi_remote_update"); // This may succeed even if HFI is not available
}
#endif
status = PAMI_Geometry_world(client, &world_geometry);
if (status != PAMI_SUCCESS) error("Unable to create the world geometry");
pami_configuration_t configuration[2];
configuration[0].name = PAMI_CLIENT_TASK_ID;
configuration[1].name = PAMI_CLIENT_NUM_TASKS;
if ((status = PAMI_Client_query(client, configuration, 2)) != PAMI_SUCCESS)
error("Unable to query the PAMI_CLIENT: %i\n", status);
int myPlaceId = configuration[0].value.intval;
if (myPlaceId == 0)
{
for (int i = 0; i < numAlgorithms; i++) {
status = PAMI_Geometry_algorithms_num(world_geometry, algorithms[i], num_algorithms);
if (status != PAMI_SUCCESS || num_algorithms[0] == 0)
error("Unable to query the algorithm counts for barrier\n");
// query what the different algorithms are
pami_algorithm_t *always_works_alg = (pami_algorithm_t*) alloca(sizeof(pami_algorithm_t)*num_algorithms[0]);
pami_metadata_t *always_works_md = (pami_metadata_t*) alloca(sizeof(pami_metadata_t)*num_algorithms[0]);
pami_algorithm_t *must_query_alg = (pami_algorithm_t*) alloca(sizeof(pami_algorithm_t)*num_algorithms[1]);
pami_metadata_t *must_query_md = (pami_metadata_t*) alloca(sizeof(pami_metadata_t)*num_algorithms[1]);
status = PAMI_Geometry_algorithms_query(world_geometry, algorithms[i], always_works_alg,
always_works_md, num_algorithms[0], must_query_alg, must_query_md, num_algorithms[1]);
if (status != PAMI_SUCCESS)
error("Unable to query the supported algorithm %s for world", algorithmNames[i]);
// print out
printf("Collective: %s\n", algorithmNames[i]);
printf("Always supported: %i algorithms\n", num_algorithms[0]);
for (int j=0; j<num_algorithms[0]; j++)
printf("\t%s = %i\n", always_works_md[j].name, j);
printf("Locally supported: %i algorithms\n", num_algorithms[1]);
for (int j=0; j<num_algorithms[1]; j++)
printf("\t%s = %i\n", must_query_md[j].name, num_algorithms[0]+j);
printf("\n");
}
}
#if !defined(__bgq__)
PAMI_Extension_close (hfi_extension);
#endif
if ((status = PAMI_Context_destroyv(&context, 1)) != PAMI_SUCCESS)
fprintf(stderr, "Error closing PAMI context: %i\n", status);
if ((status = PAMI_Client_destroy(&client)) != PAMI_SUCCESS)
fprintf(stderr, "Error closing PAMI client: %i\n", status);
return 0;
}
int main(int argc, char ** argv)
{
pami_client_t client;
pami_context_t context;
pami_result_t status = PAMI_ERROR;
pami_configuration_t pami_config;
pami_geometry_t world_geo;
size_t barrier_alg_num[2];
pami_algorithm_t* bar_always_works_algo = NULL;
pami_metadata_t* bar_always_works_md = NULL;
pami_algorithm_t* bar_must_query_algo = NULL;
pami_metadata_t* bar_must_query_md = NULL;
pami_xfer_t barrier;
int my_id;
volatile int is_fence_done = 0;
volatile int is_barrier_done = 0;
/* create PAMI client */
RC( PAMI_Client_create("TEST", &client, NULL, 0) );
DBG_FPRINTF((stderr,"Client created successfully at 0x%p\n",client));
/* create PAMI context */
RC( PAMI_Context_createv(client, NULL, 0, &context, 1) );
DBG_FPRINTF((stderr,"Context created successfully at 0x%p\n",context));
/* query my task id */
bzero(&pami_config, sizeof(pami_configuration_t));
pami_config.name = PAMI_CLIENT_TASK_ID;
RC( PAMI_Client_query(client, &pami_config, 1) );
my_id = pami_config.value.intval;
DBG_FPRINTF((stderr,"My task id is %d\n", my_id));
/* get the world geometry */
RC( PAMI_Geometry_world(client, &world_geo) );
DBG_FPRINTF((stderr,"World geometry is at 0x%p\n",world_geo));
/* query number of barrier algorithms */
RC( PAMI_Geometry_algorithms_num(world_geo, PAMI_XFER_BARRIER,
barrier_alg_num) );
DBG_FPRINTF((stderr,"%d-%d algorithms are available for barrier op\n",
barrier_alg_num[0], barrier_alg_num[1]));
if (barrier_alg_num[0] <= 0) {
fprintf (stderr, "Error. No (%lu) algorithm is available for barrier op\n",
barrier_alg_num[0]);
return 1;
}
/* query barrier algorithm list */
bar_always_works_algo =
(pami_algorithm_t*)malloc(sizeof(pami_algorithm_t)*barrier_alg_num[0]);
bar_always_works_md =
(pami_metadata_t*)malloc(sizeof(pami_metadata_t)*barrier_alg_num[0]);
bar_must_query_algo =
(pami_algorithm_t*)malloc(sizeof(pami_algorithm_t)*barrier_alg_num[1]);
bar_must_query_md =
(pami_metadata_t*)malloc(sizeof(pami_metadata_t)*barrier_alg_num[1]);
RC( PAMI_Geometry_algorithms_query(world_geo, PAMI_XFER_BARRIER,
bar_always_works_algo, bar_always_works_md, barrier_alg_num[0],
bar_must_query_algo, bar_must_query_md, barrier_alg_num[1]) );
DBG_FPRINTF((stderr,"Algorithm [%s] at 0x%p will be used for barrier op\n",
bar_always_works_md[0].name, bar_always_works_algo[0]));
/* begin PAMI fence */
RC( PAMI_Fence_begin(context) );
DBG_FPRINTF((stderr,"PAMI fence begins\n"));
/* ------------------------------------------------------------------------ */
pami_extension_t extension;
const char ext_name[] = "EXT_hfi_extension";
const char sym_name[] = "hfi_remote_update";
hfi_remote_update_fn remote_update = NULL;
hfi_remote_update_info_t remote_info;
pami_memregion_t mem_region;
size_t mem_region_sz = 0;
unsigned long long operand = 1234;
unsigned long long orig_val = 0;
int offset = (operand)%MAX_TABLE_SZ;
/* initialize table for remote update operation */
int i;
for (i = 0; i < MAX_TABLE_SZ; i ++) {
table[i] = (unsigned long long) i;
}
orig_val = table[offset];
/* open PAMI extension */
RC( PAMI_Extension_open (client, ext_name, &extension) );
DBG_FPRINTF((stderr,"Open %s successfully.\n", ext_name));
/* load PAMI extension function */
remote_update = (hfi_remote_update_fn)
PAMI_Extension_symbol (extension, sym_name);
if (remote_update == (void *)NULL)
{
fprintf (stderr, "Error. Failed to load %s function in %s\n",
sym_name, ext_name);
return 1;
} else {
DBG_FPRINTF((stderr,"Loaded function %s in %s successfully.\n",
sym_name, ext_name));
}
/* create a memory region for remote update operation */
RC( PAMI_Memregion_create(context, table,
MAX_TABLE_SZ*sizeof(unsigned long long),
&mem_region_sz, &mem_region) );
DBG_FPRINTF((stderr,"%d-byte PAMI memory region created successfully.\n",
mem_region_sz));
/* perform a PAMI barrier */
is_barrier_done = 0;
barrier.cb_done = barrier_done;
barrier.cookie = (void*)&is_barrier_done;
barrier.algorithm = bar_always_works_algo[0];
RC( PAMI_Collective(context, &barrier) );
DBG_FPRINTF((stderr,"PAMI barrier op invoked successfully.\n"));
while (is_barrier_done == 0)
PAMI_Context_advance(context, 1000);
DBG_FPRINTF((stderr,"PAMI barrier op finished successfully.\n"));
RC( PAMI_Context_lock(context) );
/* prepare remote update info */
remote_info.dest = my_id^1;
remote_info.op = 0; /* op_add */
remote_info.atomic_operand = operand;
remote_info.dest_buf = (unsigned long long)(&(table[offset]));
/* invoke remote update PAMI extension function */
RC( remote_update(context, 1, &remote_info) );
DBG_FPRINTF((stderr,"Function %s invoked successfully.\n",
sym_name));
RC( PAMI_Context_unlock(context) );
/* perform a PAMI fence */
is_fence_done = 0;
RC( PAMI_Fence_all(context, fence_done, (void*)&is_fence_done) );
DBG_FPRINTF((stderr,"PAMI_Fence_all invoked successfully.\n"));
while (is_fence_done == 0)
PAMI_Context_advance(context, 1000);
DBG_FPRINTF((stderr,"PAMI_Fence_all finished successfully.\n"));
/* perform a PAMI barrier */
is_barrier_done = 0;
barrier.cb_done = barrier_done;
barrier.cookie = (void*)&is_barrier_done;
barrier.algorithm = bar_always_works_algo[0];
RC( PAMI_Collective(context, &barrier) );
DBG_FPRINTF((stderr,"PAMI barrier op invoked successfully.\n"));
while (is_barrier_done == 0)
PAMI_Context_advance(context, 1000);
DBG_FPRINTF((stderr,"PAMI barrier op finished successfully.\n"));
/* verify data after remote update operation */
if (table[offset] != orig_val + operand) {
printf("Data verification at offset %d with operand %lu failed: "
"[%lu expected with %lu updated]\n",
offset, operand, orig_val+operand, table[offset]);
} else {
printf("Data verification at offset %d with operand %lu passed: "
"[%lu expected with %lu updated].\n",
offset, operand, orig_val+operand, table[offset]);
}
/* destroy the memory region after remote update operation */
RC( PAMI_Memregion_destroy(context, &mem_region) );
DBG_FPRINTF((stderr,"PAMI memory region removed successfully.\n"));
/* close PAMI extension */
RC( PAMI_Extension_close (extension) );
DBG_FPRINTF((stderr,"Close %s successfully.\n", ext_name));
/* ------------------------------------------------------------------------ */
/* end PAMI fence */
RC( PAMI_Fence_end(context) );
DBG_FPRINTF((stderr,"PAMI fence ends\n"));
/* destroy PAMI context */
RC( PAMI_Context_destroyv(&context, 1) );
DBG_FPRINTF((stderr, "PAMI context destroyed successfully\n"));
/* destroy PAMI client */
RC( PAMI_Client_destroy(&client) );
DBG_FPRINTF((stderr, "PAMI client destroyed successfully\n"));
return 0;
}