int gasnete_coll_bcast_set_proto(gasnet_team_handle_t team,
gasnete_dcmf_bcast_kind_t kind)
{
gasnete_coll_team_dcmf_t *dcmf_tp;
gasneti_assert(team != NULL);
dcmf_tp = (gasnete_coll_team_dcmf_t *)team->dcmf_tp;
gasneti_assert(dcmf_tp != NULL);
if (kind < G_DCMF_BCAST_PROTO_NUM
&& DCMF_Geometry_analyze(&dcmf_tp->geometry, &gasnete_dcmf_bcast_proto[kind])
&& gasnete_dcmf_bcast_enabled[kind])
{
dcmf_tp->bcast_proto = &gasnete_dcmf_bcast_proto[kind];
return 0; /* SUCCESS */
}
return 1; /* FAILED */
}
int main()
{
int i, rank, nranks, msgsize, status, expected;
long bufsize;
int *src_buffer;
int *trg_buffer;
unsigned *ranks;
DCMF_Result dcmf_result;
DCMF_CollectiveProtocol_t barrier_protocol, lbarrier_protocol;
DCMF_CollectiveProtocol_t allreduce_protocol, allreduce_notree_protocol;
DCMF_Barrier_Configuration_t barrier_conf;
DCMF_Allreduce_Configuration_t allreduce_conf;
DCMF_CollectiveRequest_t crequest, crequest1, crequest2;
DCMF_Callback_t done_callback;
volatile unsigned allreduce_active = 0;
DCMF_Messager_initialize();
dcmf_result = DCMF_Collective_initialize();
assert(dcmf_result == DCMF_SUCCESS);
rank = DCMF_Messager_rank();
nranks = DCMF_Messager_size();
ranks = (unsigned *) malloc(nranks * sizeof(int));
for(i=0; i<nranks; i++) ranks[i] = i;
bufsize = MAX_MSG_SIZE;
src_buffer = (int *) malloc(bufsize);
trg_buffer = (int *) malloc(bufsize);
barrier_conf.protocol = DCMF_GI_BARRIER_PROTOCOL;
barrier_conf.cb_geometry = getGeometry;
dcmf_result = DCMF_Barrier_register(&barrier_protocol, &barrier_conf);
assert(dcmf_result == DCMF_SUCCESS);
barrier_conf.protocol = DCMF_LOCKBOX_BARRIER_PROTOCOL;
barrier_conf.cb_geometry = getGeometry;
dcmf_result = DCMF_Barrier_register(&lbarrier_protocol, &barrier_conf);
assert(dcmf_result == DCMF_SUCCESS);
DCMF_CollectiveProtocol_t *barrier_ptr, *lbarrier_ptr;
barrier_ptr = &barrier_protocol;
lbarrier_ptr = &lbarrier_protocol;
dcmf_result = DCMF_Geometry_initialize(&geometry,
0,
ranks,
nranks,
&barrier_ptr,
1,
&lbarrier_ptr,
1,
&crequest,
0,
1);
assert(dcmf_result == DCMF_SUCCESS);
allreduce_conf.protocol = DCMF_TREE_ALLREDUCE_PROTOCOL;
allreduce_conf.cb_geometry = getGeometry;
allreduce_conf.reuse_storage = 1;
dcmf_result = DCMF_Allreduce_register(&allreduce_protocol, &allreduce_conf);
assert(dcmf_result == DCMF_SUCCESS);
allreduce_conf.protocol = DCMF_TORUS_BINOMIAL_ALLREDUCE_PROTOCOL;
allreduce_conf.cb_geometry = getGeometry;
allreduce_conf.reuse_storage = 1;
dcmf_result = DCMF_Allreduce_register(&allreduce_notree_protocol, &allreduce_conf);
assert(dcmf_result == DCMF_SUCCESS);
status = DCMF_Geometry_analyze(&geometry, &allreduce_protocol);
assert(status == 1);
status = DCMF_Geometry_analyze(&geometry, &allreduce_notree_protocol);
assert(status == 1);
done_callback.function = done;
done_callback.clientdata = (void *) &allreduce_active;
if (rank == 0)
{
printf("DCMF_Allreduce Test\n");
fflush(stdout);
}
for (msgsize = sizeof(int); msgsize < MAX_MSG_SIZE; msgsize *= 2)
{
/*initializing buffer*/
for (i = 0; i < bufsize/sizeof(int); i++)
{
src_buffer[i] = rank;
trg_buffer[i] = 0;
}
allreduce_active += 1;
/*sum reduce operation*/
dcmf_result = DCMF_Allreduce(&allreduce_protocol,
&crequest1,
done_callback,
DCMF_SEQUENTIAL_CONSISTENCY,
&geometry,
(char *) src_buffer,
(char *) trg_buffer,
msgsize/sizeof(int),
DCMF_SIGNED_INT,
DCMF_SUM);
assert(dcmf_result == DCMF_SUCCESS);
while(allreduce_active > 0) DCMF_Messager_advance();
expected = (nranks-1)*(nranks)/2;
for (i = 0; i < msgsize/sizeof(int); i++)
{
if(trg_buffer[i] - expected != 0)
{
printf("[%d] Validation has failed Expected: %d, Actual: %d, i: %d \n",
rank, expected, trg_buffer[i], i);
fflush(stdout);
exit(-1);
}
}
printf("[%d] %d message sum allreduce successful \n", rank, msgsize);
fflush(stdout);
for (i = 0; i < bufsize/sizeof(int); i++)
{
src_buffer[i] = 1;
trg_buffer[i] = 0;
}
allreduce_active += 1;
/*sum reduce operation*/
dcmf_result = DCMF_Allreduce(&allreduce_notree_protocol,
&crequest2,
done_callback,
DCMF_SEQUENTIAL_CONSISTENCY,
&geometry,
(char *) src_buffer,
(char *) trg_buffer,
msgsize/sizeof(int),
DCMF_SIGNED_INT,
DCMF_PROD);
assert(dcmf_result == DCMF_SUCCESS);
while(allreduce_active > 0) DCMF_Messager_advance();
expected = 1;
for (i = 0; i < msgsize/sizeof(int); i++)
{
if(trg_buffer[i] - expected != 0)
{
printf("[%d] Validation has failed Expected: %d, Actual: %d, i: %d \n",
rank, expected, trg_buffer[i], i);
fflush(stdout);
exit(-1);
}
}
printf("[%d] %d message product allreduce successful\n", rank, msgsize);
fflush(stdout);
}
free(src_buffer);
free(trg_buffer);
DCMF_Messager_finalize();
return 0;
}
