int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_2_0_t *module)
{
int rank, p, res, segsize;
size_t size;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Bcast_args *args, *found, search;
#endif
enum { NBC_BCAST_LINEAR, NBC_BCAST_BINOMIAL, NBC_BCAST_CHAIN } alg;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
if (1 == p) {
*request = &ompi_request_empty;
return OMPI_SUCCESS;
}
res = ompi_datatype_type_size(datatype, &size);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_size() (%i)", res);
return res;
}
segsize = 16384;
/* algorithm selection */
if( libnbc_ibcast_skip_dt_decision ) {
if (p <= 4) {
alg = NBC_BCAST_LINEAR;
}
else {
alg = NBC_BCAST_BINOMIAL;
}
}
else {
if (p <= 4) {
alg = NBC_BCAST_LINEAR;
} else if (size * count < 65536) {
alg = NBC_BCAST_BINOMIAL;
} else if (size * count < 524288) {
alg = NBC_BCAST_CHAIN;
segsize = 8192;
} else {
alg = NBC_BCAST_CHAIN;
segsize = 32768;
}
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.buffer = buffer;
search.count = count;
search.datatype = datatype;
search.root = root;
found = (NBC_Bcast_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
switch(alg) {
case NBC_BCAST_LINEAR:
res = bcast_sched_linear(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_BINOMIAL:
res = bcast_sched_binomial(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_CHAIN:
res = bcast_sched_chain(rank, p, root, schedule, buffer, count, datatype, segsize, size);
break;
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Bcast_args *) malloc (sizeof (args));
if (NULL != args) {
args->buffer = buffer;
args->count = count;
args->datatype = datatype;
args->root = root;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST], args, args, 0);
if (0 == res) {
OBJ_RETAIN (schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_BCAST] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST],
&libnbc_module->NBC_Dict_size[NBC_BCAST]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Schedule_request(schedule, comm, libnbc_module, request, NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res, size, segsize;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Bcast_args *args, *found, search;
#endif
enum { NBC_BCAST_LINEAR, NBC_BCAST_BINOMIAL, NBC_BCAST_CHAIN } alg;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Type_size(datatype, &size);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; }
segsize = 16384;
/* algorithm selection */
if(p <= 4) {
alg = NBC_BCAST_LINEAR;
} else if(size*count < 65536) {
alg = NBC_BCAST_BINOMIAL;
} else if(size*count < 524288) {
alg = NBC_BCAST_CHAIN;
segsize = 16384/2;
} else {
alg = NBC_BCAST_CHAIN;
segsize = 65536/2;
}
handle->tmpbuf=NULL;
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.buffer=buffer;
search.count=count;
search.datatype=datatype;
search.root=root;
found = (NBC_Bcast_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_BCAST], &search);
if(found == NULL) {
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create, res = %i\n", res); return res; }
switch(alg) {
case NBC_BCAST_LINEAR:
res = bcast_sched_linear(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_BINOMIAL:
res = bcast_sched_binomial(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_CHAIN:
res = bcast_sched_chain(rank, p, root, schedule, buffer, count, datatype, segsize, size);
break;
}
if (NBC_OK != res) { printf("Error in Schedule creation() (%i)\n", res); return res; }
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Bcast_args*)malloc(sizeof(NBC_Bcast_args));
args->buffer=buffer;
args->count=count;
args->datatype=datatype;
args->root=root;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_BCAST], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for A2A */
if(++handle->comminfo->NBC_Dict_size[NBC_BCAST] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_BCAST], &handle->comminfo->NBC_Dict_size[NBC_BCAST]);
}
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}