/* simple chained MPI_Ibcast */
static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, size_t size) {
int res, vrank, rpeer, speer, numfrag, fragcount, thiscount;
MPI_Aint ext;
char *buf;
RANK2VRANK(rank, vrank, root);
VRANK2RANK(rpeer, vrank-1, root);
VRANK2RANK(speer, vrank+1, root);
res = ompi_datatype_type_extent(datatype, &ext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
if (count == 0) {
return OMPI_SUCCESS;
}
numfrag = count * size/fragsize;
if ((count * size) % fragsize != 0) {
numfrag++;
}
fragcount = count/numfrag;
for (int fragnum = 0 ; fragnum < numfrag ; ++fragnum) {
buf = (char *) buffer + fragnum * fragcount * ext;
thiscount = fragcount;
if (fragnum == numfrag-1) {
/* last fragment may not be full */
thiscount = count - fragcount * fragnum;
}
/* root does not receive */
if (vrank != 0) {
res = NBC_Sched_recv (buf, false, thiscount, datatype, rpeer, schedule, true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
/* last rank does not send */
if (vrank != p-1) {
res = NBC_Sched_send (buf, false, thiscount, datatype, speer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
/* this barrier here seems awaward but isn't!!!! */
if (vrank == 0) {
res = NBC_Sched_barrier (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
}
}
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, res, rsize;
NBC_Schedule *schedule;
MPI_Aint ext;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
rank = ompi_comm_rank (comm);
rsize = ompi_comm_remote_size (comm);
res = ompi_datatype_type_extent (datatype, &ext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = NBC_Init_handle(comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
handle->tmpbuf = malloc (ext * count);
if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
res = red_sched_linear (rank, rsize, root, sendbuf, recvbuf, count, datatype, op, schedule, handle);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
res = NBC_Start(handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
static int nbc_allgather_inter_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_3_0_t *module, bool persistent)
{
int res, rsize;
MPI_Aint rcvext;
NBC_Schedule *schedule;
char *rbuf;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
res = ompi_datatype_type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error ("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
rsize = ompi_comm_remote_size (comm);
/* set up schedule */
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* do rsize - 1 rounds */
for (int r = 0 ; r < rsize ; ++r) {
/* recv from rank r */
rbuf = (char *) recvbuf + r * recvcount * rcvext;
res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
/* send to rank r */
res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
return OMPI_SUCCESS;
}
static int nbc_schedule_grow (NBC_Schedule *schedule, int additional) {
void *tmp;
int size;
/* get current size of schedule */
size = nbc_schedule_get_size (schedule);
tmp = realloc (schedule->data, size + additional);
if (NULL == tmp) {
NBC_Error ("Could not increase the size of NBC schedule");
return OMPI_ERR_OUT_OF_RESOURCE;
}
schedule->data = tmp;
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_igather_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
int recvcount, MPI_Datatype recvtype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int res, rsize;
MPI_Aint rcvext = 0;
NBC_Schedule *schedule;
char *rbuf;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
rsize = ompi_comm_remote_size (comm);
if (root == MPI_ROOT) {
res = ompi_datatype_type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* send to root */
if (root != MPI_ROOT && root != MPI_PROC_NULL) {
/* send msg to root */
res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, root, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
} else if (MPI_ROOT == root) {
for (int i = 0 ; i < rsize ; ++i) {
rbuf = ((char *)recvbuf) + (i * recvcount * rcvext);
/* root receives message to the right buffer */
res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
int recvcount, MPI_Datatype recvtype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
MPI_Aint rcvext = 0;
NBC_Schedule *schedule;
char *rbuf, inplace = 0;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
rank = ompi_comm_rank (comm);
if (root == rank) {
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
}
p = ompi_comm_size (comm);
if (rank == root) {
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
}
if (inplace) {
sendcount = recvcount;
sendtype = recvtype;
} else if (rank == root) {
rbuf = ((char *)recvbuf) + (rank*recvcount*rcvext);
/* if I am the root - just copy the message (only without MPI_IN_PLACE) */
res = NBC_Copy(sendbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
#ifdef NBC_CACHE_SCHEDULE
NBC_Gather_args *args, *found, search;
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.sendcount = sendcount;
search.sendtype = sendtype;
search.recvbuf = recvbuf;
search.recvcount = recvcount;
search.recvtype = recvtype;
search.root = root;
found = (NBC_Gather_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER],
&search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* send to root */
if (rank != root) {
/* send msg to root */
res = NBC_Sched_send(sendbuf, false, sendcount, sendtype, root, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
} else {
for (int i = 0 ; i < p ; ++i) {
rbuf = (char *)recvbuf + i * recvcount * rcvext;
if (i != root) {
/* root receives message to the right buffer */
res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Gather_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->sendcount = sendcount;
args->sendtype = sendtype;
args->recvbuf = recvbuf;
args->recvcount = recvcount;
args->recvtype = recvtype;
args->root = root;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_GATHER] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER],
&libnbc_module->NBC_Dict_size[NBC_GATHER]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int res, rsize;
MPI_Aint rcvext;
NBC_Schedule *schedule;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
rsize = ompi_comm_remote_size (comm);
res = ompi_datatype_type_extent(recvtype, &rcvext);
if (OPAL_UNLIKELY(MPI_SUCCESS != res)) {
NBC_Error ("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
schedule = OBJ_NEW(NBC_Schedule);
if (NULL == schedule) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* do rsize rounds */
for (int r = 0 ; r < rsize ; ++r) {
char *rbuf = (char *) recvbuf + displs[r] * rcvext;
if (recvcounts[r]) {
res = NBC_Sched_recv (rbuf, false, recvcounts[r], recvtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
if (sendcount) {
for (int r = 0 ; r < rsize ; ++r) {
res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_2_0_t *module)
{
int rank, p, res, segsize;
size_t size;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Bcast_args *args, *found, search;
#endif
enum { NBC_BCAST_LINEAR, NBC_BCAST_BINOMIAL, NBC_BCAST_CHAIN } alg;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
if (1 == p) {
*request = &ompi_request_empty;
return OMPI_SUCCESS;
}
res = ompi_datatype_type_size(datatype, &size);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_size() (%i)", res);
return res;
}
segsize = 16384;
/* algorithm selection */
if( libnbc_ibcast_skip_dt_decision ) {
if (p <= 4) {
alg = NBC_BCAST_LINEAR;
}
else {
alg = NBC_BCAST_BINOMIAL;
}
}
else {
if (p <= 4) {
alg = NBC_BCAST_LINEAR;
} else if (size * count < 65536) {
alg = NBC_BCAST_BINOMIAL;
} else if (size * count < 524288) {
alg = NBC_BCAST_CHAIN;
segsize = 8192;
} else {
alg = NBC_BCAST_CHAIN;
segsize = 32768;
}
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.buffer = buffer;
search.count = count;
search.datatype = datatype;
search.root = root;
found = (NBC_Bcast_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
switch(alg) {
case NBC_BCAST_LINEAR:
res = bcast_sched_linear(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_BINOMIAL:
res = bcast_sched_binomial(rank, p, root, schedule, buffer, count, datatype);
break;
case NBC_BCAST_CHAIN:
res = bcast_sched_chain(rank, p, root, schedule, buffer, count, datatype, segsize, size);
break;
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Bcast_args *) malloc (sizeof (args));
if (NULL != args) {
args->buffer = buffer;
args->count = count;
args->datatype = datatype;
args->root = root;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST], args, args, 0);
if (0 == res) {
OBJ_RETAIN (schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_BCAST] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST],
&libnbc_module->NBC_Dict_size[NBC_BCAST]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Schedule_request(schedule, comm, libnbc_module, request, NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
return OMPI_SUCCESS;
}
/* the non-blocking reduce */
int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res, segsize;
size_t size;
MPI_Aint ext;
NBC_Schedule *schedule;
char *redbuf=NULL, inplace;
enum { NBC_RED_BINOMIAL, NBC_RED_CHAIN } alg;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent(datatype, &ext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = ompi_datatype_type_size(datatype, &size);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_size() (%i)", res);
return res;
}
/* only one node -> copy data */
if ((p == 1) && !inplace) {
res = NBC_Copy (sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
*request = &ompi_request_empty;
return OMPI_SUCCESS;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
/* algorithm selection */
if (p > 4 || size * count < 65536) {
alg = NBC_RED_BINOMIAL;
if(rank == root) {
/* root reduces in receivebuffer */
handle->tmpbuf = malloc (ext * count);
} else {
/* recvbuf may not be valid on non-root nodes */
handle->tmpbuf = malloc (ext * count * 2);
redbuf = (char*) handle->tmpbuf + ext * count;
}
} else {
handle->tmpbuf = malloc (ext * count);
alg = NBC_RED_CHAIN;
segsize = 16384/2;
}
if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
#ifdef NBC_CACHE_SCHEDULE
NBC_Reduce_args *args, *found, search;
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
search.root = root;
found = (NBC_Reduce_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* make sure the schedule is released with the handle on error */
handle->schedule = schedule;
switch(alg) {
case NBC_RED_BINOMIAL:
res = red_sched_binomial(rank, p, root, sendbuf, recvbuf, count, datatype, op, redbuf, schedule, handle);
break;
case NBC_RED_CHAIN:
res = red_sched_chain(rank, p, root, sendbuf, recvbuf, count, datatype, op, ext, size, schedule, handle, segsize);
break;
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Reduce_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->root = root;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for Reduce */
if (++libnbc_module->NBC_Dict_size[NBC_REDUCE] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE],
&libnbc_module->NBC_Dict_size[NBC_REDUCE]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start(handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_iallreduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res;
OPAL_PTRDIFF_TYPE ext, lb;
NBC_Schedule *schedule;
size_t size;
#ifdef NBC_CACHE_SCHEDULE
NBC_Allreduce_args *args, *found, search;
#endif
enum { NBC_ARED_BINOMIAL, NBC_ARED_RING } alg;
char inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_get_extent(datatype, &lb, &ext);
if (OMPI_SUCCESS != res) {
NBC_Error ("MPI Error in MPI_Type_extent() (%i)", res);
return res;
}
res = ompi_datatype_type_size (datatype, &size);
if (OMPI_SUCCESS != res) {
NBC_Error ("MPI Error in MPI_Type_size() (%i)", res);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
handle->tmpbuf = malloc (ext * count);
if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if ((p == 1) && !inplace) {
/* for a single node - copy data to receivebuf */
res = NBC_Copy(sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
/* algorithm selection */
if(p < 4 || size*count < 65536 || inplace) {
alg = NBC_ARED_BINOMIAL;
} else {
alg = NBC_ARED_RING;
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Allreduce_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (NULL == schedule) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* ensure the schedule is released with the handle on error */
handle->schedule = schedule;
switch(alg) {
case NBC_ARED_BINOMIAL:
res = allred_sched_diss(rank, p, count, datatype, sendbuf, recvbuf, op, schedule, handle);
break;
case NBC_ARED_RING:
res = allred_sched_ring(rank, p, count, datatype, sendbuf, recvbuf, op, size, ext, schedule, handle);
break;
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Allreduce_args *) malloc (sizeof(args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_ALLREDUCE] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE],
&libnbc_module->NBC_Dict_size[NBC_ALLREDUCE]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
/* linear iexscan
* working principle:
* 1. each node (but node 0) receives from left neigbor
* 2. performs op
* 3. all but rank p-1 do sends to it's right neigbor and exits
*
*/
int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_2_0_t *module) {
int rank, p, res;
ptrdiff_t gap, span;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Scan_args *args, *found, search;
#endif
char inplace;
void *tmpbuf = NULL;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
span = opal_datatype_span(&datatype->super, count, &gap);
if (0 < rank) {
tmpbuf = malloc(span);
if (NULL == tmpbuf) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (inplace) {
res = NBC_Copy(recvbuf, count, datatype, (char *)tmpbuf-gap, count, datatype, comm);
} else {
res = NBC_Copy(sendbuf, count, datatype, (char *)tmpbuf-gap, count, datatype, comm);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
free(tmpbuf);
return res;
}
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
free(tmpbuf);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (rank != 0) {
res = NBC_Sched_recv (recvbuf, false, count, datatype, rank-1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
if (rank < p - 1) {
/* we have to wait until we have the data */
res = NBC_Sched_barrier(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Sched_op (recvbuf, false, (void *)(-gap), true, count,
datatype, op, schedule, true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
/* send reduced data onward */
res = NBC_Sched_send ((void *)(-gap), true, count, datatype, rank + 1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
}
} else if (p > 1) {
if (inplace) {
res = NBC_Sched_send (recvbuf, false, count, datatype, 1, schedule, false);
} else {
res = NBC_Sched_send (sendbuf, false, count, datatype, 1, schedule, false);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Scan_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_EXSCAN] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN],
&libnbc_module->NBC_Dict_size[NBC_EXSCAN]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Schedule_request(schedule, comm, libnbc_module, request, tmpbuf);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_iallgather_inter(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int res, rsize;
MPI_Aint rcvext;
NBC_Schedule *schedule;
char *rbuf;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error ("MPI Error in MPI_Type_extent() (%i)", res);
return res;
}
rsize = ompi_comm_remote_size (comm);
/* set up schedule */
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* do rsize - 1 rounds */
for (int r = 0 ; r < rsize ; ++r) {
/* recv from rank r */
rbuf = (char *) recvbuf + r * recvcount * rcvext;
res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
/* send to rank r */
res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OMPI_COLL_LIBNBC_REQUEST_RETURN(handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* progresses a request
*
* to be called *only* from the progress thread !!! */
int NBC_Progress(NBC_Handle *handle) {
int res, ret=NBC_CONTINUE;
bool flag;
unsigned long size = 0;
char *delim;
if (handle->nbc_complete) {
return NBC_OK;
}
flag = true;
if ((handle->req_count > 0) && (handle->req_array != NULL)) {
NBC_DEBUG(50, "NBC_Progress: testing for %i requests\n", handle->req_count);
#ifdef NBC_TIMING
Test_time -= MPI_Wtime();
#endif
/* don't call ompi_request_test_all as it causes a recursive call into opal_progress */
while (handle->req_count) {
ompi_request_t *subreq = handle->req_array[handle->req_count - 1];
if (REQUEST_COMPLETE(subreq)) {
if(OPAL_UNLIKELY( OMPI_SUCCESS != subreq->req_status.MPI_ERROR )) {
NBC_Error ("MPI Error in NBC subrequest %p : %d", subreq, subreq->req_status.MPI_ERROR);
/* copy the error code from the underlying request and let the
* round finish */
handle->super.req_status.MPI_ERROR = subreq->req_status.MPI_ERROR;
}
handle->req_count--;
ompi_request_free(&subreq);
} else {
flag = false;
break;
}
}
#ifdef NBC_TIMING
Test_time += MPI_Wtime();
#endif
}
/* a round is finished */
if (flag) {
/* reset handle for next round */
if (NULL != handle->req_array) {
/* free request array */
free (handle->req_array);
handle->req_array = NULL;
}
handle->req_count = 0;
/* previous round had an error */
if (OPAL_UNLIKELY(OMPI_SUCCESS != handle->super.req_status.MPI_ERROR)) {
res = handle->super.req_status.MPI_ERROR;
NBC_Error("NBC_Progress: an error %d was found during schedule %p at row-offset %li - aborting the schedule\n", res, handle->schedule, handle->row_offset);
handle->nbc_complete = true;
if (!handle->super.req_persistent) {
NBC_Free(handle);
}
return res;
}
/* adjust delim to start of current round */
NBC_DEBUG(5, "NBC_Progress: going in schedule %p to row-offset: %li\n", handle->schedule, handle->row_offset);
delim = handle->schedule->data + handle->row_offset;
NBC_DEBUG(10, "delim: %p\n", delim);
nbc_get_round_size(delim, &size);
NBC_DEBUG(10, "size: %li\n", size);
/* adjust delim to end of current round -> delimiter */
delim = delim + size;
if (*delim == 0) {
/* this was the last round - we're done */
NBC_DEBUG(5, "NBC_Progress last round finished - we're done\n");
handle->nbc_complete = true;
if (!handle->super.req_persistent) {
NBC_Free(handle);
}
return NBC_OK;
}
NBC_DEBUG(5, "NBC_Progress round finished - goto next round\n");
/* move delim to start of next round */
/* initializing handle for new virgin round */
handle->row_offset = (intptr_t) (delim + 1) - (intptr_t) handle->schedule->data;
/* kick it off */
res = NBC_Start_round(handle);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Error ("Error in NBC_Start_round() (%i)", res);
return res;
}
}
return ret;
}
static inline int NBC_Start_round(NBC_Handle *handle) {
int num; /* number of operations */
int res;
char* ptr;
MPI_Request *tmp;
NBC_Fn_type type;
NBC_Args_send sendargs;
NBC_Args_recv recvargs;
NBC_Args_op opargs;
NBC_Args_copy copyargs;
NBC_Args_unpack unpackargs;
void *buf1, *buf2;
/* get round-schedule address */
ptr = handle->schedule->data + handle->row_offset;
NBC_GET_BYTES(ptr,num);
NBC_DEBUG(10, "start_round round at offset %d : posting %i operations\n", handle->row_offset, num);
for (int i = 0 ; i < num ; ++i) {
int offset = (intptr_t)(ptr - handle->schedule->data);
memcpy (&type, ptr, sizeof (type));
switch(type) {
case SEND:
NBC_DEBUG(5," SEND (offset %li) ", offset);
NBC_GET_BYTES(ptr,sendargs);
NBC_DEBUG(5,"*buf: %p, count: %i, type: %p, dest: %i, tag: %i)\n", sendargs.buf,
sendargs.count, sendargs.datatype, sendargs.dest, handle->tag);
/* get an additional request */
handle->req_count++;
/* get buffer */
if(sendargs.tmpbuf) {
buf1=(char*)handle->tmpbuf+(long)sendargs.buf;
} else {
buf1=(void *)sendargs.buf;
}
#ifdef NBC_TIMING
Isend_time -= MPI_Wtime();
#endif
tmp = (MPI_Request *) realloc ((void *) handle->req_array, handle->req_count * sizeof (MPI_Request));
if (NULL == tmp) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
handle->req_array = tmp;
res = MCA_PML_CALL(isend(buf1, sendargs.count, sendargs.datatype, sendargs.dest, handle->tag,
MCA_PML_BASE_SEND_STANDARD, sendargs.local?handle->comm->c_local_comm:handle->comm,
handle->req_array+handle->req_count - 1));
if (OMPI_SUCCESS != res) {
NBC_Error ("Error in MPI_Isend(%lu, %i, %p, %i, %i, %lu) (%i)", (unsigned long)buf1, sendargs.count,
sendargs.datatype, sendargs.dest, handle->tag, (unsigned long)handle->comm, res);
return res;
}
#ifdef NBC_TIMING
Isend_time += MPI_Wtime();
#endif
break;
case RECV:
NBC_DEBUG(5, " RECV (offset %li) ", offset);
NBC_GET_BYTES(ptr,recvargs);
NBC_DEBUG(5, "*buf: %p, count: %i, type: %p, source: %i, tag: %i)\n", recvargs.buf, recvargs.count,
recvargs.datatype, recvargs.source, handle->tag);
/* get an additional request - TODO: req_count NOT thread safe */
handle->req_count++;
/* get buffer */
if(recvargs.tmpbuf) {
buf1=(char*)handle->tmpbuf+(long)recvargs.buf;
} else {
buf1=recvargs.buf;
}
#ifdef NBC_TIMING
Irecv_time -= MPI_Wtime();
#endif
tmp = (MPI_Request *) realloc ((void *) handle->req_array, handle->req_count * sizeof (MPI_Request));
if (NULL == tmp) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
handle->req_array = tmp;
res = MCA_PML_CALL(irecv(buf1, recvargs.count, recvargs.datatype, recvargs.source, handle->tag, recvargs.local?handle->comm->c_local_comm:handle->comm,
handle->req_array+handle->req_count-1));
if (OMPI_SUCCESS != res) {
NBC_Error("Error in MPI_Irecv(%lu, %i, %p, %i, %i, %lu) (%i)", (unsigned long)buf1, recvargs.count,
recvargs.datatype, recvargs.source, handle->tag, (unsigned long)handle->comm, res);
return res;
}
#ifdef NBC_TIMING
Irecv_time += MPI_Wtime();
#endif
break;
case OP:
NBC_DEBUG(5, " OP2 (offset %li) ", offset);
NBC_GET_BYTES(ptr,opargs);
NBC_DEBUG(5, "*buf1: %p, buf2: %p, count: %i, type: %p)\n", opargs.buf1, opargs.buf2,
opargs.count, opargs.datatype);
/* get buffers */
if(opargs.tmpbuf1) {
buf1=(char*)handle->tmpbuf+(long)opargs.buf1;
} else {
buf1=(void *)opargs.buf1;
}
if(opargs.tmpbuf2) {
buf2=(char*)handle->tmpbuf+(long)opargs.buf2;
} else {
buf2=opargs.buf2;
}
ompi_op_reduce(opargs.op, buf1, buf2, opargs.count, opargs.datatype);
break;
case COPY:
NBC_DEBUG(5, " COPY (offset %li) ", offset);
NBC_GET_BYTES(ptr,copyargs);
NBC_DEBUG(5, "*src: %lu, srccount: %i, srctype: %p, *tgt: %lu, tgtcount: %i, tgttype: %p)\n",
(unsigned long) copyargs.src, copyargs.srccount, copyargs.srctype,
(unsigned long) copyargs.tgt, copyargs.tgtcount, copyargs.tgttype);
/* get buffers */
if(copyargs.tmpsrc) {
buf1=(char*)handle->tmpbuf+(long)copyargs.src;
} else {
buf1=copyargs.src;
}
if(copyargs.tmptgt) {
buf2=(char*)handle->tmpbuf+(long)copyargs.tgt;
} else {
buf2=copyargs.tgt;
}
res = NBC_Copy (buf1, copyargs.srccount, copyargs.srctype, buf2, copyargs.tgtcount, copyargs.tgttype,
handle->comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
break;
case UNPACK:
NBC_DEBUG(5, " UNPACK (offset %li) ", offset);
NBC_GET_BYTES(ptr,unpackargs);
NBC_DEBUG(5, "*src: %lu, srccount: %i, srctype: %p, *tgt: %lu\n", (unsigned long) unpackargs.inbuf,
unpackargs.count, unpackargs.datatype, (unsigned long) unpackargs.outbuf);
/* get buffers */
if(unpackargs.tmpinbuf) {
buf1=(char*)handle->tmpbuf+(long)unpackargs.inbuf;
} else {
buf1=unpackargs.inbuf;
}
if(unpackargs.tmpoutbuf) {
buf2=(char*)handle->tmpbuf+(long)unpackargs.outbuf;
} else {
buf2=unpackargs.outbuf;
}
res = NBC_Unpack (buf1, unpackargs.count, unpackargs.datatype, buf2, handle->comm);
if (OMPI_SUCCESS != res) {
NBC_Error ("NBC_Unpack() failed (code: %i)", res);
return res;
}
break;
default:
NBC_Error ("NBC_Start_round: bad type %li at offset %li", (long)type, offset);
return OMPI_ERROR;
}
}
/* check if we can make progress - not in the first round, this allows us to leave the
* initialization faster and to reach more overlap
*
* threaded case: calling progress in the first round can lead to a
* deadlock if NBC_Free is called in this round :-( */
if (handle->row_offset) {
res = NBC_Progress(handle);
if ((NBC_OK != res) && (NBC_CONTINUE != res)) {
return OMPI_ERROR;
}
}
return OMPI_SUCCESS;
}
/* simple linear Alltoallv */
static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_3_0_t *module, bool persistent)
{
int rank, p, res;
MPI_Aint sndext, rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
ptrdiff_t gap = 0, span;
void * tmpbuf = NULL;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
/* copy data to receivbuffer */
if (inplace) {
int count = 0;
for (int i = 0; i < p; i++) {
if (recvcounts[i] > count) {
count = recvcounts[i];
}
}
span = opal_datatype_span(&recvtype->super, count, &gap);
if (OPAL_UNLIKELY(0 == span)) {
return nbc_get_noop_request(persistent, request);
}
tmpbuf = malloc(span);
if (OPAL_UNLIKELY(NULL == tmpbuf)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
sendcounts = recvcounts;
sdispls = rdispls;
} else {
res = ompi_datatype_type_extent (sendtype, &sndext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
free(tmpbuf);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (!inplace && sendcounts[rank] != 0) {
rbuf = (char *) recvbuf + rdispls[rank] * rcvext;
sbuf = (char *) sendbuf + sdispls[rank] * sndext;
res = NBC_Sched_copy (sbuf, false, sendcounts[rank], sendtype,
rbuf, false, recvcounts[rank], recvtype, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
if (inplace) {
res = a2av_sched_inplace(rank, p, schedule, recvbuf, recvcounts,
rdispls, rcvext, recvtype, gap);
} else {
res = a2av_sched_linear(rank, p, schedule,
sendbuf, sendcounts, sdispls, sndext, sendtype,
recvbuf, recvcounts, rdispls, rcvext, recvtype);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, tmpbuf);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
return OMPI_SUCCESS;
}
/* simple linear Alltoallv */
static int nbc_alltoallv_inter_init (const void* sendbuf, const int *sendcounts, const int *sdispls,
MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_3_0_t *module, bool persistent)
{
int res, rsize;
MPI_Aint sndext, rcvext;
NBC_Schedule *schedule;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
res = ompi_datatype_type_extent(sendtype, &sndext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = ompi_datatype_type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
rsize = ompi_comm_remote_size (comm);
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (int i = 0; i < rsize; i++) {
/* post all sends */
if (sendcounts[i] != 0) {
char *sbuf = (char *) sendbuf + sdispls[i] * sndext;
res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtype, i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
/* post all receives */
if (recvcounts[i] != 0) {
char *rbuf = (char *) recvbuf + rdispls[i] * rcvext;
res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
return OMPI_SUCCESS;
}
/* simple linear Alltoallv */
int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_2_0_t *module)
{
int rank, p, res;
MPI_Aint sndext, rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
ptrdiff_t gap, span;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
/* copy data to receivbuffer */
if (inplace) {
int count = 0;
for (int i = 0; i < p; i++) {
if (recvcounts[i] > count) {
count = recvcounts[i];
}
}
span = opal_datatype_span(&recvtype->super, count, &gap);
if (OPAL_UNLIKELY(0 == span)) {
*request = &ompi_request_empty;
NBC_Return_handle (handle);
return MPI_SUCCESS;
}
handle->tmpbuf = malloc(span);
if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
sendcounts = recvcounts;
sdispls = rdispls;
} else {
res = ompi_datatype_type_extent (sendtype, &sndext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
NBC_Return_handle (handle);
return res;
}
if (sendcounts[rank] != 0) {
rbuf = (char *) recvbuf + rdispls[rank] * rcvext;
sbuf = (char *) sendbuf + sdispls[rank] * sndext;
res = NBC_Copy (sbuf, sendcounts[rank], sendtype, rbuf, recvcounts[rank], recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (inplace) {
res = a2av_sched_inplace(rank, p, schedule, recvbuf, recvcounts,
rdispls, rcvext, recvtype, gap);
} else {
res = a2av_sched_linear(rank, p, schedule,
sendbuf, sendcounts, sdispls, sndext, sendtype,
recvbuf, recvcounts, rdispls, rcvext, recvtype);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start(handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
OBJ_RELEASE(schedule);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* linear iscan
* working principle:
* 1. each node (but node 0) receives from left neighbor
* 2. performs op
* 3. all but rank p-1 do sends to it's right neighbor and exits
*
*/
int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
ptrdiff_t gap, span;
NBC_Schedule *schedule;
char inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
if (!inplace) {
/* copy data to receivebuf */
res = NBC_Copy (sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
res = NBC_Init_handle(comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
#ifdef NBC_CACHE_SCHEDULE
NBC_Scan_args *args, *found, search;
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* ensure the schedule is released with the handle */
handle->schedule = schedule;
if(rank != 0) {
span = opal_datatype_span(&datatype->super, count, &gap);
handle->tmpbuf = malloc (span);
if (NULL == handle->tmpbuf) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* we have to wait until we have the data */
res = NBC_Sched_recv ((void *)(-gap), true, count, datatype, rank-1, schedule, true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
/* perform the reduce in my local buffer */
/* this cannot be done until handle->tmpbuf is unused :-( so barrier after the op */
res = NBC_Sched_op ((void *)(-gap), true, recvbuf, false, count, datatype, op, schedule,
true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
if (rank != p-1) {
res = NBC_Sched_send (recvbuf, false, count, datatype, rank+1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Scan_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_SCAN] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN],
&libnbc_module->NBC_Dict_size[NBC_SCAN]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start(handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
/* simple linear MPI_Iallgatherv
* the algorithm uses p-1 rounds
* first round:
* each node sends to it's left node (rank+1)%p sendcount elements
* each node begins with it's right node (rank-11)%p and receives from it recvcounts[(rank+1)%p] elements
* second round:
* each node sends to node (rank+2)%p sendcount elements
* each node receives from node (rank-2)%p recvcounts[(rank+2)%p] elements */
int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res, speer, rpeer;
MPI_Aint rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (OPAL_UNLIKELY(MPI_SUCCESS != res)) {
NBC_Error ("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
if (inplace) {
sendtype = recvtype;
sendcount = recvcounts[rank];
} else {
/* copy my data to receive buffer */
rbuf = (char *) recvbuf + displs[rank] * rcvext;
res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcounts[rank], recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (NULL == schedule) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
sbuf = (char *) recvbuf + displs[rank] * rcvext;
/* do p-1 rounds */
for (int r = 1 ; r < p ; ++r) {
speer = (rank + r) % p;
rpeer = (rank - r + p) % p;
rbuf = (char *)recvbuf + displs[rpeer] * rcvext;
res = NBC_Sched_recv (rbuf, false, recvcounts[rpeer], recvtype, rpeer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_send (sbuf, false, sendcount, sendtype, speer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* progresses a request
*
* to be called *only* from the progress thread !!! */
int NBC_Progress(NBC_Handle *handle) {
int flag, res, ret=NBC_CONTINUE;
unsigned long size;
char *delim;
/* the handle is done if there is no schedule attached */
if (NULL == handle->schedule) {
return NBC_OK;
}
if ((handle->req_count > 0) && (handle->req_array != NULL)) {
NBC_DEBUG(50, "NBC_Progress: testing for %i requests\n", handle->req_count);
#ifdef NBC_TIMING
Test_time -= MPI_Wtime();
#endif
res = ompi_request_test_all(handle->req_count, handle->req_array, &flag, MPI_STATUSES_IGNORE);
if(res != OMPI_SUCCESS) {
NBC_Error ("MPI Error in MPI_Testall() (%i)", res);
return res;
}
#ifdef NBC_TIMING
Test_time += MPI_Wtime();
#endif
} else {
flag = 1; /* we had no open requests -> proceed to next round */
}
/* a round is finished */
if (flag) {
/* adjust delim to start of current round */
NBC_DEBUG(5, "NBC_Progress: going in schedule %p to row-offset: %li\n", handle->schedule, handle->row_offset);
delim = handle->schedule->data + handle->row_offset;
NBC_DEBUG(10, "delim: %p\n", delim);
nbc_get_round_size(delim, &size);
NBC_DEBUG(10, "size: %li\n", size);
/* adjust delim to end of current round -> delimiter */
delim = delim + size;
if (NULL != handle->req_array) {
/* free request array */
free (handle->req_array);
handle->req_array = NULL;
}
handle->req_count = 0;
if (*delim == 0) {
/* this was the last round - we're done */
NBC_DEBUG(5, "NBC_Progress last round finished - we're done\n");
NBC_Free(handle);
return NBC_OK;
}
NBC_DEBUG(5, "NBC_Progress round finished - goto next round\n");
/* move delim to start of next round */
/* initializing handle for new virgin round */
handle->row_offset = (intptr_t) (delim + 1) - (intptr_t) handle->schedule->data;
/* kick it off */
res = NBC_Start_round(handle);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Error ("Error in NBC_Start_round() (%i)", res);
return res;
}
}
return ret;
}
int ompi_coll_libnbc_ineighbor_alltoallv(void *sbuf, int *scounts, int *sdispls, MPI_Datatype stype,
void *rbuf, int *rcounts, int *rdispls, MPI_Datatype rtype,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int res, indegree, outdegree, *srcs, *dsts;
MPI_Aint sndext, rcvext;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_Schedule *schedule;
res = MPI_Type_extent (stype, &sndext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in MPI_Type_extent() (%i)", res);
return res;
}
res = MPI_Type_extent (rtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in MPI_Type_extent() (%i)", res);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
NBC_Ineighbor_alltoallv_args *args, *found, search;
/* search schedule in communicator specific tree */
search.sbuf = sbuf;
search.scount = scount;
search.stype = stype;
search.rbuf = rbuf;
search.rcount = rcount;
search.rtype = rtype;
found = (NBC_Ineighbor_alltoallv_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_NEIGHBOR_ALLTOALLV],
&search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
res = NBC_Comm_neighbors (comm, &srcs, &indegree, &dsts, &outdegree);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
/* simply loop over neighbors and post send/recv operations */
for (int i = 0 ; i < indegree ; ++i) {
if (srcs[i] != MPI_PROC_NULL) {
res = NBC_Sched_recv ((char *) rbuf + rdispls[i] * rcvext, false, rcounts[i], rtype, srcs[i], schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
break;
}
}
}
free (srcs);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free (dsts);
return res;
}
for (int i = 0 ; i < outdegree ; ++i) {
if (dsts[i] != MPI_PROC_NULL) {
res = NBC_Sched_send ((char *) sbuf + sdispls[i] * sndext, false, scounts[i], stype, dsts[i], schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
break;
}
}
}
free (dsts);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Ineighbor_alltoallv_args *) malloc (sizeof (args));
if (NULL != args) {
args->sbuf = sbuf;
args->scount = scount;
args->stype = stype;
args->rbuf = rbuf;
args->rcount = rcount;
args->rtype = rtype;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_NEIGHBOR_ALLTOALLV], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_NEIGHBOR_ALLTOALLV] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_NEIGHBOR_ALLTOALLV],
&libnbc_module->NBC_Dict_size[NBC_NEIGHBOR_ALLTOALLV]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Init_handle(comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* linear iexscan
* working principle:
* 1. each node (but node 0) receives from left neigbor
* 2. performs op
* 3. all but rank p-1 do sends to it's right neigbor and exits
*
*/
int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
MPI_Aint ext;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Scan_args *args, *found, search;
#endif
char inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent(datatype, &ext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = NBC_Init_handle(comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
if (inplace && rank < p - 1) {
/* need more buffer space for the inplace case */
handle->tmpbuf = malloc(ext * count * 2);
} else {
handle->tmpbuf = malloc(ext * count);
}
if (handle->tmpbuf == NULL) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* make sure the schedule is released with the handle on error */
handle->schedule = schedule;
if (rank != 0) {
if (inplace && rank < p - 1) {
/* if sendbuf == recvbuf do not clobber the send buffer until it has been combined
* with the incoming data. */
res = NBC_Sched_recv ((void *) (ext * count), true, count, datatype, rank-1, schedule, false);
} else {
res = NBC_Sched_recv (recvbuf, false, count, datatype, rank-1, schedule, false);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
if (rank < p - 1) {
/* we have to wait until we have the data */
res = NBC_Sched_barrier(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
/* perform the reduce in my temporary buffer */
/* this cannot be done until handle->tmpbuf is unused :-( so barrier after */
if (inplace) {
res = NBC_Sched_op (0, true, sendbuf, false, (void *)(ext * count), true, count,
datatype, op, schedule, true);
} else {
res = NBC_Sched_op (0, true, sendbuf, false, recvbuf, false, count, datatype, op,
schedule, true);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
/* send reduced data onward */
res = NBC_Sched_send (0, true, count, datatype, rank + 1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
if (inplace) {
/* copy the received data into the receive buffer */
res = NBC_Sched_copy ((void *)(ext * count), true, count, datatype, recvbuf,
false, count, datatype, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
}
} else if (p > 1) {
res = NBC_Sched_send (sendbuf, false, count, datatype, 1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Scan_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_EXSCAN] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN],
&libnbc_module->NBC_Dict_size[NBC_EXSCAN]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_igatherv(void* sendbuf, int sendcount, MPI_Datatype sendtype,
void* recvbuf, int *recvcounts, int *displs, MPI_Datatype recvtype,
int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
MPI_Aint rcvext = 0;
NBC_Schedule *schedule;
char *rbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
if (rank == root) {
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in MPI_Type_extent() (%i)", res);
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* send to root */
if (rank != root) {
/* send msg to root */
res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, root, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
} else {
for (int i = 0 ; i < p ; ++i) {
rbuf = (char *) recvbuf + displs[i] * rcvext;
if (i == root) {
if (!inplace) {
/* if I am the root - just copy the message */
res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcounts[i], recvtype,
comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
} else {
/* root receives message to the right buffer */
res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}