int NBC_Ineighbor_alltoallw(void *sbuf, int *scounts, MPI_Datatype *stypes,
void *rbuf, int *rcounts, MPI_Datatype *rtypes, MPI_Comm comm, NBC_Handle* handle) {
int rank, size, res, worldsize, i;
MPI_Aint *sndexts, *rcvexts;
double t[10];
t[0] = PMPI_Wtime();
res = NBC_Init_handle(handle, comm);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
res = MPI_Comm_size(comm, &size);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Comm_size(MPI_COMM_WORLD, &worldsize);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
t[1] = PMPI_Wtime();
char inplace;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Ineighbor_alltoallw_args *args, *found, search;
#endif
NBC_IN_PLACE(sbuf, rbuf, inplace);
handle->tmpbuf=NULL;
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sbuf=sbuf;
search.scount=scount;
search.stype=stype;
search.rbuf=rbuf;
search.rcount=rcount;
search.rtype=rtype;
found = (NBC_Ineighbor_alltoallw_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_NEIGHBOR_ALLTOALLW], &search);
if(found == NULL) {
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create, res = %i\n", res); return res; }
{
int indegree, outdegree, weighted, *srcs, *dsts, i;
res = NBC_Comm_neighbors_count(comm, &indegree, &outdegree, &weighted);
if(res != NBC_OK) return res;
srcs = (int*)malloc(sizeof(int)*indegree);
dsts = (int*)malloc(sizeof(int)*outdegree);
sndexts = (MPI_Aint*)malloc(sizeof(MPI_Aint)*outdegree);
for(i=0; i<outdegree; ++i) {
res = MPI_Type_extent(stypes[i], &sndexts[i]);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
}
rcvexts = (MPI_Aint*)malloc(sizeof(MPI_Aint)*indegree);
for(i=0; i<indegree; ++i) {
res = MPI_Type_extent(rtypes[i], &rcvexts[i]);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
}
res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
if(res != NBC_OK) return res;
if(inplace) { /* we need an extra buffer to be deadlock-free */
int sumrbytes=0;
int offset=0;
for(i=0; i<indegree; ++i) sumrbytes += rcounts[i]*rcvexts[i];
handle->tmpbuf = malloc(sumrbytes);
for(i = 0; i < indegree; i++) {
res = NBC_Sched_recv((char*)0+offset, true, rcounts[i], rtypes[i], srcs[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
offset += rcounts[i]*rcvexts[i];
}
offset=0;
for(i = 0; i < outdegree; i++) {
res = NBC_Sched_send((char*)sbuf+offset, false, scounts[i], stypes[i], dsts[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
offset += scounts[i]*sndexts[i];
}
/* unpack from buffer */
offset=0;
for(i = 0; i < indegree; i++) {
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
res = NBC_Sched_copy((char*)0+offset, true, rcounts[i], rtypes[i], (char*)rbuf+offset, false, rcounts[i], rtypes[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
offset += rcounts[i]*rcvexts[i];
}
} else { /* non INPLACE case */
/* simply loop over neighbors and post send/recv operations */
int offset=0;
for(i = 0; i < indegree; i++) {
res = NBC_Sched_recv((char*)rbuf+offset, false, rcounts[i], rtypes[i], srcs[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
offset += rcounts[i]*rcvexts[i];
}
offset = 0;
for(i = 0; i < outdegree; i++) {
res = NBC_Sched_send((char*)sbuf+offset, false, scounts[i], stypes[i], dsts[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
offset += scounts[i]*sndexts[i];
}
}
}
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Ineighbor_alltoallw_args*)malloc(sizeof(NBC_Ineighbor_alltoallw_args));
args->sbuf=sbuf;
args->scount=scount;
args->stype=stype;
args->rbuf=rbuf;
args->rcount=rcount;
args->rtype=rtype;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_NEIGHBOR_ALLTOALLW], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for A2A */
if(++handle->comminfo->NBC_Dict_size[NBC_NEIGHBOR_ALLTOALLW] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_NEIGHBOR_ALLTOALLW], &handle->comminfo->NBC_Dict_size[NBC_NEIGHBOR_ALLTOALLW]);
}
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
int ompi_coll_libnbc_iallreduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_0_0_t *module)
{
int rank, p, res;
OPAL_PTRDIFF_TYPE ext, lb;
NBC_Schedule *schedule;
size_t size;
#ifdef NBC_CACHE_SCHEDULE
NBC_Allreduce_args *args, *found, search;
#endif
enum { NBC_ARED_BINOMIAL, NBC_ARED_RING } alg;
char inplace;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_get_extent(datatype, &lb, &ext);
if (OMPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
res = ompi_datatype_type_size (datatype, &size);
if (OMPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; }
handle->tmpbuf = malloc(ext*count);
if(handle->tmpbuf == NULL) { printf("Error in malloc() (%i)\n", res); return NBC_OOR; }
if((p == 1) && !inplace) {
/* for a single node - copy data to receivebuf */
res = NBC_Copy(sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
/* algorithm selection */
if(p < 4 || size*count < 65536 || inplace) {
alg = NBC_ARED_BINOMIAL;
} else {
alg = NBC_ARED_RING;
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf=sendbuf;
search.recvbuf=recvbuf;
search.count=count;
search.datatype=datatype;
search.op=op;
found = (NBC_Allreduce_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLREDUCE], &search);
if(found == NULL) {
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc()\n"); return res; }
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
switch(alg) {
case NBC_ARED_BINOMIAL:
res = allred_sched_diss(rank, p, count, datatype, sendbuf, recvbuf, op, schedule, handle);
break;
case NBC_ARED_RING:
res = allred_sched_ring(rank, p, count, datatype, sendbuf, recvbuf, op, size, ext, schedule, handle);
break;
}
if (NBC_OK != res) { printf("Error in Schedule creation() (%i)\n", res); return res; }
res = NBC_Sched_commit(schedule);
if(res != NBC_OK) { free(handle->tmpbuf); printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Allreduce_args*)malloc(sizeof(NBC_Allreduce_args));
args->sendbuf=sendbuf;
args->recvbuf=recvbuf;
args->count=count;
args->datatype=datatype;
args->op=op;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLREDUCE], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for A2A */
if(++handle->comminfo->NBC_Dict_size[NBC_ALLREDUCE] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLREDUCE], &handle->comminfo->NBC_Dict_size[NBC_ALLREDUCE]);
}
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if(res != NBC_OK) { free(handle->tmpbuf); printf("Error in NBC_Start() (%i)\n", res); return res; }
/* tmpbuf is freed with the handle */
return NBC_OK;
}
int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
int recvcount, MPI_Datatype recvtype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
MPI_Aint rcvext = 0;
NBC_Schedule *schedule;
char *rbuf, inplace = 0;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
rank = ompi_comm_rank (comm);
if (root == rank) {
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
}
p = ompi_comm_size (comm);
if (rank == root) {
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
}
if (inplace) {
sendcount = recvcount;
sendtype = recvtype;
} else if (rank == root) {
rbuf = ((char *)recvbuf) + (rank*recvcount*rcvext);
/* if I am the root - just copy the message (only without MPI_IN_PLACE) */
res = NBC_Copy(sendbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
#ifdef NBC_CACHE_SCHEDULE
NBC_Gather_args *args, *found, search;
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.sendcount = sendcount;
search.sendtype = sendtype;
search.recvbuf = recvbuf;
search.recvcount = recvcount;
search.recvtype = recvtype;
search.root = root;
found = (NBC_Gather_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER],
&search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* send to root */
if (rank != root) {
/* send msg to root */
res = NBC_Sched_send(sendbuf, false, sendcount, sendtype, root, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
} else {
for (int i = 0 ; i < p ; ++i) {
rbuf = (char *)recvbuf + i * recvcount * rcvext;
if (i != root) {
/* root receives message to the right buffer */
res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Gather_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->sendcount = sendcount;
args->sendtype = sendtype;
args->recvbuf = recvbuf;
args->recvcount = recvcount;
args->recvtype = recvtype;
args->root = root;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_GATHER] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER],
&libnbc_module->NBC_Dict_size[NBC_GATHER]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_igatherv(void* sendbuf, int sendcount, MPI_Datatype sendtype,
void* recvbuf, int *recvcounts, int *displs, MPI_Datatype recvtype,
int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
MPI_Aint rcvext = 0;
NBC_Schedule *schedule;
char *rbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
if (rank == root) {
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in MPI_Type_extent() (%i)", res);
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* send to root */
if (rank != root) {
/* send msg to root */
res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, root, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
} else {
for (int i = 0 ; i < p ; ++i) {
rbuf = (char *) recvbuf + displs[i] * rcvext;
if (i == root) {
if (!inplace) {
/* if I am the root - just copy the message */
res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcounts[i], recvtype,
comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
} else {
/* root receives message to the right buffer */
res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* the non-blocking reduce */
int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res, segsize;
size_t size;
MPI_Aint ext;
NBC_Schedule *schedule;
char *redbuf=NULL, inplace;
enum { NBC_RED_BINOMIAL, NBC_RED_CHAIN } alg;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent(datatype, &ext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = ompi_datatype_type_size(datatype, &size);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_size() (%i)", res);
return res;
}
/* only one node -> copy data */
if ((p == 1) && !inplace) {
res = NBC_Copy (sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
*request = &ompi_request_empty;
return OMPI_SUCCESS;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
/* algorithm selection */
if (p > 4 || size * count < 65536) {
alg = NBC_RED_BINOMIAL;
if(rank == root) {
/* root reduces in receivebuffer */
handle->tmpbuf = malloc (ext * count);
} else {
/* recvbuf may not be valid on non-root nodes */
handle->tmpbuf = malloc (ext * count * 2);
redbuf = (char*) handle->tmpbuf + ext * count;
}
} else {
handle->tmpbuf = malloc (ext * count);
alg = NBC_RED_CHAIN;
segsize = 16384/2;
}
if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
#ifdef NBC_CACHE_SCHEDULE
NBC_Reduce_args *args, *found, search;
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
search.root = root;
found = (NBC_Reduce_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* make sure the schedule is released with the handle on error */
handle->schedule = schedule;
switch(alg) {
case NBC_RED_BINOMIAL:
res = red_sched_binomial(rank, p, root, sendbuf, recvbuf, count, datatype, op, redbuf, schedule, handle);
break;
case NBC_RED_CHAIN:
res = red_sched_chain(rank, p, root, sendbuf, recvbuf, count, datatype, op, ext, size, schedule, handle, segsize);
break;
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Reduce_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->root = root;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for Reduce */
if (++libnbc_module->NBC_Dict_size[NBC_REDUCE] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE],
&libnbc_module->NBC_Dict_size[NBC_REDUCE]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start(handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_iallreduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res;
OPAL_PTRDIFF_TYPE ext, lb;
NBC_Schedule *schedule;
size_t size;
#ifdef NBC_CACHE_SCHEDULE
NBC_Allreduce_args *args, *found, search;
#endif
enum { NBC_ARED_BINOMIAL, NBC_ARED_RING } alg;
char inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_get_extent(datatype, &lb, &ext);
if (OMPI_SUCCESS != res) {
NBC_Error ("MPI Error in MPI_Type_extent() (%i)", res);
return res;
}
res = ompi_datatype_type_size (datatype, &size);
if (OMPI_SUCCESS != res) {
NBC_Error ("MPI Error in MPI_Type_size() (%i)", res);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
handle->tmpbuf = malloc (ext * count);
if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if ((p == 1) && !inplace) {
/* for a single node - copy data to receivebuf */
res = NBC_Copy(sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
/* algorithm selection */
if(p < 4 || size*count < 65536 || inplace) {
alg = NBC_ARED_BINOMIAL;
} else {
alg = NBC_ARED_RING;
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Allreduce_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (NULL == schedule) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* ensure the schedule is released with the handle on error */
handle->schedule = schedule;
switch(alg) {
case NBC_ARED_BINOMIAL:
res = allred_sched_diss(rank, p, count, datatype, sendbuf, recvbuf, op, schedule, handle);
break;
case NBC_ARED_RING:
res = allred_sched_ring(rank, p, count, datatype, sendbuf, recvbuf, op, size, ext, schedule, handle);
break;
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Allreduce_args *) malloc (sizeof(args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_ALLREDUCE] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE],
&libnbc_module->NBC_Dict_size[NBC_ALLREDUCE]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
/* simple linear MPI_Iscatter */
int ompi_coll_libnbc_iscatter(void* sendbuf, int sendcount, MPI_Datatype sendtype,
void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_0_0_t *module) {
int rank, p, res, i;
MPI_Aint sndext = 0;
NBC_Schedule *schedule;
char *sbuf, inplace;
#ifdef NBC_CACHE_SCHEDULE
NBC_Scatter_args *args, *found, search;
#endif
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
if (rank == root) {
res = MPI_Type_extent(sendtype, &sndext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
}
handle->tmpbuf=NULL;
if((rank == root) && (!inplace)) {
sbuf = ((char *)sendbuf) + (rank*sendcount*sndext);
/* if I am the root - just copy the message (not for MPI_IN_PLACE) */
res = NBC_Copy(sbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf=sendbuf;
search.sendcount=sendcount;
search.sendtype=sendtype;
search.recvbuf=recvbuf;
search.recvcount=recvcount;
search.recvtype=recvtype;
search.root=root;
found = (NBC_Scatter_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_SCATTER], &search);
if(found == NULL) {
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc()\n"); return res; }
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
/* receive from root */
if(rank != root) {
/* recv msg from root */
res = NBC_Sched_recv(recvbuf, false, recvcount, recvtype, root, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
} else {
for(i=0;i<p;i++) {
sbuf = ((char *)sendbuf) + (i*sendcount*sndext);
if(i != root) {
/* root sends the right buffer to the right receiver */
res = NBC_Sched_send(sbuf, false, sendcount, sendtype, i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
}
}
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Scatter_args*)malloc(sizeof(NBC_Scatter_args));
args->sendbuf=sendbuf;
args->sendcount=sendcount;
args->sendtype=sendtype;
args->recvbuf=recvbuf;
args->recvcount=recvcount;
args->recvtype=recvtype;
args->root=root;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_SCATTER], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for A2A */
if(++handle->comminfo->NBC_Dict_size[NBC_SCATTER] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_SCATTER], &handle->comminfo->NBC_Dict_size[NBC_SCATTER]);
}
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
/* the non-blocking reduce */
int ompi_coll_libnbc_ireduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_0_0_t *module) {
int rank, p, res, segsize, size;
MPI_Aint ext;
NBC_Schedule *schedule;
char *redbuf=NULL, inplace;
#ifdef NBC_CACHE_SCHEDULE
NBC_Reduce_args *args, *found, search;
#endif
enum { NBC_RED_BINOMIAL, NBC_RED_CHAIN } alg;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Type_extent(datatype, &ext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
res = MPI_Type_size(datatype, &size);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; }
/* only one node -> copy data */
if((p == 1) && !inplace) {
res = NBC_Copy(sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
/* algorithm selection */
if(p > 4 || size*count < 65536) {
alg = NBC_RED_BINOMIAL;
if(rank == root) {
/* root reduces in receivebuffer */
handle->tmpbuf = malloc(ext*count);
} else {
/* recvbuf may not be valid on non-root nodes */
handle->tmpbuf = malloc(ext*count*2);
redbuf = ((char*)handle->tmpbuf)+(ext*count);
}
} else {
handle->tmpbuf = malloc(ext*count);
alg = NBC_RED_CHAIN;
segsize = 16384/2;
}
if (NULL == handle->tmpbuf) { printf("Error in malloc() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf=sendbuf;
search.recvbuf=recvbuf;
search.count=count;
search.datatype=datatype;
search.op=op;
search.root=root;
found = (NBC_Reduce_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_REDUCE], &search);
if(found == NULL) {
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; }
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
switch(alg) {
case NBC_RED_BINOMIAL:
res = red_sched_binomial(rank, p, root, sendbuf, recvbuf, count, datatype, op, redbuf, schedule, handle);
break;
case NBC_RED_CHAIN:
res = red_sched_chain(rank, p, root, sendbuf, recvbuf, count, datatype, op, ext, size, schedule, handle, segsize);
break;
}
if (NBC_OK != res) { printf("Error in Schedule creation() (%i)\n", res); return res; }
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Reduce_args*)malloc(sizeof(NBC_Alltoall_args));
args->sendbuf=sendbuf;
args->recvbuf=recvbuf;
args->count=count;
args->datatype=datatype;
args->op=op;
args->root=root;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_REDUCE], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for Reduce */
if(++handle->comminfo->NBC_Dict_size[NBC_REDUCE] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_REDUCE], &handle->comminfo->NBC_Dict_size[NBC_REDUCE]);
}
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Start() (%i)\n", res); return res; }
/* tmpbuf is freed with the handle */
return NBC_OK;
}
/* simple linear MPI_Iscatterv */
int ompi_coll_libnbc_iscatterv(void* sendbuf, int *sendcounts, int *displs, MPI_Datatype sendtype,
void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_0_0_t *module) {
int rank, p, res, i;
MPI_Aint sndext;
NBC_Schedule *schedule;
char *sbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Type_extent(sendtype, &sndext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc()\n"); return res; }
handle->tmpbuf=NULL;
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
/* receive from root */
if(rank != root) {
/* recv msg from root */
res = NBC_Sched_recv(recvbuf, false, recvcount, recvtype, root, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
} else {
for(i=0;i<p;i++) {
sbuf = ((char *)sendbuf) + (displs[i]*sndext);
if(i == root) {
if(!inplace) {
/* if I am the root - just copy the message */
res = NBC_Copy(sbuf, sendcounts[i], sendtype, recvbuf, recvcount, recvtype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
} else {
/* root sends the right buffer to the right receiver */
res = NBC_Sched_send(sbuf, false, sendcounts[i], sendtype, i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
}
}
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
/* simple linear MPI_Iallgatherv
* the algorithm uses p-1 rounds
* first round:
* each node sends to it's left node (rank+1)%p sendcount elements
* each node begins with it's right node (rank-11)%p and receives from it recvcounts[(rank+1)%p] elements
* second round:
* each node sends to node (rank+2)%p sendcount elements
* each node receives from node (rank-2)%p recvcounts[(rank+2)%p] elements */
int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res, speer, rpeer;
MPI_Aint rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (OPAL_UNLIKELY(MPI_SUCCESS != res)) {
NBC_Error ("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
if (inplace) {
sendtype = recvtype;
sendcount = recvcounts[rank];
} else {
/* copy my data to receive buffer */
rbuf = (char *) recvbuf + displs[rank] * rcvext;
res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcounts[rank], recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (NULL == schedule) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
sbuf = (char *) recvbuf + displs[rank] * rcvext;
/* do p-1 rounds */
for (int r = 1 ; r < p ; ++r) {
speer = (rank + r) % p;
rpeer = (rank - r + p) % p;
rbuf = (char *)recvbuf + displs[rpeer] * rcvext;
res = NBC_Sched_recv (rbuf, false, recvcounts[rpeer], recvtype, rpeer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_send (sbuf, false, sendcount, sendtype, speer, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* simple linear Alltoallw */
static int nbc_alltoallw_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_3_0_t *module, bool persistent)
{
int rank, p, res;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
ptrdiff_t span=0;
void *tmpbuf = NULL;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
/* copy data to receivbuffer */
if (inplace) {
ptrdiff_t lgap, lspan;
for (int i = 0; i < p; i++) {
lspan = opal_datatype_span(&recvtypes[i]->super, recvcounts[i], &lgap);
if (lspan > span) {
span = lspan;
}
}
if (OPAL_UNLIKELY(0 == span)) {
return nbc_get_noop_request(persistent, request);
}
tmpbuf = malloc(span);
if (OPAL_UNLIKELY(NULL == tmpbuf)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
sendcounts = recvcounts;
sdispls = rdispls;
sendtypes = recvtypes;
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
free(tmpbuf);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (!inplace && sendcounts[rank] != 0) {
rbuf = (char *) recvbuf + rdispls[rank];
sbuf = (char *) sendbuf + sdispls[rank];
res = NBC_Sched_copy(sbuf, false, sendcounts[rank], sendtypes[rank],
rbuf, false, recvcounts[rank], recvtypes[rank], schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
if (inplace) {
res = a2aw_sched_inplace(rank, p, schedule, recvbuf,
recvcounts, rdispls, recvtypes);
} else {
res = a2aw_sched_linear(rank, p, schedule,
sendbuf, sendcounts, sdispls, sendtypes,
recvbuf, recvcounts, rdispls, recvtypes);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, tmpbuf);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
return OMPI_SUCCESS;
}
/* linear iexscan
* working principle:
* 1. each node (but node 0) receives from left neigbor
* 2. performs op
* 3. all but rank p-1 do sends to it's right neigbor and exits
*
*/
int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_2_0_t *module) {
int rank, p, res;
ptrdiff_t gap, span;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Scan_args *args, *found, search;
#endif
char inplace;
void *tmpbuf = NULL;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
span = opal_datatype_span(&datatype->super, count, &gap);
if (0 < rank) {
tmpbuf = malloc(span);
if (NULL == tmpbuf) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (inplace) {
res = NBC_Copy(recvbuf, count, datatype, (char *)tmpbuf-gap, count, datatype, comm);
} else {
res = NBC_Copy(sendbuf, count, datatype, (char *)tmpbuf-gap, count, datatype, comm);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
free(tmpbuf);
return res;
}
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
free(tmpbuf);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (rank != 0) {
res = NBC_Sched_recv (recvbuf, false, count, datatype, rank-1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
if (rank < p - 1) {
/* we have to wait until we have the data */
res = NBC_Sched_barrier(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Sched_op (recvbuf, false, (void *)(-gap), true, count,
datatype, op, schedule, true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
/* send reduced data onward */
res = NBC_Sched_send ((void *)(-gap), true, count, datatype, rank + 1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
}
} else if (p > 1) {
if (inplace) {
res = NBC_Sched_send (recvbuf, false, count, datatype, 1, schedule, false);
} else {
res = NBC_Sched_send (sendbuf, false, count, datatype, 1, schedule, false);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Scan_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_EXSCAN] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN],
&libnbc_module->NBC_Dict_size[NBC_EXSCAN]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Schedule_request(schedule, comm, libnbc_module, request, tmpbuf);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
return OMPI_SUCCESS;
}
/* simple linear Alltoallw */
int ompi_coll_libnbc_ialltoallw(void* sendbuf, int *sendcounts, int *sdispls,
MPI_Datatype sendtypes[], void* recvbuf, int *recvcounts, int *rdispls,
MPI_Datatype recvtypes[], struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_0_0_t *module)
{
int rank, p, res, i;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res= MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; }
handle->tmpbuf=NULL;
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
/* copy data to receivbuffer */
if((sendcounts[rank] != 0) && !inplace) {
rbuf = ((char *) recvbuf) + rdispls[rank];
sbuf = ((char *) sendbuf) + sdispls[rank];
res = NBC_Copy(sbuf, sendcounts[rank], sendtypes[rank], rbuf, recvcounts[rank], recvtypes[rank], comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
for (i = 0; i < p; i++) {
if (i == rank) { continue; }
/* post all sends */
if(sendcounts[i] != 0) {
sbuf = ((char *) sendbuf) + sdispls[i];
res = NBC_Sched_send(sbuf, false, sendcounts[i], sendtypes[i], i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
/* post all receives */
if(recvcounts[i] != 0) {
rbuf = ((char *) recvbuf) + rdispls[i];
res = NBC_Sched_recv(rbuf, false, recvcounts[i], recvtypes[i], i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
}
/*NBC_PRINT_SCHED(*schedule);*/
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
static int nbc_allgather_init(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_3_0_t *module, bool persistent)
{
int rank, p, res;
MPI_Aint rcvext;
NBC_Schedule *schedule;
char *rbuf, inplace;
#ifdef NBC_CACHE_SCHEDULE
NBC_Allgather_args *args, *found, search;
#endif
enum { NBC_ALLGATHER_LINEAR, NBC_ALLGATHER_RDBL} alg;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
int is_commsize_pow2 = !(p & (p - 1));
if (libnbc_iallgather_algorithm == 0) {
alg = NBC_ALLGATHER_LINEAR;
} else {
/* user forced dynamic decision */
if (libnbc_iallgather_algorithm == 1) {
alg = NBC_ALLGATHER_LINEAR;
} else if (libnbc_iallgather_algorithm == 2 && is_commsize_pow2) {
alg = NBC_ALLGATHER_RDBL;
} else {
alg = NBC_ALLGATHER_LINEAR;
}
}
res = ompi_datatype_type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
return res;
}
if (inplace) {
sendtype = recvtype;
sendcount = recvcount;
} else if (!persistent) { /* for persistent, the copy must be scheduled */
/* copy my data to receive buffer */
rbuf = (char *) recvbuf + rank * recvcount * rcvext;
res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
if (1 == p && (!persistent || inplace)) {
return nbc_get_noop_request(persistent, request);
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.sendcount = sendcount;
search.sendtype = sendtype;
search.recvbuf = recvbuf;
search.recvcount = recvcount;
search.recvtype = recvtype;
found = (NBC_Allgather_args *) hb_tree_search ((hb_tree*)libnbc_module->NBC_Dict[NBC_ALLGATHER], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (persistent && !inplace) {
/* for nonblocking, data has been copied already */
/* copy my data to receive buffer (= send buffer of NBC_Sched_send) */
rbuf = (char *)recvbuf + rank * recvcount * rcvext;
res = NBC_Sched_copy((void *)sendbuf, false, sendcount, sendtype,
rbuf, false, recvcount, recvtype, schedule, true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
switch (alg) {
case NBC_ALLGATHER_LINEAR:
res = allgather_sched_linear(rank, p, schedule, sendbuf, sendcount, sendtype,
recvbuf, recvcount, recvtype);
break;
case NBC_ALLGATHER_RDBL:
res = allgather_sched_recursivedoubling(rank, p, schedule, sendbuf, sendcount,
sendtype, recvbuf, recvcount, recvtype);
break;
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Allgather_args *) malloc (sizeof (args));
args->sendbuf = sendbuf;
args->sendcount = sendcount;
args->sendtype = sendtype;
args->recvbuf = recvbuf;
args->recvcount = recvcount;
args->recvtype = recvtype;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLGATHER], args, args, 0);
if (res != 0) {
free (args);
} else {
OBJ_RETAIN(schedule);
}
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_ALLGATHER] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLGATHER], &libnbc_module->NBC_Dict_size[NBC_ALLGATHER]);
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
return OMPI_SUCCESS;
}
/* simple linear MPI_Iallgather
* the algorithm uses p-1 rounds
* each node sends the packet it received last round (or has in round 0) to it's right neighbor (modulo p)
* each node receives from it's left (modulo p) neighbor */
int ompi_coll_libnbc_iallgather(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res;
MPI_Aint rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
#ifdef NBC_CACHE_SCHEDULE
NBC_Allgather_args *args, *found, search;
#endif
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) {
return res;
}
if (inplace) {
sendtype = recvtype;
sendcount = recvcount;
} else {
/* copy my data to receive buffer */
rbuf = (char *) recvbuf + rank * recvcount * rcvext;
res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.sendcount = sendcount;
search.sendtype = sendtype;
search.recvbuf = recvbuf;
search.recvcount = recvcount;
search.recvtype = recvtype;
found = (NBC_Allgather_args *) hb_tree_search ((hb_tree*)libnbc_module->NBC_Dict[NBC_ALLGATHER], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
sbuf = (char *)recvbuf + rank * recvcount * rcvext;
/* do p-1 rounds */
for(int r = 0 ; r < p ; ++r) {
if(r != rank) {
/* recv from rank r */
rbuf = (char *)recvbuf + r * recvcount * rcvext;
res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
/* send to rank r - not from the sendbuf to optimize MPI_IN_PLACE */
res = NBC_Sched_send (sbuf, false, sendcount, sendtype, r, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Allgather_args *) malloc (sizeof (args));
args->sendbuf = sendbuf;
args->sendcount = sendcount;
args->sendtype = sendtype;
args->recvbuf = recvbuf;
args->recvcount = recvcount;
args->recvtype = recvtype;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLGATHER], args, args, 0);
if (res != 0) {
free (args);
} else {
OBJ_RETAIN(schedule);
}
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_ALLGATHER] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLGATHER], &libnbc_module->NBC_Dict_size[NBC_ALLGATHER]);
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OMPI_COLL_LIBNBC_REQUEST_RETURN(handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* simple linear MPI_Ialltoall the (simple) algorithm just sends to all nodes */
int ompi_coll_libnbc_ialltoall(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res, a2asize, sndsize, datasize;
NBC_Schedule *schedule;
MPI_Aint rcvext, sndext;
#ifdef NBC_CACHE_SCHEDULE
NBC_Alltoall_args *args, *found, search;
#endif
char *rbuf, *sbuf, inplace;
enum {NBC_A2A_LINEAR, NBC_A2A_PAIRWISE, NBC_A2A_DISS} alg;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Type_extent(sendtype, &sndext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
res = MPI_Type_size(sendtype, &sndsize);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; }
/* algorithm selection */
a2asize = sndsize*sendcount*p;
/* this number is optimized for TCP on odin.cs.indiana.edu */
if((p <= 8) && ((a2asize < 1<<17) || (sndsize*sendcount < 1<<12))) {
/* just send as fast as we can if we have less than 8 peers, if the
* total communicated size is smaller than 1<<17 *and* if we don't
* have eager messages (msgsize < 1<<13) */
alg = NBC_A2A_LINEAR;
} else if(a2asize < (1<<12)*p) {
/*alg = NBC_A2A_DISS;*/
alg = NBC_A2A_LINEAR;
} else
alg = NBC_A2A_LINEAR; /*NBC_A2A_PAIRWISE;*/
if(!inplace) {
/* copy my data to receive buffer */
rbuf = ((char *)recvbuf) + (rank*recvcount*rcvext);
sbuf = ((char *)sendbuf) + (rank*sendcount*sndext);
res = NBC_Copy(sbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
/* allocate temp buffer if we need one */
if(alg == NBC_A2A_DISS) {
/* only A2A_DISS needs buffers */
if(NBC_Type_intrinsic(sendtype)) {
datasize = sndext*sendcount;
} else {
res = MPI_Pack_size(sendcount, sendtype, comm, &datasize);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Pack_size() (%i)\n", res); return res; }
}
/* allocate temporary buffers */
if(p % 2 == 0) {
handle->tmpbuf=malloc(datasize*p*2);
} else {
/* we cannot divide p by two, so alloc more to be safe ... */
handle->tmpbuf=malloc(datasize*(p/2+1)*2*2);
}
/* phase 1 - rotate n data blocks upwards into the tmpbuffer */
#if OPAL_CUDA_SUPPORT
if(NBC_Type_intrinsic(sendtype) && !(opal_cuda_check_bufs((char *)sendbuf, (char *)recvbuf))) {
#else
if(NBC_Type_intrinsic(sendtype)) {
#endif /* OPAL_CUDA_SUPPORT */
/* contiguous - just copy (1st copy) */
memcpy(handle->tmpbuf, (char*)sendbuf+datasize*rank, datasize*(p-rank));
if(rank != 0) memcpy((char*)handle->tmpbuf+datasize*(p-rank), sendbuf, datasize*(rank));
} else {
int pos=0;
/* non-contiguous - pack */
res = MPI_Pack((char*)sendbuf+rank*sendcount*sndext, (p-rank)*sendcount, sendtype, handle->tmpbuf, (p-rank)*datasize, &pos, comm);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Pack() (%i)\n", res); return res; }
if(rank != 0) {
pos = 0;
MPI_Pack(sendbuf, rank*sendcount, sendtype, (char*)handle->tmpbuf+datasize*(p-rank), rank*datasize, &pos, comm);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Pack() (%i)\n", res); return res; }
}
}
} else {
handle->tmpbuf=NULL;
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf=sendbuf;
search.sendcount=sendcount;
search.sendtype=sendtype;
search.recvbuf=recvbuf;
search.recvcount=recvcount;
search.recvtype=recvtype;
found = (NBC_Alltoall_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLTOALL], &search);
if(found == NULL) {
#endif
/* not found - generate new schedule */
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc()\n"); return res; }
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
switch(alg) {
case NBC_A2A_LINEAR:
res = a2a_sched_linear(rank, p, sndext, rcvext, schedule, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm);
break;
case NBC_A2A_DISS:
res = a2a_sched_diss(rank, p, sndext, rcvext, schedule, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, handle);
break;
case NBC_A2A_PAIRWISE:
res = a2a_sched_pairwise(rank, p, sndext, rcvext, schedule, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm);
break;
}
if (NBC_OK != res) { return res; }
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Alltoall_args*)malloc(sizeof(NBC_Alltoall_args));
args->sendbuf=sendbuf;
args->sendcount=sendcount;
args->sendtype=sendtype;
args->recvbuf=recvbuf;
args->recvcount=recvcount;
args->recvtype=recvtype;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLTOALL], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for A2A */
if(++handle->comminfo->NBC_Dict_size[NBC_ALLTOALL] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLTOALL], &handle->comminfo->NBC_Dict_size[NBC_ALLTOALL]);
/*if(!rank) printf("[%i] removing %i elements - new size: %i \n", rank, SCHED_DICT_UPPER-SCHED_DICT_LOWER, handle->comminfo->NBC_Alltoall_size);*/
}
/*if(!rank) printf("[%i] added new schedule to tree - number %i\n", rank, handle->comminfo->NBC_Dict_size[NBC_ALLTOALL]);*/
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
int ompi_coll_libnbc_ialltoall_inter (void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, res, i, rsize;
MPI_Aint sndext, rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_remote_size (comm, &rsize);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_remote_size() (%i)\n", res); return res; }
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Type_extent(sendtype, &sndext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; }
handle->tmpbuf=NULL;
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
for (i = 0; i < rsize; i++) {
/* post all sends */
sbuf = ((char *) sendbuf) + (i * sendcount * sndext);
res = NBC_Sched_send(sbuf, false, sendcount, sendtype, i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
/* post all receives */
rbuf = ((char *) recvbuf) + (i * recvcount * rcvext);
res = NBC_Sched_recv(rbuf, false, recvcount, recvtype, i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
/*NBC_PRINT_SCHED(*schedule);*/
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
/* simple linear Alltoallv */
int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_2_0_t *module)
{
int rank, p, res;
MPI_Aint sndext, rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
ptrdiff_t gap, span;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
/* copy data to receivbuffer */
if (inplace) {
int count = 0;
for (int i = 0; i < p; i++) {
if (recvcounts[i] > count) {
count = recvcounts[i];
}
}
span = opal_datatype_span(&recvtype->super, count, &gap);
if (OPAL_UNLIKELY(0 == span)) {
*request = &ompi_request_empty;
NBC_Return_handle (handle);
return MPI_SUCCESS;
}
handle->tmpbuf = malloc(span);
if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
sendcounts = recvcounts;
sdispls = rdispls;
} else {
res = ompi_datatype_type_extent (sendtype, &sndext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
NBC_Return_handle (handle);
return res;
}
if (sendcounts[rank] != 0) {
rbuf = (char *) recvbuf + rdispls[rank] * rcvext;
sbuf = (char *) sendbuf + sdispls[rank] * sndext;
res = NBC_Copy (sbuf, sendcounts[rank], sendtype, rbuf, recvcounts[rank], recvtype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (inplace) {
res = a2av_sched_inplace(rank, p, schedule, recvbuf, recvcounts,
rdispls, rcvext, recvtype, gap);
} else {
res = a2av_sched_linear(rank, p, schedule,
sendbuf, sendcounts, sdispls, sndext, sendtype,
recvbuf, recvcounts, rdispls, rcvext, recvtype);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start(handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
OBJ_RELEASE(schedule);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}
/* linear iexscan
* working principle:
* 1. each node (but node 0) receives from left neigbor
* 2. performs op
* 3. all but rank p-1 do sends to it's right neigbor and exits
*
*/
int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
MPI_Aint ext;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Scan_args *args, *found, search;
#endif
char inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent(datatype, &ext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
res = NBC_Init_handle(comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
if (inplace && rank < p - 1) {
/* need more buffer space for the inplace case */
handle->tmpbuf = malloc(ext * count * 2);
} else {
handle->tmpbuf = malloc(ext * count);
}
if (handle->tmpbuf == NULL) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* make sure the schedule is released with the handle on error */
handle->schedule = schedule;
if (rank != 0) {
if (inplace && rank < p - 1) {
/* if sendbuf == recvbuf do not clobber the send buffer until it has been combined
* with the incoming data. */
res = NBC_Sched_recv ((void *) (ext * count), true, count, datatype, rank-1, schedule, false);
} else {
res = NBC_Sched_recv (recvbuf, false, count, datatype, rank-1, schedule, false);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
if (rank < p - 1) {
/* we have to wait until we have the data */
res = NBC_Sched_barrier(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
/* perform the reduce in my temporary buffer */
/* this cannot be done until handle->tmpbuf is unused :-( so barrier after */
if (inplace) {
res = NBC_Sched_op (0, true, sendbuf, false, (void *)(ext * count), true, count,
datatype, op, schedule, true);
} else {
res = NBC_Sched_op (0, true, sendbuf, false, recvbuf, false, count, datatype, op,
schedule, true);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
/* send reduced data onward */
res = NBC_Sched_send (0, true, count, datatype, rank + 1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
if (inplace) {
/* copy the received data into the receive buffer */
res = NBC_Sched_copy ((void *)(ext * count), true, count, datatype, recvbuf,
false, count, datatype, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
}
} else if (p > 1) {
res = NBC_Sched_send (sendbuf, false, count, datatype, 1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
res = NBC_Sched_commit(schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Scan_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_EXSCAN] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN],
&libnbc_module->NBC_Dict_size[NBC_EXSCAN]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
/* simple linear Alltoallv */
static int nbc_alltoallv_init(const void* sendbuf, const int *sendcounts, const int *sdispls,
MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_3_0_t *module, bool persistent)
{
int rank, p, res;
MPI_Aint sndext, rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
ptrdiff_t gap = 0, span;
void * tmpbuf = NULL;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
res = ompi_datatype_type_extent (recvtype, &rcvext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
/* copy data to receivbuffer */
if (inplace) {
int count = 0;
for (int i = 0; i < p; i++) {
if (recvcounts[i] > count) {
count = recvcounts[i];
}
}
span = opal_datatype_span(&recvtype->super, count, &gap);
if (OPAL_UNLIKELY(0 == span)) {
return nbc_get_noop_request(persistent, request);
}
tmpbuf = malloc(span);
if (OPAL_UNLIKELY(NULL == tmpbuf)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
sendcounts = recvcounts;
sdispls = rdispls;
} else {
res = ompi_datatype_type_extent (sendtype, &sndext);
if (MPI_SUCCESS != res) {
NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
free(tmpbuf);
return OMPI_ERR_OUT_OF_RESOURCE;
}
if (!inplace && sendcounts[rank] != 0) {
rbuf = (char *) recvbuf + rdispls[rank] * rcvext;
sbuf = (char *) sendbuf + sdispls[rank] * sndext;
res = NBC_Sched_copy (sbuf, false, sendcounts[rank], sendtype,
rbuf, false, recvcounts[rank], recvtype, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
if (inplace) {
res = a2av_sched_inplace(rank, p, schedule, recvbuf, recvcounts,
rdispls, rcvext, recvtype, gap);
} else {
res = a2av_sched_linear(rank, p, schedule,
sendbuf, sendcounts, sdispls, sndext, sendtype,
recvbuf, recvcounts, rdispls, rcvext, recvtype);
}
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
res = NBC_Schedule_request(schedule, comm, libnbc_module, persistent, request, tmpbuf);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
free(tmpbuf);
return res;
}
return OMPI_SUCCESS;
}
int ompi_coll_libnbc_ineighbor_allgatherv(void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
int *rcounts, int *displs, MPI_Datatype rtype,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, size, res, worldsize;
MPI_Aint sndext, rcvext;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
res = NBC_Init_handle(comm, coll_req, libnbc_module);
handle = *coll_req;
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
res = MPI_Comm_size(comm, &size);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Comm_size(MPI_COMM_WORLD, &worldsize);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Type_extent(stype, &sndext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
res = MPI_Type_extent(rtype, &rcvext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
char inplace;
NBC_Schedule *schedule;
#ifdef NBC_CACHE_SCHEDULE
NBC_Ineighbor_allgatherv_args *args, *found, search;
#endif
NBC_IN_PLACE(sbuf, rbuf, inplace);
handle->tmpbuf=NULL;
#ifdef NBC_CACHE_SCHEDULE
/* search schedule in communicator specific tree */
search.sbuf=sbuf;
search.scount=scount;
search.stype=stype;
search.rbuf=rbuf;
search.rcount=rcount;
search.rtype=rtype;
found = (NBC_Ineighbor_allgatherv_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_NEIGHBOR_ALLGATHERV], &search);
if(found == NULL) {
#endif
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create, res = %i\n", res); return res; }
{
int indegree, outdegree, weighted, *srcs, *dsts, i;
res = NBC_Comm_neighbors_count(comm, &indegree, &outdegree, &weighted);
if(res != NBC_OK) return res;
srcs = (int*)malloc(sizeof(int)*indegree);
dsts = (int*)malloc(sizeof(int)*outdegree);
res = NBC_Comm_neighbors(comm, indegree, srcs, MPI_UNWEIGHTED, outdegree, dsts, MPI_UNWEIGHTED);
if(res != NBC_OK) return res;
if(inplace) { /* we need an extra buffer to be deadlock-free */
int sumrcounts=0;
int offset=0;
for(i=0; i<indegree; ++i) sumrcounts += rcounts[i];
handle->tmpbuf = malloc(rcvext*sumrcounts);
for(i = 0; i < indegree; i++) {
if(srcs[i] != MPI_PROC_NULL) {
res = NBC_Sched_recv((char*)0+offset, true, rcounts[i], rtype, srcs[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
offset += rcounts[i]*rcvext;
}
for(i = 0; i < outdegree; i++) {
if(dsts[i] != MPI_PROC_NULL) {
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
}
/* unpack from buffer */
offset=0;
for(i = 0; i < indegree; i++) {
if(srcs[i] != MPI_PROC_NULL) {
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
res = NBC_Sched_copy((char*)0+offset, true, rcounts[i], rtype, (char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
}
offset += rcounts[i]*rcvext;
}
} else { /* non INPLACE case */
/* simply loop over neighbors and post send/recv operations */
for(i = 0; i < indegree; i++) {
if(srcs[i] != MPI_PROC_NULL) {
res = NBC_Sched_recv((char*)rbuf+displs[i]*rcvext, false, rcounts[i], rtype, srcs[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
}
for(i = 0; i < outdegree; i++) {
if(dsts[i] != MPI_PROC_NULL) {
res = NBC_Sched_send((char*)sbuf, false, scount, stype, dsts[i], schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
}
}
}
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Ineighbor_allgatherv_args*)malloc(sizeof(NBC_Ineighbor_allgatherv_args));
args->sbuf=sbuf;
args->scount=scount;
args->stype=stype;
args->rbuf=rbuf;
args->rcount=rcount;
args->rtype=rtype;
args->schedule=schedule;
res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_NEIGHBOR_ALLGATHERV], args, args, 0);
if(res != 0) printf("error in dict_insert() (%i)\n", res);
/* increase number of elements for A2A */
if(++handle->comminfo->NBC_Dict_size[NBC_NEIGHBOR_ALLGATHERV] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_NEIGHBOR_ALLGATHERV], &handle->comminfo->NBC_Dict_size[NBC_NEIGHBOR_ALLGATHERV]);
}
} else {
/* found schedule */
schedule=found->schedule;
}
#endif
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
/* simple linear MPI_Iallgatherv
* the algorithm uses p-1 rounds
* first round:
* each node sends to it's left node (rank+1)%p sendcount elements
* each node begins with it's right node (rank-11)%p and receives from it recvcounts[(rank+1)%p] elements
* second round:
* each node sends to node (rank+2)%p sendcount elements
* each node receives from node (rank-2)%p recvcounts[(rank+2)%p] elements */
int ompi_coll_libnbc_iallgatherv(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int *recvcounts, int *displs,
MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res, r, speer, rpeer;
MPI_Aint rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; }
handle->tmpbuf=NULL;
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create, (%i)\n", res); return res; }
if (inplace) {
sendtype = recvtype;
sendcount = recvcounts[rank];
} else {
/* copy my data to receive buffer */
rbuf = ((char *)recvbuf) + (displs[rank]*rcvext);
res = NBC_Copy(sendbuf, sendcount, sendtype, rbuf, recvcounts[rank], recvtype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
sbuf = ((char*) recvbuf) + (displs[rank]*rcvext);
/* do p-1 rounds */
for(r=1;r<p;r++) {
speer = (rank+r)%p;
rpeer = (rank-r+p)%p;
rbuf = ((char *)recvbuf) + (displs[rpeer]*rcvext);
res = NBC_Sched_recv(rbuf, false, recvcounts[rpeer], recvtype, rpeer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
res = NBC_Sched_send(sbuf, false, sendcount, sendtype, speer, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
int ompi_coll_libnbc_ireduce_scatter(void* sendbuf, void* recvbuf, int *recvcounts, MPI_Datatype datatype,
MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_0_0_t *module) {
int peer, rank, maxr, p, r, res, count, offset, firstred;
MPI_Aint ext;
char *redbuf, *sbuf, inplace;
NBC_Schedule *schedule;
NBC_Handle *handle;
ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(comm, coll_req, libnbc_module);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
handle = (*coll_req);
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res = MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
MPI_Type_extent(datatype, &ext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc()\n"); return NBC_OOR; }
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
maxr = (int)ceil((log(p)/LOG2));
count = 0;
for(r=0;r<p;r++) count += recvcounts[r];
handle->tmpbuf = malloc(ext*count*2);
if(handle->tmpbuf == NULL) { printf("Error in malloc()\n"); return NBC_OOR; }
redbuf = ((char*)handle->tmpbuf)+(ext*count);
/* copy data to redbuf if we only have a single node */
if((p==1) && !inplace) {
res = NBC_Copy(sendbuf, count, datatype, redbuf, count, datatype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
firstred = 1;
for(r=1; r<=maxr; r++) {
if((rank % (1<<r)) == 0) {
/* we have to receive this round */
peer = rank + (1<<(r-1));
if(peer<p) {
res = NBC_Sched_recv(0, true, count, datatype, peer, schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
/* we have to wait until we have the data */
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
if(firstred) {
/* take reduce data from the sendbuf in the first round -> save copy */
res = NBC_Sched_op(redbuf-(unsigned long)handle->tmpbuf, true, sendbuf, false, 0, true, count, datatype, op, schedule);
firstred = 0;
} else {
/* perform the reduce in my local buffer */
res = NBC_Sched_op(redbuf-(unsigned long)handle->tmpbuf, true, redbuf-(unsigned long)handle->tmpbuf, true, 0, true, count, datatype, op, schedule);
}
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_op() (%i)\n", res); return res; }
/* this cannot be done until handle->tmpbuf is unused :-( */
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
}
} else {
/* we have to send this round */
peer = rank - (1<<(r-1));
if(firstred) {
/* we have to send the senbuf */
res = NBC_Sched_send(sendbuf, false, count, datatype, peer, schedule);
} else {
/* we send an already reduced value from redbuf */
res = NBC_Sched_send(redbuf-(unsigned long)handle->tmpbuf, true, count, datatype, peer, schedule);
}
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
/* leave the game */
break;
}
}
res = NBC_Sched_barrier(schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; }
/* rank 0 is root and sends - all others receive */
if(rank != 0) {
res = NBC_Sched_recv(recvbuf, false, recvcounts[rank], datatype, 0, schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
if(rank == 0) {
offset = 0;
for(r=1;r<p;r++) {
offset += recvcounts[r-1];
sbuf = ((char *)redbuf) + (offset*ext);
/* root sends the right buffer to the right receiver */
res = NBC_Sched_send(sbuf-(unsigned long)handle->tmpbuf, true, recvcounts[r], datatype, r, schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
res = NBC_Sched_copy(redbuf-(unsigned long)handle->tmpbuf, true, recvcounts[0], datatype, recvbuf, false, recvcounts[0], datatype, schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_copy() (%i)\n", res); return res; }
}
/*NBC_PRINT_SCHED(*schedule);*/
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Start() (%i)\n", res); return res; }
/* tmpbuf is freed with the handle */
return NBC_OK;
}
/* simple linear Alltoallv */
int NBC_Ialltoallv(void* sendbuf, int *sendcounts, int *sdispls,
MPI_Datatype sendtype, void* recvbuf, int *recvcounts, int *rdispls,
MPI_Datatype recvtype, MPI_Comm comm, NBC_Handle* handle) {
int rank, p, res, i;
MPI_Aint sndext, rcvext;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
res = NBC_Init_handle(handle, comm);
if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; }
res = MPI_Comm_rank(comm, &rank);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; }
res= MPI_Comm_size(comm, &p);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; }
res = MPI_Type_extent(sendtype, &sndext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
res = MPI_Type_extent(recvtype, &rcvext);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; }
schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule));
if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; }
handle->tmpbuf=NULL;
res = NBC_Sched_create(schedule);
if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; }
/* copy data to receivbuffer */
if((sendcounts[rank] != 0) && !inplace) {
rbuf = ((char *) recvbuf) + (rdispls[rank] * rcvext);
sbuf = ((char *) sendbuf) + (sdispls[rank] * sndext);
res = NBC_Copy(sbuf, sendcounts[rank], sendtype, rbuf, recvcounts[rank], recvtype, comm);
if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; }
}
for (i = 0; i < p; i++) {
if (i == rank) { continue; }
/* post all sends */
if(sendcounts[i] != 0) {
sbuf = ((char *) sendbuf) + (sdispls[i] * sndext);
res = NBC_Sched_send(sbuf, false, sendcounts[i], sendtype, i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; }
}
/* post all receives */
if(recvcounts[i] != 0) {
rbuf = ((char *) recvbuf) + (rdispls[i] * rcvext);
res = NBC_Sched_recv(rbuf, false, recvcounts[i], recvtype, i, schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; }
}
}
/*NBC_PRINT_SCHED(*schedule);*/
res = NBC_Sched_commit(schedule);
if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; }
res = NBC_Start(handle, schedule);
if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; }
return NBC_OK;
}
/* linear iscan
* working principle:
* 1. each node (but node 0) receives from left neighbor
* 2. performs op
* 3. all but rank p-1 do sends to it's right neighbor and exits
*
*/
int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module) {
int rank, p, res;
ptrdiff_t gap, span;
NBC_Schedule *schedule;
char inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
if (!inplace) {
/* copy data to receivebuf */
res = NBC_Copy (sendbuf, count, datatype, recvbuf, count, datatype, comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
res = NBC_Init_handle(comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
#ifdef NBC_CACHE_SCHEDULE
NBC_Scan_args *args, *found, search;
/* search schedule in communicator specific tree */
search.sendbuf = sendbuf;
search.recvbuf = recvbuf;
search.count = count;
search.datatype = datatype;
search.op = op;
found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN], &search);
if (NULL == found) {
#endif
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* ensure the schedule is released with the handle */
handle->schedule = schedule;
if(rank != 0) {
span = opal_datatype_span(&datatype->super, count, &gap);
handle->tmpbuf = malloc (span);
if (NULL == handle->tmpbuf) {
NBC_Return_handle (handle);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* we have to wait until we have the data */
res = NBC_Sched_recv ((void *)(-gap), true, count, datatype, rank-1, schedule, true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
/* perform the reduce in my local buffer */
/* this cannot be done until handle->tmpbuf is unused :-( so barrier after the op */
res = NBC_Sched_op ((void *)(-gap), true, recvbuf, false, count, datatype, op, schedule,
true);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
if (rank != p-1) {
res = NBC_Sched_send (recvbuf, false, count, datatype, rank+1, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
#ifdef NBC_CACHE_SCHEDULE
/* save schedule to tree */
args = (NBC_Scan_args *) malloc (sizeof (args));
if (NULL != args) {
args->sendbuf = sendbuf;
args->recvbuf = recvbuf;
args->count = count;
args->datatype = datatype;
args->op = op;
args->schedule = schedule;
res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN], args, args, 0);
if (0 == res) {
OBJ_RETAIN(schedule);
/* increase number of elements for A2A */
if (++libnbc_module->NBC_Dict_size[NBC_SCAN] > NBC_SCHED_DICT_UPPER) {
NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN],
&libnbc_module->NBC_Dict_size[NBC_SCAN]);
}
} else {
NBC_Error("error in dict_insert() (%i)", res);
free (args);
}
}
} else {
/* found schedule */
schedule = found->schedule;
OBJ_RETAIN(schedule);
}
#endif
res = NBC_Start(handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
/* tmpbuf is freed with the handle */
return OMPI_SUCCESS;
}
/* simple linear Alltoallw */
int ompi_coll_libnbc_ialltoallw(void* sendbuf, int *sendcounts, int *sdispls,
MPI_Datatype sendtypes[], void* recvbuf, int *recvcounts, int *rdispls,
MPI_Datatype recvtypes[], struct ompi_communicator_t *comm, ompi_request_t ** request,
struct mca_coll_base_module_2_1_0_t *module)
{
int rank, p, res;
NBC_Schedule *schedule;
char *rbuf, *sbuf, inplace;
NBC_Handle *handle;
ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module;
NBC_IN_PLACE(sendbuf, recvbuf, inplace);
rank = ompi_comm_rank (comm);
p = ompi_comm_size (comm);
/* copy data to receivbuffer */
if ((sendcounts[rank] != 0) && !inplace) {
rbuf = (char *) recvbuf + rdispls[rank];
sbuf = (char *) sendbuf + sdispls[rank];
res = NBC_Copy(sbuf, sendcounts[rank], sendtypes[rank], rbuf, recvcounts[rank], recvtypes[rank], comm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
return res;
}
}
schedule = OBJ_NEW(NBC_Schedule);
if (OPAL_UNLIKELY(NULL == schedule)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (int i = 0; i < p; i++) {
if (i == rank) {
continue;
}
/* post all sends */
if (sendcounts[i] != 0) {
sbuf = (char *) sendbuf + sdispls[i];
res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtypes[i], i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
/* post all receives */
if (recvcounts[i] != 0) {
rbuf = (char *) recvbuf + rdispls[i];
res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtypes[i], i, schedule, false);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
}
}
res = NBC_Sched_commit (schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Init_handle (comm, &handle, libnbc_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
OBJ_RELEASE(schedule);
return res;
}
res = NBC_Start (handle, schedule);
if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
NBC_Return_handle (handle);
return res;
}
*request = (ompi_request_t *) handle;
return OMPI_SUCCESS;
}