int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, res, rsize; NBC_Schedule *schedule; MPI_Aint ext; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rank = ompi_comm_rank (comm); rsize = ompi_comm_remote_size (comm); res = ompi_datatype_type_extent (datatype, &ext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } res = NBC_Init_handle(comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } handle->tmpbuf = malloc (ext * count); if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } res = red_sched_linear (rank, rsize, root, sendbuf, recvbuf, count, datatype, op, schedule, handle); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } res = NBC_Sched_commit(schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } res = NBC_Start(handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } *request = (ompi_request_t *) handle; /* tmpbuf is freed with the handle */ return OMPI_SUCCESS; }
int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module) { int res = nbc_barrier_inter_init(comm, request, module, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); *request = &ompi_request_null.request; return res; } return OMPI_SUCCESS; }
int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module) { int res = nbc_scan_init(sendbuf, recvbuf, count, datatype, op, comm, request, module, false); if (OPAL_LIKELY(OMPI_SUCCESS != res)) { return res; } res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); *request = &ompi_request_null.request; return res; } return OMPI_SUCCESS; }
int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module) { int res = nbc_bcast_inter_init(buffer, count, datatype, root, comm, request, module, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); *request = &ompi_request_null.request; return res; } return OMPI_SUCCESS; }
int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module) { int res = nbc_allgather_inter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, module, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (*(ompi_coll_libnbc_request_t **)request); *request = &ompi_request_null.request; return res; } return OMPI_SUCCESS; }
int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls, struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls, struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_3_0_t *module) { int res = nbc_alltoallw_init(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, request, module, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); *request = &ompi_request_null.request; return res; } return OMPI_SUCCESS; }
/* simple linear MPI_Iallgatherv * the algorithm uses p-1 rounds * first round: * each node sends to it's left node (rank+1)%p sendcount elements * each node begins with it's right node (rank-11)%p and receives from it recvcounts[(rank+1)%p] elements * second round: * each node sends to node (rank+2)%p sendcount elements * each node receives from node (rank-2)%p recvcounts[(rank+2)%p] elements */ int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res, speer, rpeer; MPI_Aint rcvext; NBC_Schedule *schedule; char *rbuf, *sbuf, inplace; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); res = ompi_datatype_type_extent (recvtype, &rcvext); if (OPAL_UNLIKELY(MPI_SUCCESS != res)) { NBC_Error ("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } if (inplace) { sendtype = recvtype; sendcount = recvcounts[rank]; } else { /* copy my data to receive buffer */ rbuf = (char *) recvbuf + displs[rank] * rcvext; res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcounts[rank], recvtype, comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } } schedule = OBJ_NEW(NBC_Schedule); if (NULL == schedule) { return OMPI_ERR_OUT_OF_RESOURCE; } sbuf = (char *) recvbuf + displs[rank] * rcvext; /* do p-1 rounds */ for (int r = 1 ; r < p ; ++r) { speer = (rank + r) % p; rpeer = (rank - r + p) % p; rbuf = (char *)recvbuf + displs[rpeer] * rcvext; res = NBC_Sched_recv (rbuf, false, recvcounts[rpeer], recvtype, rpeer, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Sched_send (sbuf, false, sendcount, sendtype, speer, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
/* linear iexscan * working principle: * 1. each node (but node 0) receives from left neigbor * 2. performs op * 3. all but rank p-1 do sends to it's right neigbor and exits * */ int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res; MPI_Aint ext; NBC_Schedule *schedule; #ifdef NBC_CACHE_SCHEDULE NBC_Scan_args *args, *found, search; #endif char inplace; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); res = ompi_datatype_type_extent(datatype, &ext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } res = NBC_Init_handle(comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } if (inplace && rank < p - 1) { /* need more buffer space for the inplace case */ handle->tmpbuf = malloc(ext * count * 2); } else { handle->tmpbuf = malloc(ext * count); } if (handle->tmpbuf == NULL) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } #ifdef NBC_CACHE_SCHEDULE /* search schedule in communicator specific tree */ search.sendbuf = sendbuf; search.recvbuf = recvbuf; search.count = count; search.datatype = datatype; search.op = op; found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], &search); if (NULL == found) { #endif schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* make sure the schedule is released with the handle on error */ handle->schedule = schedule; if (rank != 0) { if (inplace && rank < p - 1) { /* if sendbuf == recvbuf do not clobber the send buffer until it has been combined * with the incoming data. */ res = NBC_Sched_recv ((void *) (ext * count), true, count, datatype, rank-1, schedule, false); } else { res = NBC_Sched_recv (recvbuf, false, count, datatype, rank-1, schedule, false); } if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } if (rank < p - 1) { /* we have to wait until we have the data */ res = NBC_Sched_barrier(schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } /* perform the reduce in my temporary buffer */ /* this cannot be done until handle->tmpbuf is unused :-( so barrier after */ if (inplace) { res = NBC_Sched_op (0, true, sendbuf, false, (void *)(ext * count), true, count, datatype, op, schedule, true); } else { res = NBC_Sched_op (0, true, sendbuf, false, recvbuf, false, count, datatype, op, schedule, true); } if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } /* send reduced data onward */ res = NBC_Sched_send (0, true, count, datatype, rank + 1, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } if (inplace) { /* copy the received data into the receive buffer */ res = NBC_Sched_copy ((void *)(ext * count), true, count, datatype, recvbuf, false, count, datatype, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } } } } else if (p > 1) { res = NBC_Sched_send (sendbuf, false, count, datatype, 1, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } } res = NBC_Sched_commit(schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Scan_args *) malloc (sizeof (args)); if (NULL != args) { args->sendbuf = sendbuf; args->recvbuf = recvbuf; args->count = count; args->datatype = datatype; args->op = op; args->schedule = schedule; res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], args, args, 0); if (0 == res) { OBJ_RETAIN(schedule); /* increase number of elements for A2A */ if (++libnbc_module->NBC_Dict_size[NBC_EXSCAN] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_EXSCAN], &libnbc_module->NBC_Dict_size[NBC_EXSCAN]); } } else { NBC_Error("error in dict_insert() (%i)", res); free (args); } } } else { /* found schedule */ schedule = found->schedule; OBJ_RETAIN(schedule); } #endif res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; /* tmpbuf is freed with the handle */ return OMPI_SUCCESS; }
int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res, segsize; size_t size; NBC_Schedule *schedule; #ifdef NBC_CACHE_SCHEDULE NBC_Bcast_args *args, *found, search; #endif enum { NBC_BCAST_LINEAR, NBC_BCAST_BINOMIAL, NBC_BCAST_CHAIN } alg; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); res = ompi_datatype_type_size(datatype, &size); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_size() (%i)", res); return res; } segsize = 16384; /* algorithm selection */ if (p <= 4) { alg = NBC_BCAST_LINEAR; } else if (size * count < 65536) { alg = NBC_BCAST_BINOMIAL; } else if (size * count < 524288) { alg = NBC_BCAST_CHAIN; segsize = 8192; } else { alg = NBC_BCAST_CHAIN; segsize = 32768; } #ifdef NBC_CACHE_SCHEDULE /* search schedule in communicator specific tree */ search.buffer = buffer; search.count = count; search.datatype = datatype; search.root = root; found = (NBC_Bcast_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST], &search); if (NULL == found) { #endif schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } switch(alg) { case NBC_BCAST_LINEAR: res = bcast_sched_linear(rank, p, root, schedule, buffer, count, datatype); break; case NBC_BCAST_BINOMIAL: res = bcast_sched_binomial(rank, p, root, schedule, buffer, count, datatype); break; case NBC_BCAST_CHAIN: res = bcast_sched_chain(rank, p, root, schedule, buffer, count, datatype, segsize, size); break; } if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Bcast_args *) malloc (sizeof (args)); if (NULL != args) { args->buffer = buffer; args->count = count; args->datatype = datatype; args->root = root; args->schedule = schedule; res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST], args, args, 0); if (0 == res) { OBJ_RETAIN (schedule); /* increase number of elements for A2A */ if (++libnbc_module->NBC_Dict_size[NBC_BCAST] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_BCAST], &libnbc_module->NBC_Dict_size[NBC_BCAST]); } } else { NBC_Error("error in dict_insert() (%i)", res); free (args); } } } else { /* found schedule */ schedule = found->schedule; OBJ_RETAIN(schedule); } #endif res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
int ompi_coll_libnbc_igatherv(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int *recvcounts, int *displs, MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res; MPI_Aint rcvext = 0; NBC_Schedule *schedule; char *rbuf, inplace; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); if (rank == root) { res = MPI_Type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in MPI_Type_extent() (%i)", res); return res; } } schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } /* send to root */ if (rank != root) { /* send msg to root */ res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, root, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } else { for (int i = 0 ; i < p ; ++i) { rbuf = (char *) recvbuf + displs[i] * rcvext; if (i == root) { if (!inplace) { /* if I am the root - just copy the message */ res = NBC_Copy (sendbuf, sendcount, sendtype, rbuf, recvcounts[i], recvtype, comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } else { /* root receives message to the right buffer */ res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
/* simple linear Alltoallw */ int ompi_coll_libnbc_ialltoallw(void* sendbuf, int *sendcounts, int *sdispls, MPI_Datatype sendtypes[], void* recvbuf, int *recvcounts, int *rdispls, MPI_Datatype recvtypes[], struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res; NBC_Schedule *schedule; char *rbuf, *sbuf, inplace; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); /* copy data to receivbuffer */ if ((sendcounts[rank] != 0) && !inplace) { rbuf = (char *) recvbuf + rdispls[rank]; sbuf = (char *) sendbuf + sdispls[rank]; res = NBC_Copy(sbuf, sendcounts[rank], sendtypes[rank], rbuf, recvcounts[rank], recvtypes[rank], comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } } schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } for (int i = 0; i < p; i++) { if (i == rank) { continue; } /* post all sends */ if (sendcounts[i] != 0) { sbuf = (char *) sendbuf + sdispls[i]; res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtypes[i], i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } /* post all receives */ if (recvcounts[i] != 0) { rbuf = (char *) recvbuf + rdispls[i]; res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtypes[i], i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res; MPI_Aint rcvext = 0; NBC_Schedule *schedule; char *rbuf, inplace = 0; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rank = ompi_comm_rank (comm); if (root == rank) { NBC_IN_PLACE(sendbuf, recvbuf, inplace); } p = ompi_comm_size (comm); if (rank == root) { res = ompi_datatype_type_extent (recvtype, &rcvext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } } if (inplace) { sendcount = recvcount; sendtype = recvtype; } else if (rank == root) { rbuf = ((char *)recvbuf) + (rank*recvcount*rcvext); /* if I am the root - just copy the message (only without MPI_IN_PLACE) */ res = NBC_Copy(sendbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } } #ifdef NBC_CACHE_SCHEDULE NBC_Gather_args *args, *found, search; /* search schedule in communicator specific tree */ search.sendbuf = sendbuf; search.sendcount = sendcount; search.sendtype = sendtype; search.recvbuf = recvbuf; search.recvcount = recvcount; search.recvtype = recvtype; search.root = root; found = (NBC_Gather_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER], &search); if (NULL == found) { #endif schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } /* send to root */ if (rank != root) { /* send msg to root */ res = NBC_Sched_send(sendbuf, false, sendcount, sendtype, root, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } else { for (int i = 0 ; i < p ; ++i) { rbuf = (char *)recvbuf + i * recvcount * rcvext; if (i != root) { /* root receives message to the right buffer */ res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Gather_args *) malloc (sizeof (args)); if (NULL != args) { args->sendbuf = sendbuf; args->sendcount = sendcount; args->sendtype = sendtype; args->recvbuf = recvbuf; args->recvcount = recvcount; args->recvtype = recvtype; args->root = root; args->schedule = schedule; res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER], args, args, 0); if (0 == res) { OBJ_RETAIN(schedule); /* increase number of elements for A2A */ if (++libnbc_module->NBC_Dict_size[NBC_GATHER] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_GATHER], &libnbc_module->NBC_Dict_size[NBC_GATHER]); } } else { NBC_Error("error in dict_insert() (%i)", res); free (args); } } } else { /* found schedule */ schedule = found->schedule; OBJ_RETAIN(schedule); } #endif res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
int ompi_coll_libnbc_iallreduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res; OPAL_PTRDIFF_TYPE ext, lb; NBC_Schedule *schedule; size_t size; #ifdef NBC_CACHE_SCHEDULE NBC_Allreduce_args *args, *found, search; #endif enum { NBC_ARED_BINOMIAL, NBC_ARED_RING } alg; char inplace; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); res = ompi_datatype_get_extent(datatype, &lb, &ext); if (OMPI_SUCCESS != res) { NBC_Error ("MPI Error in MPI_Type_extent() (%i)", res); return res; } res = ompi_datatype_type_size (datatype, &size); if (OMPI_SUCCESS != res) { NBC_Error ("MPI Error in MPI_Type_size() (%i)", res); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } handle->tmpbuf = malloc (ext * count); if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } if ((p == 1) && !inplace) { /* for a single node - copy data to receivebuf */ res = NBC_Copy(sendbuf, count, datatype, recvbuf, count, datatype, comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } } /* algorithm selection */ if(p < 4 || size*count < 65536 || inplace) { alg = NBC_ARED_BINOMIAL; } else { alg = NBC_ARED_RING; } #ifdef NBC_CACHE_SCHEDULE /* search schedule in communicator specific tree */ search.sendbuf = sendbuf; search.recvbuf = recvbuf; search.count = count; search.datatype = datatype; search.op = op; found = (NBC_Allreduce_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE], &search); if (NULL == found) { #endif schedule = OBJ_NEW(NBC_Schedule); if (NULL == schedule) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* ensure the schedule is released with the handle on error */ handle->schedule = schedule; switch(alg) { case NBC_ARED_BINOMIAL: res = allred_sched_diss(rank, p, count, datatype, sendbuf, recvbuf, op, schedule, handle); break; case NBC_ARED_RING: res = allred_sched_ring(rank, p, count, datatype, sendbuf, recvbuf, op, size, ext, schedule, handle); break; } if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } res = NBC_Sched_commit(schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Allreduce_args *) malloc (sizeof(args)); if (NULL != args) { args->sendbuf = sendbuf; args->recvbuf = recvbuf; args->count = count; args->datatype = datatype; args->op = op; args->schedule = schedule; res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE], args, args, 0); if (0 == res) { OBJ_RETAIN(schedule); /* increase number of elements for A2A */ if (++libnbc_module->NBC_Dict_size[NBC_ALLREDUCE] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_ALLREDUCE], &libnbc_module->NBC_Dict_size[NBC_ALLREDUCE]); } } else { NBC_Error("error in dict_insert() (%i)", res); free (args); } } } else { /* found schedule */ schedule = found->schedule; OBJ_RETAIN(schedule); } #endif res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; /* tmpbuf is freed with the handle */ return OMPI_SUCCESS; }
/* the non-blocking reduce */ int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res, segsize; size_t size; MPI_Aint ext; NBC_Schedule *schedule; char *redbuf=NULL, inplace; enum { NBC_RED_BINOMIAL, NBC_RED_CHAIN } alg; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); res = ompi_datatype_type_extent(datatype, &ext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } res = ompi_datatype_type_size(datatype, &size); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_size() (%i)", res); return res; } /* only one node -> copy data */ if ((p == 1) && !inplace) { res = NBC_Copy (sendbuf, count, datatype, recvbuf, count, datatype, comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } *request = &ompi_request_empty; return OMPI_SUCCESS; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } /* algorithm selection */ if (p > 4 || size * count < 65536) { alg = NBC_RED_BINOMIAL; if(rank == root) { /* root reduces in receivebuffer */ handle->tmpbuf = malloc (ext * count); } else { /* recvbuf may not be valid on non-root nodes */ handle->tmpbuf = malloc (ext * count * 2); redbuf = (char*) handle->tmpbuf + ext * count; } } else { handle->tmpbuf = malloc (ext * count); alg = NBC_RED_CHAIN; segsize = 16384/2; } if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } #ifdef NBC_CACHE_SCHEDULE NBC_Reduce_args *args, *found, search; /* search schedule in communicator specific tree */ search.sendbuf = sendbuf; search.recvbuf = recvbuf; search.count = count; search.datatype = datatype; search.op = op; search.root = root; found = (NBC_Reduce_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE], &search); if (NULL == found) { #endif schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* make sure the schedule is released with the handle on error */ handle->schedule = schedule; switch(alg) { case NBC_RED_BINOMIAL: res = red_sched_binomial(rank, p, root, sendbuf, recvbuf, count, datatype, op, redbuf, schedule, handle); break; case NBC_RED_CHAIN: res = red_sched_chain(rank, p, root, sendbuf, recvbuf, count, datatype, op, ext, size, schedule, handle, segsize); break; } if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } res = NBC_Sched_commit(schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Reduce_args *) malloc (sizeof (args)); if (NULL != args) { args->sendbuf = sendbuf; args->recvbuf = recvbuf; args->count = count; args->datatype = datatype; args->op = op; args->root = root; args->schedule = schedule; res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE], args, args, 0); if (0 == res) { OBJ_RETAIN(schedule); /* increase number of elements for Reduce */ if (++libnbc_module->NBC_Dict_size[NBC_REDUCE] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_REDUCE], &libnbc_module->NBC_Dict_size[NBC_REDUCE]); } } else { NBC_Error("error in dict_insert() (%i)", res); free (args); } } } else { /* found schedule */ schedule = found->schedule; OBJ_RETAIN(schedule); } #endif res = NBC_Start(handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; /* tmpbuf is freed with the handle */ return OMPI_SUCCESS; }
int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int res, rsize; MPI_Aint rcvext; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rsize = ompi_comm_remote_size (comm); res = ompi_datatype_type_extent(recvtype, &rcvext); if (OPAL_UNLIKELY(MPI_SUCCESS != res)) { NBC_Error ("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } schedule = OBJ_NEW(NBC_Schedule); if (NULL == schedule) { return OMPI_ERR_OUT_OF_RESOURCE; } /* do rsize rounds */ for (int r = 0 ; r < rsize ; ++r) { char *rbuf = (char *) recvbuf + displs[r] * rcvext; if (recvcounts[r]) { res = NBC_Sched_recv (rbuf, false, recvcounts[r], recvtype, r, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } if (sendcount) { for (int r = 0 ; r < rsize ; ++r) { res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, r, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
/* Dissemination implementation of MPI_Ibarrier */ int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, maxround, res, recvpeer, sendpeer; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); res = NBC_Init_handle(comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } #ifdef NBC_CACHE_SCHEDULE /* there only one argument set per communicator -> hang it directly at * the tree-position, NBC_Dict_size[...] is 0 for not initialized and * 1 for initialized. NBC_Dict[...] is a pointer to the schedule in * this case */ if (libnbc_module->NBC_Dict_size[NBC_BARRIER] == 0) { /* we did not init it yet */ #endif schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* ensure the schedule is released with the handle on error */ handle->schedule = schedule; maxround = (int)ceil((log((double)p)/LOG2)-1); for (int round = 0 ; round <= maxround ; ++round) { sendpeer = (rank + (1 << round)) % p; /* add p because modulo does not work with negative values */ recvpeer = ((rank - (1 << round)) + p) % p; /* send msg to sendpeer */ res = NBC_Sched_send (NULL, false, 0, MPI_BYTE, sendpeer, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* recv msg from recvpeer */ res = NBC_Sched_recv (NULL, false, 0, MPI_BYTE, recvpeer, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* end communication round */ if (round < maxround) { res = NBC_Sched_barrier (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } #ifdef NBC_CACHE_SCHEDULE /* add it */ libnbc_module->NBC_Dict[NBC_BARRIER] = (hb_tree *) schedule; libnbc_module->NBC_Dict_size[NBC_BARRIER] = 1; } else { /* we found it */ handle->schedule = schedule = (NBC_Schedule *) libnbc_module->NBC_Dict[NBC_BARRIER]; } OBJ_RETAIN(schedule); #endif res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, res, rsize; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rank = ompi_comm_rank (comm); rsize = ompi_comm_remote_size (comm); res = NBC_Init_handle(comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* ensure the schedule is released with the handle on error */ handle->schedule = schedule; if (0 == rank) { for (int peer = 1 ; peer < rsize ; ++peer) { res = NBC_Sched_recv (NULL, false, 0, MPI_BYTE, peer, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } } } /* synchronize with the remote root */ res = NBC_Sched_recv (NULL, false, 0, MPI_BYTE, 0, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } res = NBC_Sched_send (NULL, false, 0, MPI_BYTE, 0, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } if (0 == rank) { /* wait for the remote root */ res = NBC_Sched_barrier (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* inform remote peers that all local peers have entered the barrier */ for (int peer = 1; peer < rsize ; ++peer) { res = NBC_Sched_send (NULL, false, 0, MPI_BYTE, peer, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
int ompi_coll_libnbc_ineighbor_alltoallv(void *sbuf, int *scounts, int *sdispls, MPI_Datatype stype, void *rbuf, int *rcounts, int *rdispls, MPI_Datatype rtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int res, indegree, outdegree, *srcs, *dsts; MPI_Aint sndext, rcvext; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_Schedule *schedule; res = MPI_Type_extent (stype, &sndext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in MPI_Type_extent() (%i)", res); return res; } res = MPI_Type_extent (rtype, &rcvext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in MPI_Type_extent() (%i)", res); return res; } #ifdef NBC_CACHE_SCHEDULE NBC_Ineighbor_alltoallv_args *args, *found, search; /* search schedule in communicator specific tree */ search.sbuf = sbuf; search.scount = scount; search.stype = stype; search.rbuf = rbuf; search.rcount = rcount; search.rtype = rtype; found = (NBC_Ineighbor_alltoallv_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_NEIGHBOR_ALLTOALLV], &search); if (NULL == found) { #endif schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } res = NBC_Comm_neighbors (comm, &srcs, &indegree, &dsts, &outdegree); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } /* simply loop over neighbors and post send/recv operations */ for (int i = 0 ; i < indegree ; ++i) { if (srcs[i] != MPI_PROC_NULL) { res = NBC_Sched_recv ((char *) rbuf + rdispls[i] * rcvext, false, rcounts[i], rtype, srcs[i], schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { break; } } } free (srcs); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); free (dsts); return res; } for (int i = 0 ; i < outdegree ; ++i) { if (dsts[i] != MPI_PROC_NULL) { res = NBC_Sched_send ((char *) sbuf + sdispls[i] * sndext, false, scounts[i], stype, dsts[i], schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { break; } } } free (dsts); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Ineighbor_alltoallv_args *) malloc (sizeof (args)); if (NULL != args) { args->sbuf = sbuf; args->scount = scount; args->stype = stype; args->rbuf = rbuf; args->rcount = rcount; args->rtype = rtype; args->schedule = schedule; res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_NEIGHBOR_ALLTOALLV], args, args, 0); if (0 == res) { OBJ_RETAIN(schedule); /* increase number of elements for A2A */ if (++libnbc_module->NBC_Dict_size[NBC_NEIGHBOR_ALLTOALLV] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_NEIGHBOR_ALLTOALLV], &libnbc_module->NBC_Dict_size[NBC_NEIGHBOR_ALLTOALLV]); } } else { NBC_Error("error in dict_insert() (%i)", res); free (args); } } } else { /* found schedule */ schedule = found->schedule; OBJ_RETAIN(schedule); } #endif res = NBC_Init_handle(comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
/* simple linear Alltoallv */ int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint sndext, rcvext; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = ompi_datatype_type_extent(sendtype, &sndext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } res = ompi_datatype_type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } rsize = ompi_comm_remote_size (comm); schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } for (int i = 0; i < rsize; i++) { /* post all sends */ if (sendcounts[i] != 0) { char *sbuf = (char *) sendbuf + sdispls[i] * sndext; res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtype, i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } /* post all receives */ if (recvcounts[i] != 0) { char *rbuf = (char *) recvbuf + rdispls[i] * rcvext; res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtype, i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } res = NBC_Sched_commit(schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Init_handle(comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start(handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); OBJ_RELEASE(schedule); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
int ompi_coll_libnbc_igather_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int res, rsize; MPI_Aint rcvext = 0; NBC_Schedule *schedule; char *rbuf; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rsize = ompi_comm_remote_size (comm); if (root == MPI_ROOT) { res = ompi_datatype_type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } } schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } /* send to root */ if (root != MPI_ROOT && root != MPI_PROC_NULL) { /* send msg to root */ res = NBC_Sched_send (sendbuf, false, sendcount, sendtype, root, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } else if (MPI_ROOT == root) { for (int i = 0 ; i < rsize ; ++i) { rbuf = ((char *)recvbuf) + (i * recvcount * rcvext); /* root receives message to the right buffer */ res = NBC_Sched_recv (rbuf, false, recvcount, recvtype, i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
/* simple linear Alltoallv */ int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; MPI_Aint sndext, rcvext; NBC_Schedule *schedule; char *rbuf, *sbuf, inplace; ptrdiff_t gap, span; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); res = ompi_datatype_type_extent (recvtype, &rcvext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } /* copy data to receivbuffer */ if (inplace) { int count = 0; for (int i = 0; i < p; i++) { if (recvcounts[i] > count) { count = recvcounts[i]; } } span = opal_datatype_span(&recvtype->super, count, &gap); if (OPAL_UNLIKELY(0 == span)) { *request = &ompi_request_empty; NBC_Return_handle (handle); return MPI_SUCCESS; } handle->tmpbuf = malloc(span); if (OPAL_UNLIKELY(NULL == handle->tmpbuf)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } sendcounts = recvcounts; sdispls = rdispls; } else { res = ompi_datatype_type_extent (sendtype, &sndext); if (MPI_SUCCESS != res) { NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); NBC_Return_handle (handle); return res; } if (sendcounts[rank] != 0) { rbuf = (char *) recvbuf + rdispls[rank] * rcvext; sbuf = (char *) sendbuf + sdispls[rank] * sndext; res = NBC_Copy (sbuf, sendcounts[rank], sendtype, rbuf, recvcounts[rank], recvtype, comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } } } schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } if (inplace) { res = a2av_sched_inplace(rank, p, schedule, recvbuf, recvcounts, rdispls, rcvext, recvtype, gap); } else { res = a2av_sched_linear(rank, p, schedule, sendbuf, sendcounts, sdispls, sndext, sendtype, recvbuf, recvcounts, rdispls, rcvext, recvtype); } if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); OBJ_RELEASE(schedule); return res; } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); OBJ_RELEASE(schedule); return res; } res = NBC_Start(handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); OBJ_RELEASE(schedule); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }
/* linear iscan * working principle: * 1. each node (but node 0) receives from left neighbor * 2. performs op * 3. all but rank p-1 do sends to it's right neighbor and exits * */ int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res; ptrdiff_t gap, span; NBC_Schedule *schedule; char inplace; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); rank = ompi_comm_rank (comm); p = ompi_comm_size (comm); if (!inplace) { /* copy data to receivebuf */ res = NBC_Copy (sendbuf, count, datatype, recvbuf, count, datatype, comm); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } } res = NBC_Init_handle(comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } #ifdef NBC_CACHE_SCHEDULE NBC_Scan_args *args, *found, search; /* search schedule in communicator specific tree */ search.sendbuf = sendbuf; search.recvbuf = recvbuf; search.count = count; search.datatype = datatype; search.op = op; found = (NBC_Scan_args *) hb_tree_search ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN], &search); if (NULL == found) { #endif schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* ensure the schedule is released with the handle */ handle->schedule = schedule; if(rank != 0) { span = opal_datatype_span(&datatype->super, count, &gap); handle->tmpbuf = malloc (span); if (NULL == handle->tmpbuf) { NBC_Return_handle (handle); return OMPI_ERR_OUT_OF_RESOURCE; } /* we have to wait until we have the data */ res = NBC_Sched_recv ((void *)(-gap), true, count, datatype, rank-1, schedule, true); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } /* perform the reduce in my local buffer */ /* this cannot be done until handle->tmpbuf is unused :-( so barrier after the op */ res = NBC_Sched_op ((void *)(-gap), true, recvbuf, false, count, datatype, op, schedule, true); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } } if (rank != p-1) { res = NBC_Sched_send (recvbuf, false, count, datatype, rank+1, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Scan_args *) malloc (sizeof (args)); if (NULL != args) { args->sendbuf = sendbuf; args->recvbuf = recvbuf; args->count = count; args->datatype = datatype; args->op = op; args->schedule = schedule; res = hb_tree_insert ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN], args, args, 0); if (0 == res) { OBJ_RETAIN(schedule); /* increase number of elements for A2A */ if (++libnbc_module->NBC_Dict_size[NBC_SCAN] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe ((hb_tree *) libnbc_module->NBC_Dict[NBC_SCAN], &libnbc_module->NBC_Dict_size[NBC_SCAN]); } } else { NBC_Error("error in dict_insert() (%i)", res); free (args); } } } else { /* found schedule */ schedule = found->schedule; OBJ_RETAIN(schedule); } #endif res = NBC_Start(handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; /* tmpbuf is freed with the handle */ return OMPI_SUCCESS; }
/* simple linear Alltoallw */ int ompi_coll_libnbc_ialltoallw_inter (void* sendbuf, int *sendcounts, int *sdispls, MPI_Datatype sendtypes[], void* recvbuf, int *recvcounts, int *rdispls, MPI_Datatype recvtypes[], struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int res, rsize; NBC_Schedule *schedule; char *rbuf, *sbuf; NBC_Handle *handle; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; rsize = ompi_comm_remote_size (comm); schedule = OBJ_NEW(NBC_Schedule); if (OPAL_UNLIKELY(NULL == schedule)) { return OMPI_ERR_OUT_OF_RESOURCE; } for (int i = 0 ; i < rsize ; ++i) { /* post all sends */ if (sendcounts[i] != 0) { sbuf = (char *) sendbuf + sdispls[i]; res = NBC_Sched_send (sbuf, false, sendcounts[i], sendtypes[i], i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } /* post all receives */ if (recvcounts[i] != 0) { rbuf = (char *) recvbuf + rdispls[i]; res = NBC_Sched_recv (rbuf, false, recvcounts[i], recvtypes[i], i, schedule, false); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } } } res = NBC_Sched_commit (schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Init_handle (comm, &handle, libnbc_module); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { OBJ_RELEASE(schedule); return res; } res = NBC_Start (handle, schedule); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { NBC_Return_handle (handle); return res; } *request = (ompi_request_t *) handle; return OMPI_SUCCESS; }