/* * reduce_log_inter * * Function: - reduction using O(N) algorithm * Accepts: - same as MPI_Reduce() * Returns: - MPI_SUCCESS or error code */ int mca_coll_cuda_reduce(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module) { mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module; ptrdiff_t true_lb, true_extent, lb, extent; char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL; const char *sbuf2; size_t bufsize; int rc; ompi_datatype_get_extent(dtype, &lb, &extent); ompi_datatype_get_true_extent(dtype, &true_lb, &true_extent); bufsize = true_extent + (ptrdiff_t)(count - 1) * extent; if ((MPI_IN_PLACE != sbuf) && (opal_cuda_check_bufs((char *)sbuf, NULL))) { sbuf1 = (char*)malloc(bufsize); if (NULL == sbuf1) { return OMPI_ERR_OUT_OF_RESOURCE; } opal_cuda_memcpy_sync(sbuf1, sbuf, bufsize); sbuf2 = sbuf; /* save away original buffer */ sbuf = sbuf1 - lb; } if (opal_cuda_check_bufs(rbuf, NULL)) { rbuf1 = (char*)malloc(bufsize); if (NULL == rbuf1) { if (NULL != sbuf1) free(sbuf1); return OMPI_ERR_OUT_OF_RESOURCE; } opal_cuda_memcpy_sync(rbuf1, rbuf, bufsize); rbuf2 = rbuf; /* save away original buffer */ rbuf = rbuf1 - lb; } rc = s->c_coll.coll_reduce((void *) sbuf, rbuf, count, dtype, op, root, comm, s->c_coll.coll_reduce_module); if (NULL != sbuf1) { free(sbuf1); } if (NULL != rbuf1) { rbuf = rbuf2; opal_cuda_memcpy_sync(rbuf, rbuf1, bufsize); free(rbuf1); } return rc; }
/* * reduce_scatter_block * * Function: - reduce then scatter * Accepts: - same as MPI_Reduce_scatter_block() * Returns: - MPI_SUCCESS or error code * * Algorithm: * reduce and scatter (needs to be cleaned * up at some point) */ int mca_coll_cuda_reduce_scatter_block(const void *sbuf, void *rbuf, int rcount, struct ompi_datatype_t *dtype, struct ompi_op_t *op, struct ompi_communicator_t *comm, mca_coll_base_module_t *module) { mca_coll_cuda_module_t *s = (mca_coll_cuda_module_t*) module; ptrdiff_t gap; char *rbuf1 = NULL, *sbuf1 = NULL, *rbuf2 = NULL; const char *sbuf2; size_t sbufsize, rbufsize; int rc; rbufsize = opal_datatype_span(&dtype->super, rcount, &gap); sbufsize = rbufsize * ompi_comm_size(comm); if ((MPI_IN_PLACE != sbuf) && (opal_cuda_check_bufs((char *)sbuf, NULL))) { sbuf1 = (char*)malloc(sbufsize); if (NULL == sbuf1) { return OMPI_ERR_OUT_OF_RESOURCE; } opal_cuda_memcpy_sync(sbuf1, sbuf, sbufsize); sbuf2 = sbuf; /* save away original buffer */ sbuf = sbuf1 - gap; } if (opal_cuda_check_bufs(rbuf, NULL)) { rbuf1 = (char*)malloc(rbufsize); if (NULL == rbuf1) { if (NULL != sbuf1) free(sbuf1); return OMPI_ERR_OUT_OF_RESOURCE; } opal_cuda_memcpy_sync(rbuf1, rbuf, rbufsize); rbuf2 = rbuf; /* save away original buffer */ rbuf = rbuf1 - gap; } rc = s->c_coll.coll_reduce_scatter_block(sbuf, rbuf, rcount, dtype, op, comm, s->c_coll.coll_reduce_scatter_block_module); if (NULL != sbuf1) { free(sbuf1); } if (NULL != rbuf1) { rbuf = rbuf2; opal_cuda_memcpy_sync(rbuf, rbuf1, rbufsize); free(rbuf1); } return rc; }
int32_t opal_datatype_copy_content_same_ddt( const opal_datatype_t* datatype, int32_t count, char* destination_base, char* source_base ) { OPAL_PTRDIFF_TYPE extent; size_t iov_len_local; int32_t (*fct)( const opal_datatype_t*, int32_t, char*, char*); #if OPAL_CUDA_SUPPORT bool cuda_device_bufs = opal_cuda_check_bufs(destination_base, source_base); #endif DO_DEBUG( opal_output( 0, "opal_datatype_copy_content_same_ddt( %p, %d, dst %p, src %p )\n", (void*)datatype, count, destination_base, source_base ); );
/* simple linear MPI_Ialltoall the (simple) algorithm just sends to all nodes */ int ompi_coll_libnbc_ialltoall(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, p, res, a2asize, sndsize, datasize; NBC_Schedule *schedule; MPI_Aint rcvext, sndext; #ifdef NBC_CACHE_SCHEDULE NBC_Alltoall_args *args, *found, search; #endif char *rbuf, *sbuf, inplace; enum {NBC_A2A_LINEAR, NBC_A2A_PAIRWISE, NBC_A2A_DISS} alg; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_IN_PLACE(sendbuf, recvbuf, inplace); res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Comm_size(comm, &p); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; } res = MPI_Type_extent(sendtype, &sndext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } res = MPI_Type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } res = MPI_Type_size(sendtype, &sndsize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; } /* algorithm selection */ a2asize = sndsize*sendcount*p; /* this number is optimized for TCP on odin.cs.indiana.edu */ if((p <= 8) && ((a2asize < 1<<17) || (sndsize*sendcount < 1<<12))) { /* just send as fast as we can if we have less than 8 peers, if the * total communicated size is smaller than 1<<17 *and* if we don't * have eager messages (msgsize < 1<<13) */ alg = NBC_A2A_LINEAR; } else if(a2asize < (1<<12)*p) { /*alg = NBC_A2A_DISS;*/ alg = NBC_A2A_LINEAR; } else alg = NBC_A2A_LINEAR; /*NBC_A2A_PAIRWISE;*/ if(!inplace) { /* copy my data to receive buffer */ rbuf = ((char *)recvbuf) + (rank*recvcount*rcvext); sbuf = ((char *)sendbuf) + (rank*sendcount*sndext); res = NBC_Copy(sbuf, sendcount, sendtype, rbuf, recvcount, recvtype, comm); if (NBC_OK != res) { printf("Error in NBC_Copy() (%i)\n", res); return res; } } /* allocate temp buffer if we need one */ if(alg == NBC_A2A_DISS) { /* only A2A_DISS needs buffers */ if(NBC_Type_intrinsic(sendtype)) { datasize = sndext*sendcount; } else { res = MPI_Pack_size(sendcount, sendtype, comm, &datasize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Pack_size() (%i)\n", res); return res; } } /* allocate temporary buffers */ if(p % 2 == 0) { handle->tmpbuf=malloc(datasize*p*2); } else { /* we cannot divide p by two, so alloc more to be safe ... */ handle->tmpbuf=malloc(datasize*(p/2+1)*2*2); } /* phase 1 - rotate n data blocks upwards into the tmpbuffer */ #if OPAL_CUDA_SUPPORT if(NBC_Type_intrinsic(sendtype) && !(opal_cuda_check_bufs((char *)sendbuf, (char *)recvbuf))) { #else if(NBC_Type_intrinsic(sendtype)) { #endif /* OPAL_CUDA_SUPPORT */ /* contiguous - just copy (1st copy) */ memcpy(handle->tmpbuf, (char*)sendbuf+datasize*rank, datasize*(p-rank)); if(rank != 0) memcpy((char*)handle->tmpbuf+datasize*(p-rank), sendbuf, datasize*(rank)); } else { int pos=0; /* non-contiguous - pack */ res = MPI_Pack((char*)sendbuf+rank*sendcount*sndext, (p-rank)*sendcount, sendtype, handle->tmpbuf, (p-rank)*datasize, &pos, comm); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Pack() (%i)\n", res); return res; } if(rank != 0) { pos = 0; MPI_Pack(sendbuf, rank*sendcount, sendtype, (char*)handle->tmpbuf+datasize*(p-rank), rank*datasize, &pos, comm); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Pack() (%i)\n", res); return res; } } } } else { handle->tmpbuf=NULL; } #ifdef NBC_CACHE_SCHEDULE /* search schedule in communicator specific tree */ search.sendbuf=sendbuf; search.sendcount=sendcount; search.sendtype=sendtype; search.recvbuf=recvbuf; search.recvcount=recvcount; search.recvtype=recvtype; found = (NBC_Alltoall_args*)hb_tree_search((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLTOALL], &search); if(found == NULL) { #endif /* not found - generate new schedule */ schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc()\n"); return res; } res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; } switch(alg) { case NBC_A2A_LINEAR: res = a2a_sched_linear(rank, p, sndext, rcvext, schedule, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); break; case NBC_A2A_DISS: res = a2a_sched_diss(rank, p, sndext, rcvext, schedule, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, handle); break; case NBC_A2A_PAIRWISE: res = a2a_sched_pairwise(rank, p, sndext, rcvext, schedule, sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); break; } if (NBC_OK != res) { return res; } res = NBC_Sched_commit(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } #ifdef NBC_CACHE_SCHEDULE /* save schedule to tree */ args = (NBC_Alltoall_args*)malloc(sizeof(NBC_Alltoall_args)); args->sendbuf=sendbuf; args->sendcount=sendcount; args->sendtype=sendtype; args->recvbuf=recvbuf; args->recvcount=recvcount; args->recvtype=recvtype; args->schedule=schedule; res = hb_tree_insert ((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLTOALL], args, args, 0); if(res != 0) printf("error in dict_insert() (%i)\n", res); /* increase number of elements for A2A */ if(++handle->comminfo->NBC_Dict_size[NBC_ALLTOALL] > NBC_SCHED_DICT_UPPER) { NBC_SchedCache_dictwipe((hb_tree*)handle->comminfo->NBC_Dict[NBC_ALLTOALL], &handle->comminfo->NBC_Dict_size[NBC_ALLTOALL]); /*if(!rank) printf("[%i] removing %i elements - new size: %i \n", rank, SCHED_DICT_UPPER-SCHED_DICT_LOWER, handle->comminfo->NBC_Alltoall_size);*/ } /*if(!rank) printf("[%i] added new schedule to tree - number %i\n", rank, handle->comminfo->NBC_Dict_size[NBC_ALLTOALL]);*/ } else { /* found schedule */ schedule=found->schedule; } #endif res = NBC_Start(handle, schedule); if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; } return NBC_OK; } int ompi_coll_libnbc_ialltoall_inter (void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, res, i, rsize; MPI_Aint sndext, rcvext; NBC_Schedule *schedule; char *rbuf, *sbuf; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_remote_size (comm, &rsize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_remote_size() (%i)\n", res); return res; } res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Type_extent(sendtype, &sndext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } res = MPI_Type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; } handle->tmpbuf=NULL; res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; } for (i = 0; i < rsize; i++) { /* post all sends */ sbuf = ((char *) sendbuf) + (i * sendcount * sndext); res = NBC_Sched_send(sbuf, false, sendcount, sendtype, i, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } /* post all receives */ rbuf = ((char *) recvbuf) + (i * recvcount * rcvext); res = NBC_Sched_recv(rbuf, false, recvcount, recvtype, i, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } } /*NBC_PRINT_SCHED(*schedule);*/ res = NBC_Sched_commit(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; } return NBC_OK; }