/**
* \brief NCCL implementation of \ref gpucomm_free.
*/
static void comm_free(gpucomm *comm) {
ASSERT_COMM(comm);
cuda_enter(comm->ctx);
ncclCommDestroy(comm->c);
cuda_exit(comm->ctx);
comm_clear(comm);
}
NCCL<Dtype>::~NCCL() {
if (solver_->param().layer_wise_reduce()) {
CUDA_CHECK(cudaStreamDestroy(stream_));
}
if (comm_) {
ncclCommDestroy(comm_);
}
}
~NcclCommList() {
if (comms) {
for (int i = 0; i < ndevices; i++) {
int dummy_var;
if (cudaGetDevice(&dummy_var) != cudaSuccess) {
/* there are cases when this destructor is called after the
CUDA driver is already unloaded from the process.
In these cases, skip ncclCommDestroy */
return;
}
ncclCommDestroy(comms[i]);
}
}
}
