TEST(PeerAccess, DisableAccessInvalidDevice) { cudaError_t ret; int devices; ret = cudaGetDeviceCount(&devices); ASSERT_EQ(cudaSuccess, ret); ret = cudaDeviceDisablePeerAccess(devices); EXPECT_EQ(cudaErrorInvalidDevice, ret); ret = cudaDeviceDisablePeerAccess(-1); EXPECT_EQ(cudaErrorInvalidDevice, ret); }
void THCState_setPeerToPeerAccess(THCState* state, int dev, int devToAccess, int enable) { /* This will perform device bounds checking for us */ int prevEnabled = THCState_getPeerToPeerAccess(state, dev, devToAccess); if (enable != prevEnabled) { /* If we're attempting to enable p2p access but p2p access isn't */ /* supported, throw an error */ if (enable) { int access = 0; THCudaCheck(cudaDeviceCanAccessPeer(&access, dev, devToAccess)); if (!access) { THError("p2p access not supported for %d accessing %d", dev, devToAccess); } } state->p2pAccessEnabled[dev][devToAccess] = enable; int prevDev = 0; THCudaCheck(cudaGetDevice(&prevDev)); THCudaCheck(cudaSetDevice(dev)); /* This should be in sync with the current access state */ if (enable) { THCudaCheck(cudaDeviceEnablePeerAccess(devToAccess, 0)); } else { THCudaCheck(cudaDeviceDisablePeerAccess(devToAccess)); } THCudaCheck(cudaSetDevice(prevDev)); } }
TEST(PeerAccess, EnableDisable) { cudaError_t ret; int devices; ret = cudaGetDeviceCount(&devices); ASSERT_EQ(cudaSuccess, ret); if (devices <= 1) { return; } int version; ret = cudaRuntimeGetVersion(&version); ASSERT_EQ(cudaSuccess, ret); typedef std::pair<int, int> peer_t; std::vector<peer_t> peers; for (int i = 0; i < devices; i++) { ret = cudaSetDevice(i); ASSERT_EQ(cudaSuccess, ret); for (int j = 0; j < devices; j++) { int peer; ret = cudaDeviceCanAccessPeer(&peer, i, j); ASSERT_EQ(cudaSuccess, ret); cudaError_t expected; if (peer) { expected = cudaSuccess; peers.push_back(peer_t(i, j)); #if CUDA_VERSION >= 5000 } else if (version >= 5000 /* 5.0 */) { expected = cudaErrorPeerAccessUnsupported; #endif } else { expected = cudaErrorInvalidDevice; } ret = cudaDeviceEnablePeerAccess(j, 0); EXPECT_EQ(expected, ret); } } /* Cleanup. */ const size_t n_peers = peers.size(); for (size_t i = 0; i < n_peers; i++) { ret = cudaSetDevice(peers[i].first); ASSERT_EQ(cudaSuccess, ret); ret = cudaDeviceDisablePeerAccess(peers[i].second); EXPECT_EQ(cudaSuccess, ret); } }
TEST(PeerAccess, DeviceReset) { cudaError_t ret; int devices; ret = cudaGetDeviceCount(&devices); ASSERT_EQ(cudaSuccess, ret); if (devices <= 1) { return; } bool found = false; int dj; for (int i = 0; i < devices && !(found); i++) { ret = cudaSetDevice(i); ASSERT_EQ(cudaSuccess, ret); for (int j = 0; j < devices; j++) { int peer; ret = cudaDeviceCanAccessPeer(&peer, i, j); ASSERT_EQ(cudaSuccess, ret); if (peer) { ret = cudaDeviceEnablePeerAccess(j, 0); ASSERT_EQ(cudaSuccess, ret); found = true; dj = j; break; } } } if (!(found)) { return; } /* Perform a device reset. */ ret = cudaDeviceReset(); EXPECT_EQ(cudaSuccess, ret); ret = cudaDeviceDisablePeerAccess(dj); EXPECT_EQ(cudaErrorPeerAccessNotEnabled, ret); }
P2PSync<Dtype>::~P2PSync() { #ifndef CPU_ONLY int initial_device; CUDA_CHECK(cudaGetDevice(&initial_device)); const int self = solver_->param().device_id(); CUDA_CHECK(cudaSetDevice(self)); if (parent_) { CUDA_CHECK(cudaFree(parent_grads_)); const int peer = parent_->solver_->param().device_id(); int access; CUDA_CHECK(cudaDeviceCanAccessPeer(&access, self, peer)); if (access) { CUDA_CHECK(cudaDeviceDisablePeerAccess(peer)); } } CUDA_CHECK(cudaSetDevice(initial_device)); #endif }