void comm_init(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data) { if ( QMP_is_initialized() != QMP_TRUE ) { errorQuda("QMP has not been initialized"); } int grid_size = 1; for (int i = 0; i < ndim; i++) { grid_size *= dims[i]; } if (grid_size != QMP_get_number_of_nodes()) { errorQuda("Communication grid size declared via initCommsGridQuda() does not match" " total number of QMP nodes (%d != %d)", grid_size, QMP_get_number_of_nodes()); } Topology *topo = comm_create_topology(ndim, dims, rank_from_coords, map_data); comm_set_default_topology(topo); // determine which GPU this process will use (FIXME: adopt the scheme in comm_mpi.cpp) int device_count; cudaGetDeviceCount(&device_count); if (device_count == 0) { errorQuda("No CUDA devices found"); } gpuid = (comm_rank() % device_count); }
void comm_init(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data) { int initialized; MPI_CHECK( MPI_Initialized(&initialized) ); if (!initialized) { errorQuda("MPI has not been initialized"); } MPI_CHECK( MPI_Comm_rank(MPI_COMM_WORLD, &rank) ); MPI_CHECK( MPI_Comm_size(MPI_COMM_WORLD, &size) ); int grid_size = 1; for (int i = 0; i < ndim; i++) { grid_size *= dims[i]; } if (grid_size != size) { errorQuda("Communication grid size declared via initCommsGridQuda() does not match" " total number of MPI ranks (%d != %d)", grid_size, size); } Topology *topo = comm_create_topology(ndim, dims, rank_from_coords, map_data); comm_set_default_topology(topo); // determine which GPU this MPI rank will use char *hostname = comm_hostname(); char *hostname_recv_buf = (char *)safe_malloc(128*size); MPI_CHECK( MPI_Allgather(hostname, 128, MPI_CHAR, hostname_recv_buf, 128, MPI_CHAR, MPI_COMM_WORLD) ); gpuid = 0; for (int i = 0; i < rank; i++) { if (!strncmp(hostname, &hostname_recv_buf[128*i], 128)) { gpuid++; } } host_free(hostname_recv_buf); int device_count; cudaGetDeviceCount(&device_count); if (device_count == 0) { errorQuda("No CUDA devices found"); } if (gpuid >= device_count) { errorQuda("Too few GPUs available on %s", hostname); } }
void comm_init(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data) { Topology *topo = comm_create_topology(ndim, dims, rank_from_coords, map_data); comm_set_default_topology(topo); }
void comm_finalize(void) { Topology *topo = comm_default_topology(); comm_destroy_topology(topo); comm_set_default_topology(NULL); }