void DomainsTest::subTestGridDomains(const Dimensions mpiSize, const Dimensions gridSize, uint32_t rank, uint32_t iteration) { int max_rank = 1; for (int i = 0; i < rank; i++) max_rank *= mpiSize[i]; int mpi_rank = totalMpiRank; if (mpi_rank < max_rank) { // write data int *data_write = new int[gridSize.getDimSize()]; // initialize data for writing with mpi index (mpi rank) Dimensions mpi_position(mpi_rank % mpiSize[0], (mpi_rank / mpiSize[0]) % mpiSize[1], (mpi_rank / mpiSize[0]) / mpiSize[1]); for (size_t i = 0; i < gridSize.getDimSize(); ++i) data_write[i] = mpi_rank; DataCollector::FileCreationAttr fattr; fattr.fileAccType = DataCollector::FAT_CREATE; fattr.mpiSize.set(mpiSize); fattr.mpiPosition.set(mpi_position); dataCollector->open(hdf5_file_grid, fattr); Dimensions domain_offset = mpi_position * gridSize; #if defined TESTS_DEBUG std::cout << "writing..." << std::endl; std::cout << "mpi_position = " << mpi_position.toString() << std::endl; std::cout << "domain_offset = " << domain_offset.toString() << std::endl; #endif dataCollector->writeDomain(iteration, ctInt, rank, gridSize, "grid_data", domain_offset, gridSize, DomainCollector::GridType, data_write); dataCollector->close(); int *data_read = new int[gridSize.getDimSize()]; fattr.fileAccType = DataCollector::FAT_READ; dataCollector->open(hdf5_file_grid, fattr); DomainCollector::DomDataClass data_class = DomainCollector::UndefinedType; DataContainer *container = dataCollector->readDomain(iteration, "grid_data", domain_offset, gridSize, &data_class, false); dataCollector->close(); CPPUNIT_ASSERT(container != NULL); CPPUNIT_ASSERT(container->getNumSubdomains() == 1); CPPUNIT_ASSERT(container->getNumElements() == gridSize.getDimSize()); for (size_t i = 0; i < gridSize.getDimSize(); ++i) CPPUNIT_ASSERT(*((int*)(container->getElement(i))) == mpi_rank); delete container; delete[] data_read; delete[] data_write; data_write = NULL; } MPI_Barrier(MPI_COMM_WORLD); if (mpi_rank == 0) { // now read and test domain subdomains DataCollector::FileCreationAttr fattr; fattr.fileAccType = DataCollector::FAT_READ_MERGED; fattr.mpiSize = mpiSize; dataCollector->open(hdf5_file_grid, fattr); size_t global_domain_elements = dataCollector->getTotalElements(iteration, "grid_data"); Dimensions global_grid_size = mpiSize * gridSize; #if defined TESTS_DEBUG std::cout << "global_domain_elements = " << global_domain_elements << std::endl; std::cout << "global_grid_size = " << global_grid_size.toString() << std::endl; #endif CPPUNIT_ASSERT(global_domain_elements == global_grid_size.getDimSize()); // test different domain offsets for (uint32_t i = 0; i < 5; ++i) { Dimensions offset(rand() % global_grid_size[0], rand() % global_grid_size[1], rand() % global_grid_size[2]); Dimensions partition_size = global_grid_size - offset; #if defined TESTS_DEBUG std::cout << "offset = " << offset.toString() << std::endl; std::cout << "partition_size = " << partition_size.toString() << std::endl; #endif IDomainCollector::DomDataClass data_class = IDomainCollector::UndefinedType; Domain total_domain; total_domain = dataCollector->getTotalDomain(iteration, "grid_data"); CPPUNIT_ASSERT(total_domain.getStart() == Dimensions(0, 0, 0)); CPPUNIT_ASSERT(total_domain.getSize() == global_grid_size); // read data container DataContainer *container = dataCollector->readDomain(iteration, "grid_data", offset, partition_size, &data_class); #if defined TESTS_DEBUG std::cout << "container->getNumSubdomains() = " << container->getNumSubdomains() << std::endl; #endif // check the container CPPUNIT_ASSERT(data_class == IDomainCollector::GridType); CPPUNIT_ASSERT(container->getNumSubdomains() == 1); // check all DomainData entries in the container DomainData *subdomain = container->getIndex(0); CPPUNIT_ASSERT(subdomain != NULL); CPPUNIT_ASSERT(subdomain->getData() != NULL); Dimensions subdomain_elements = subdomain->getElements(); Dimensions subdomain_start = subdomain->getStart(); #if defined TESTS_DEBUG std::cout << "subdomain->getStart() = " << subdomain->getStart().toString() << std::endl; std::cout << "subdomain->getElements() = " << subdomain_elements.toString() << std::endl; #endif int *subdomain_data = (int*) (subdomain->getData()); CPPUNIT_ASSERT(subdomain_elements.getDimSize() != 0); CPPUNIT_ASSERT(gridSize.getDimSize() != 0); for (int j = 0; j < subdomain_elements.getDimSize(); ++j) { // Find out the expected value (original mpi rank) // for exactly this data element. Dimensions j_grid_position(j % subdomain_elements[0], (j / subdomain_elements[0]) % subdomain_elements[1], (j / subdomain_elements[0]) / subdomain_elements[1]); Dimensions total_grid_position = (subdomain_start + j_grid_position) / gridSize; int expected_value = total_grid_position[2] * mpiSize[0] * mpiSize[1] + total_grid_position[1] * mpiSize[0] + total_grid_position[0]; #if defined TESTS_DEBUG std::cout << "j = " << j << ", subdomain_data[j] = " << subdomain_data[j] << ", expected_value = " << expected_value << std::endl; #endif CPPUNIT_ASSERT(subdomain_data[j] == expected_value); } delete container; container = NULL; } dataCollector->close(); } }
void DomainsTest::subTestPolyDomains(const Dimensions mpiSize, uint32_t numElements, uint32_t iteration) { Dimensions grid_size(20, 10, 5); Dimensions global_grid_size = grid_size * mpiSize; int max_rank = 1; for (int i = 0; i < 3; i++) max_rank *= mpiSize[i]; int mpi_rank = totalMpiRank; size_t mpi_elements = numElements * (mpi_rank + 1); if (mpi_rank < max_rank) { // write data float *data_write = new float[mpi_elements]; // initialize data for writing with mpi index (mpi rank) Dimensions mpi_position(mpi_rank % mpiSize[0], (mpi_rank / mpiSize[0]) % mpiSize[1], (mpi_rank / mpiSize[0]) / mpiSize[1]); for (size_t i = 0; i < mpi_elements; ++i) data_write[i] = (float) mpi_rank; DataCollector::FileCreationAttr fattr; fattr.fileAccType = DataCollector::FAT_CREATE; fattr.mpiSize.set(mpiSize); fattr.mpiPosition.set(mpi_position); dataCollector->open(hdf5_file_poly, fattr); Dimensions domain_offset = mpi_position * grid_size; #if defined TESTS_DEBUG std::cout << "writing..." << std::endl; std::cout << "mpi_position = " << mpi_position.toString() << std::endl; std::cout << "domain_offset = " << domain_offset.toString() << std::endl; #endif dataCollector->writeDomain(iteration, ctFloat, 1, Dimensions(mpi_elements, 1, 1), "poly_data", domain_offset, grid_size, DomainCollector::PolyType, data_write); dataCollector->close(); delete[] data_write; data_write = NULL; } MPI_Barrier(MPI_COMM_WORLD); if (mpi_rank == 0) { // now read and test domain subdomains DataCollector::FileCreationAttr fattr; fattr.fileAccType = DataCollector::FAT_READ_MERGED; fattr.mpiSize = mpiSize; dataCollector->open(hdf5_file_poly, fattr); size_t global_domain_elements = dataCollector->getTotalElements(iteration, "poly_data"); size_t global_num_elements = 0; for (int i = 0; i < mpiSize.getDimSize(); ++i) global_num_elements += numElements * (i + 1); #if defined TESTS_DEBUG std::cout << "global_domain_elements = " << global_domain_elements << std::endl; std::cout << "global_num_elements = " << global_num_elements << std::endl; #endif CPPUNIT_ASSERT(global_domain_elements == global_num_elements); // test different domain offsets for (uint32_t i = 0; i < 5; ++i) { Dimensions offset(rand() % global_grid_size[0], rand() % global_grid_size[1], rand() % global_grid_size[2]); Dimensions partition_size = global_grid_size - offset; #if defined TESTS_DEBUG std::cout << "offset = " << offset.toString() << std::endl; std::cout << "partition_size = " << partition_size.toString() << std::endl; #endif IDomainCollector::DomDataClass data_class = IDomainCollector::UndefinedType; // read data container DataContainer *container = dataCollector->readDomain(iteration, "poly_data", offset, partition_size, &data_class); #if defined TESTS_DEBUG std::cout << "container->getNumSubdomains() = " << container->getNumSubdomains() << std::endl; #endif // check the container CPPUNIT_ASSERT(data_class == IDomainCollector::PolyType); CPPUNIT_ASSERT(container->getNumSubdomains() >= 1); // check all DomainData entries in the container for (int i = 0; i < container->getNumSubdomains(); ++i) { DomainData *subdomain = container->getIndex(i); CPPUNIT_ASSERT(subdomain != NULL); CPPUNIT_ASSERT(subdomain->getData() != NULL); Dimensions subdomain_elements = subdomain->getElements(); Dimensions subdomain_start = subdomain->getStart(); #if defined TESTS_DEBUG std::cout << "subdomain->getElements() = " << subdomain->getElements().toString() << std::endl; std::cout << "subdomain->getSize() = " << subdomain->getSize().toString() << std::endl; #endif float *subdomain_data = (float*) (subdomain->getData()); CPPUNIT_ASSERT(subdomain_elements.getDimSize() != 0); // Find out the expected value (original mpi rank) // for this subdomain. Dimensions subdomain_mpi_pos = subdomain_start / grid_size; int subdomain_mpi_rank = subdomain_mpi_pos[2] * mpiSize[1] * mpiSize[0] + subdomain_mpi_pos[1] * mpiSize[0] + subdomain_mpi_pos[0]; CPPUNIT_ASSERT(subdomain_elements.getDimSize() == numElements * (subdomain_mpi_rank + 1)); for (int j = 0; j < subdomain_elements.getDimSize(); ++j) { #if defined TESTS_DEBUG std::cout << "j = " << j << ", subdomain_data[j]) = " << subdomain_data[j] << ", subdomain_mpi_rank = " << subdomain_mpi_rank << std::endl; #endif CPPUNIT_ASSERT(subdomain_data[j] == (float) subdomain_mpi_rank); } } delete container; container = NULL; } dataCollector->close(); } MPI_Barrier(MPI_COMM_WORLD); }