void test_read_parallel(int num_verts) { Core moab; Interface& mb = moab; EntityHandle file_set; ErrorCode rval; rval = mb.create_meshset(MESHSET_SET, file_set); CHECK_ERR(rval); std::string opt = std::string("PARALLEL=READ_PART;PARTITION=;PARTITION_DISTRIBUTE;PARALLEL_RESOLVE_SHARED_ENTS") + partition_method; rval = mb.load_file(example, &file_set, opt.c_str()); CHECK_ERR(rval); ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0); rval = pcomm->check_all_shared_handles(); CHECK_ERR(rval); // get the total # owned verts Range verts; rval = mb.get_entities_by_type(0, MBVERTEX, verts); CHECK_ERR(rval); rval = pcomm->filter_pstatus(verts, PSTATUS_NOT_OWNED, PSTATUS_NOT); CHECK_ERR(rval); int my_num = verts.size(), total_verts; MPI_Reduce(&my_num, &total_verts, 1, MPI_INTEGER, MPI_SUM, 0, pcomm->proc_config().proc_comm()); if (0 == pcomm->proc_config().proc_rank()) CHECK_EQUAL(total_verts, num_verts); }
void gather_one_cell_var(int gather_set_rank) { Core moab; Interface& mb = moab; EntityHandle file_set; ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set); CHECK_ERR(rval); read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS"; std::ostringstream gather_set_option; gather_set_option << ";GATHER_SET=" << gather_set_rank; read_options += gather_set_option.str(); rval = mb.load_file(example, &file_set, read_options.c_str()); CHECK_ERR(rval); ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0); int procs = pcomm->proc_config().proc_size(); int rank = pcomm->proc_config().proc_rank(); // Make sure gather_set_rank is valid if (gather_set_rank < 0 || gather_set_rank >= procs) return; Range cells, cells_owned; rval = mb.get_entities_by_type(file_set, MBPOLYGON, cells); CHECK_ERR(rval); // Get local owned cells rval = pcomm->filter_pstatus(cells, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &cells_owned); CHECK_ERR(rval); EntityHandle gather_set = 0; if (gather_set_rank == rank) { // Get gather set ReadUtilIface* readUtilIface; mb.query_interface(readUtilIface); rval = readUtilIface->get_gather_set(gather_set); CHECK_ERR(rval); assert(gather_set != 0); } Tag vorticity_tag0, gid_tag; rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0, MB_TAG_DENSE); CHECK_ERR(rval); rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE); CHECK_ERR(rval); pcomm->gather_data(cells_owned, vorticity_tag0, gid_tag, gather_set, gather_set_rank); if (gather_set_rank == rank) { // Get gather set cells Range gather_set_cells; rval = mb.get_entities_by_type(gather_set, MBPOLYGON, gather_set_cells); CHECK_ERR(rval); CHECK_EQUAL((size_t)642, gather_set_cells.size()); CHECK_EQUAL((size_t)1, gather_set_cells.psize()); // Check vorticity0 tag values on 4 gather set cells: first cell, two median cells, and last cell EntityHandle cell_ents[] = {gather_set_cells[0], gather_set_cells[320], gather_set_cells[321], gather_set_cells[641]}; double vorticity0_val[4 * layers]; rval = mb.tag_get_data(vorticity_tag0, &cell_ents[0], 4, vorticity0_val); CHECK_ERR(rval); // Only check first two layers // Layer 0 CHECK_REAL_EQUAL(3.629994, vorticity0_val[0 * layers], eps); CHECK_REAL_EQUAL(0.131688, vorticity0_val[1 * layers], eps); CHECK_REAL_EQUAL(-0.554888, vorticity0_val[2 * layers], eps); CHECK_REAL_EQUAL(-0.554888, vorticity0_val[3 * layers], eps); // Layer 1 CHECK_REAL_EQUAL(3.629944, vorticity0_val[0 * layers + 1], eps); CHECK_REAL_EQUAL(0.131686, vorticity0_val[1 * layers + 1], eps); CHECK_REAL_EQUAL(-0.554881, vorticity0_val[2 * layers + 1], eps); CHECK_REAL_EQUAL(-0.554881, vorticity0_val[3 * layers + 1], eps); } }
void read_mesh_parallel(bool rcbzoltan) { Core moab; Interface& mb = moab; read_options = "PARALLEL=READ_PART;PARTITION_METHOD=TRIVIAL;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE="; if (rcbzoltan) read_options = "PARALLEL=READ_PART;PARTITION_METHOD=RCBZOLTAN;PARALLEL_RESOLVE_SHARED_ENTS;VARIABLE="; ErrorCode rval = mb.load_file(example, NULL, read_options.c_str()); CHECK_ERR(rval); ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0); int procs = pcomm->proc_config().proc_size(); int rank = pcomm->proc_config().proc_rank(); rval = pcomm->check_all_shared_handles(); CHECK_ERR(rval); // Get local vertices Range local_verts; rval = mb.get_entities_by_type(0, MBVERTEX, local_verts); CHECK_ERR(rval); int verts_num = local_verts.size(); if (2 == procs) { if (rcbzoltan) { if (0 == rank) CHECK_EQUAL(684, verts_num); else if (1 == rank) CHECK_EQUAL(691, verts_num); // Not owned vertices included } else { if (0 == rank) CHECK_EQUAL(687, verts_num); else if (1 == rank) CHECK_EQUAL(688, verts_num); // Not owned vertices included } } rval = pcomm->filter_pstatus(local_verts, PSTATUS_NOT_OWNED, PSTATUS_NOT); CHECK_ERR(rval); verts_num = local_verts.size(); if (2 == procs) { if (rcbzoltan) { if (0 == rank) CHECK_EQUAL(684, verts_num); else if (1 == rank) CHECK_EQUAL(596, verts_num); // Not owned vertices excluded } else { if (0 == rank) CHECK_EQUAL(687, verts_num); else if (1 == rank) CHECK_EQUAL(593, verts_num); // Not owned vertices excluded } } // Get local edges Range local_edges; rval = mb.get_entities_by_type(0, MBEDGE, local_edges); CHECK_ERR(rval); int edges_num = local_edges.size(); if (2 == procs) { if (rcbzoltan) { if (0 == rank) CHECK_EQUAL(1002, edges_num); else if (1 == rank) CHECK_EQUAL(1013, edges_num); // Not owned edges included } else { if (0 == rank) CHECK_EQUAL(1007, edges_num); else if (1 == rank) CHECK_EQUAL(1008, edges_num); // Not owned edges included } } rval = pcomm->filter_pstatus(local_edges, PSTATUS_NOT_OWNED, PSTATUS_NOT); CHECK_ERR(rval); edges_num = local_edges.size(); if (2 == procs) { if (rcbzoltan) { if (0 == rank) CHECK_EQUAL(1002, edges_num); else if (1 == rank) CHECK_EQUAL(918, edges_num); // Not owned edges excluded } else { if (0 == rank) CHECK_EQUAL(1007, edges_num); else if (1 == rank) CHECK_EQUAL(913, edges_num); // Not owned edges excluded } } // Get local cells Range local_cells; rval = mb.get_entities_by_type(0, MBPOLYGON, local_cells); CHECK_ERR(rval); // No mixed elements CHECK_EQUAL((size_t)1, local_cells.psize()); int cells_num = local_cells.size(); if (2 == procs) { if (rcbzoltan) { if (0 == rank) CHECK_EQUAL(319, cells_num); else CHECK_EQUAL(323, cells_num); } else CHECK_EQUAL(321, cells_num); } rval = pcomm->filter_pstatus(local_cells, PSTATUS_NOT_OWNED, PSTATUS_NOT); CHECK_ERR(rval); cells_num = local_cells.size(); if (2 == procs) { if (rcbzoltan) { if (0 == rank) CHECK_EQUAL(319, cells_num); else CHECK_EQUAL(323, cells_num); } else CHECK_EQUAL(321, cells_num); } std::cout << "proc: " << rank << " verts:" << verts_num << "\n"; int total_verts_num; MPI_Reduce(&verts_num, &total_verts_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm()); if (0 == rank) { std::cout << "total vertices: " << total_verts_num << "\n"; CHECK_EQUAL(1280, total_verts_num); } std::cout << "proc: " << rank << " edges:" << edges_num << "\n"; int total_edges_num; MPI_Reduce(&edges_num, &total_edges_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm()); if (0 == rank) { std::cout << "total edges: " << total_edges_num << "\n"; CHECK_EQUAL(1920, total_edges_num); } std::cout << "proc: " << rank << " cells:" << cells_num << "\n"; int total_cells_num; MPI_Reduce(&cells_num, &total_cells_num, 1, MPI_INT, MPI_SUM, 0, pcomm->proc_config().proc_comm()); if (0 == rank) { std::cout << "total cells: " << total_cells_num << "\n"; CHECK_EQUAL(642, total_cells_num); } #ifdef MOAB_HAVE_HDF5_PARALLEL std::string write_options("PARALLEL=WRITE_PART;"); std::string output_file = "test_gcrm"; if (rcbzoltan) output_file += "_rcbzoltan"; output_file += ".h5m"; mb.write_file(output_file.c_str(), NULL, write_options.c_str()); #endif }
void test_gather_onevar() { Core moab; Interface& mb = moab; EntityHandle file_set; ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set); CHECK_ERR(rval); std::string opts; get_options(opts); // Read cell variable vorticity and create gather set on processor 0 opts += ";VARIABLE=vorticity;GATHER_SET=0"; rval = mb.load_file(example, &file_set, opts.c_str()); CHECK_ERR(rval); #ifdef MOAB_HAVE_MPI ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0); int rank = pcomm->proc_config().proc_rank(); Range cells, cells_owned; rval = mb.get_entities_by_type(file_set, MBPOLYGON, cells); CHECK_ERR(rval); // Get local owned cells rval = pcomm->filter_pstatus(cells, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &cells_owned); CHECK_ERR(rval); EntityHandle gather_set = 0; if (0 == rank) { // Get gather set ReadUtilIface* readUtilIface; mb.query_interface(readUtilIface); rval = readUtilIface->get_gather_set(gather_set); CHECK_ERR(rval); assert(gather_set != 0); } Tag vorticity_tag0, gid_tag; rval = mb.tag_get_handle("vorticity0", layers, MB_TYPE_DOUBLE, vorticity_tag0, MB_TAG_DENSE); CHECK_ERR(rval); rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE); CHECK_ERR(rval); pcomm->gather_data(cells_owned, vorticity_tag0, gid_tag, gather_set, 0); if (0 == rank) { // Get gather set cells Range gather_set_cells; rval = mb.get_entities_by_type(gather_set, MBPOLYGON, gather_set_cells); CHECK_ERR(rval); CHECK_EQUAL((size_t)642, gather_set_cells.size()); CHECK_EQUAL((size_t)1, gather_set_cells.psize()); // Check vorticity0 tag values on 4 gather set cells: first cell, two median cells, and last cell EntityHandle cell_ents[] = {gather_set_cells[0], gather_set_cells[320], gather_set_cells[321], gather_set_cells[641]}; double vorticity0_val[4 * layers]; rval = mb.tag_get_data(vorticity_tag0, cell_ents, 4, vorticity0_val); CHECK_ERR(rval); // Only check first two layers // Layer 0 CHECK_REAL_EQUAL(3.629994, vorticity0_val[0 * layers], eps); CHECK_REAL_EQUAL(0.131688, vorticity0_val[1 * layers], eps); CHECK_REAL_EQUAL(-0.554888, vorticity0_val[2 * layers], eps); CHECK_REAL_EQUAL(-0.554888, vorticity0_val[3 * layers], eps); // Layer 1 CHECK_REAL_EQUAL(3.629944, vorticity0_val[0 * layers + 1], eps); CHECK_REAL_EQUAL(0.131686, vorticity0_val[1 * layers + 1], eps); CHECK_REAL_EQUAL(-0.554881, vorticity0_val[2 * layers + 1], eps); CHECK_REAL_EQUAL(-0.554881, vorticity0_val[3 * layers + 1], eps); } #endif }
void test_gather_onevar() { Core moab; Interface& mb = moab; EntityHandle file_set; ErrorCode rval = mb.create_meshset(MESHSET_SET, file_set); CHECK_ERR(rval); std::string opts; get_options(opts); // Read vertex variable T and create gather set on processor 0 opts += ";VARIABLE=T;GATHER_SET=0"; #ifdef MOAB_HAVE_MPI opts += ";PARALLEL_RESOLVE_SHARED_ENTS"; #endif rval = mb.load_file(example, &file_set, opts.c_str()); CHECK_ERR(rval); #ifdef MOAB_HAVE_MPI ParallelComm* pcomm = ParallelComm::get_pcomm(&mb, 0); int rank = pcomm->proc_config().proc_rank(); Range verts, verts_owned; rval = mb.get_entities_by_type(file_set, MBVERTEX, verts); CHECK_ERR(rval); // Get local owned vertices rval = pcomm->filter_pstatus(verts, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &verts_owned); CHECK_ERR(rval); EntityHandle gather_set = 0; if (0 == rank) { // Get gather set ReadUtilIface* readUtilIface; mb.query_interface(readUtilIface); rval = readUtilIface->get_gather_set(gather_set); CHECK_ERR(rval); assert(gather_set != 0); } Tag Ttag0, gid_tag; rval = mb.tag_get_handle("T0", levels, MB_TYPE_DOUBLE, Ttag0, MB_TAG_DENSE); CHECK_ERR(rval); rval = mb.tag_get_handle(GLOBAL_ID_TAG_NAME, 1, MB_TYPE_INTEGER, gid_tag, MB_TAG_DENSE); CHECK_ERR(rval); pcomm->gather_data(verts_owned, Ttag0, gid_tag, gather_set, 0); if (0 == rank) { // Get gather set vertices Range gather_set_verts; rval = mb.get_entities_by_type(gather_set, MBVERTEX, gather_set_verts); CHECK_ERR(rval); CHECK_EQUAL((size_t)3458, gather_set_verts.size()); // Get T0 tag values on 4 strategically selected gather set vertices double T0_val[4 * levels]; EntityHandle vert_ents[] = {gather_set_verts[0], gather_set_verts[1728], gather_set_verts[1729], gather_set_verts[3457]}; rval = mb.tag_get_data(Ttag0, vert_ents, 4, T0_val); CHECK_ERR(rval); const double eps = 0.001; // Check first level values CHECK_REAL_EQUAL(233.1136, T0_val[0 * levels], eps); // First vert CHECK_REAL_EQUAL(236.1505, T0_val[1 * levels], eps); // Median vert CHECK_REAL_EQUAL(235.7722, T0_val[2 * levels], eps); // Median vert CHECK_REAL_EQUAL(234.0416, T0_val[3 * levels], eps); // Last vert } #endif }
ErrorCode get_max_volume(Core &mb, EntityHandle fileset, int dim, double &vmax) { ErrorCode error; VerdictWrapper vw(&mb); QualityType q; switch (dim) { case 1: q = MB_LENGTH; break; case 2: q = MB_AREA; break; case 3: q = MB_VOLUME; break; default: return MB_FAILURE; break; } //Get all entities of the highest dimension which is passed as a command line argument. Range allents, owned; error = mb.get_entities_by_handle(fileset, allents);MB_CHK_ERR(error); owned = allents.subset_by_dimension(dim);MB_CHK_ERR(error); //Get all owned entities #ifdef MOAB_HAVE_MPI int size = 1; MPI_Comm_size( MPI_COMM_WORLD, &size ); int mpi_err; if (size>1) { // filter the entities not owned, so we do not process them more than once ParallelComm* pcomm = moab::ParallelComm::get_pcomm(&mb, 0); Range current = owned; owned.clear(); error = pcomm->filter_pstatus(current, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &owned); if (error != MB_SUCCESS) { MPI_Finalize(); return MB_FAILURE; } } #endif double vmax_local=0; //Find the maximum volume of an entity in the owned mesh for (Range::iterator it=owned.begin(); it != owned.end(); it++) { double volume; error = vw.quality_measure(*it, q, volume);MB_CHK_ERR(error); if (volume >vmax_local) vmax_local = volume; } //Get the global maximum double vmax_global = vmax_local; #ifdef MOAB_HAVE_MPI mpi_err = MPI_Reduce(&vmax_local, &vmax_global, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); if (mpi_err) { MPI_Finalize(); return MB_FAILURE; } #endif vmax = vmax_global; return MB_SUCCESS; }
int main(int argc, char **argv) { #ifdef MOAB_HAVE_MPI MPI_Init(&argc, &argv); string options; // Need option handling here for input filename if (argc > 1) { // User has input a mesh file test_file_name = argv[1]; } int nbComms = 1; if (argc > 2) nbComms = atoi(argv[2]); options = "PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS"; // Get MOAB instance Interface* mb = new (std::nothrow) Core; if (NULL == mb) return 1; MPI_Comm comm; int global_rank, global_size; MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); MPI_Comm_rank(MPI_COMM_WORLD, &global_size); int color = global_rank % nbComms; // For each angle group a different color if (nbComms > 1) { // Split the communicator, into ngroups = nbComms MPI_Comm_split(MPI_COMM_WORLD, color, global_rank, &comm); } else comm = MPI_COMM_WORLD; // Get the ParallelComm instance ParallelComm* pcomm = new ParallelComm(mb, comm); int nprocs = pcomm->proc_config().proc_size(); int rank = pcomm->proc_config().proc_rank(); #ifndef NDEBUG MPI_Comm rcomm = pcomm->proc_config().proc_comm(); assert(rcomm == comm); #endif if (0 == global_rank) cout << " global rank:" << global_rank << " color:" << color << " rank:" << rank << " of " << nprocs << " processors\n"; if (1 == global_rank) cout << " global rank:" << global_rank << " color:" << color << " rank:" << rank << " of " << nprocs << " processors\n"; MPI_Barrier(MPI_COMM_WORLD); if (0 == global_rank) cout << "Reading file " << test_file_name << "\n with options: " << options << "\n on " << nprocs << " processors on " << nbComms << " communicator(s)\n"; // Read the file with the specified options ErrorCode rval = mb->load_file(test_file_name.c_str(), 0, options.c_str());MB_CHK_ERR(rval); Range shared_ents; // Get entities shared with all other processors rval = pcomm->get_shared_entities(-1, shared_ents);MB_CHK_ERR(rval); // Filter shared entities with not not_owned, which means owned Range owned_entities; rval = pcomm->filter_pstatus(shared_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &owned_entities);MB_CHK_ERR(rval); unsigned int nums[4] = {0}; // to store the owned entities per dimension for (int i = 0; i < 4; i++) nums[i] = (int)owned_entities.num_of_dimension(i); vector<int> rbuf(nprocs*4, 0); MPI_Gather(nums, 4, MPI_INT, &rbuf[0], 4, MPI_INT, 0, comm); // Print the stats gathered: if (0 == global_rank) { for (int i = 0; i < nprocs; i++) cout << " Shared, owned entities on proc " << i << ": " << rbuf[4*i] << " verts, " << rbuf[4*i + 1] << " edges, " << rbuf[4*i + 2] << " faces, " << rbuf[4*i + 3] << " elements" << endl; } // Now exchange 1 layer of ghost elements, using vertices as bridge // (we could have done this as part of reading process, using the PARALLEL_GHOSTS read option) rval = pcomm->exchange_ghost_cells(3, // int ghost_dim 0, // int bridge_dim 1, // int num_layers 0, // int addl_ents true);MB_CHK_ERR(rval); // bool store_remote_handles // Repeat the reports, after ghost exchange shared_ents.clear(); owned_entities.clear(); rval = pcomm->get_shared_entities(-1, shared_ents);MB_CHK_ERR(rval); rval = pcomm->filter_pstatus(shared_ents, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, &owned_entities);MB_CHK_ERR(rval); // Find out how many shared entities of each dimension are owned on this processor for (int i = 0; i < 4; i++) nums[i] = (int)owned_entities.num_of_dimension(i); // Gather the statistics on processor 0 MPI_Gather(nums, 4, MPI_INT, &rbuf[0], 4, MPI_INT, 0, comm); if (0 == global_rank) { cout << " \n\n After exchanging one ghost layer: \n"; for (int i = 0; i < nprocs; i++) { cout << " Shared, owned entities on proc " << i << ": " << rbuf[4*i] << " verts, " << rbuf[4*i + 1] << " edges, " << rbuf[4*i + 2] << " faces, " << rbuf[4*i + 3] << " elements" << endl; } } delete mb; MPI_Finalize(); #else std::cout<<" compile with MPI and hdf5 for this example to work\n"; #endif return 0; }