DTKInterpolationAdapter::DTKInterpolationAdapter(Teuchos::RCP<const Teuchos::MpiComm<int> > in_comm, EquationSystems & in_es, const Point & offset, unsigned int from_dim): comm(in_comm), es(in_es), _offset(offset), mesh(in_es.get_mesh()), dim(mesh.mesh_dimension()) { MPI_Comm old_comm = Moose::swapLibMeshComm(*comm->getRawMpiComm()); std::set<GlobalOrdinal> semi_local_nodes; get_semi_local_nodes(semi_local_nodes); num_local_nodes = semi_local_nodes.size(); vertices.resize(num_local_nodes); Teuchos::ArrayRCP<double> coordinates(num_local_nodes * dim); Teuchos::ArrayRCP<double> target_coordinates(num_local_nodes * from_dim); // Fill in the vertices and coordinates { GlobalOrdinal i = 0; for (std::set<GlobalOrdinal>::iterator it = semi_local_nodes.begin(); it != semi_local_nodes.end(); ++it) { const Node & node = mesh.node(*it); vertices[i] = node.id(); for (GlobalOrdinal j=0; j<dim; j++) coordinates[(j*num_local_nodes) + i] = node(j) + offset(j); for (GlobalOrdinal j=0; j<from_dim; j++) target_coordinates[(j*num_local_nodes) + i] = node(j) + offset(j); i++; } } // Currently assuming all elements are the same! DataTransferKit::DTK_ElementTopology element_topology = get_element_topology(mesh.elem(0)); GlobalOrdinal n_nodes_per_elem = mesh.elem(0)->n_nodes(); GlobalOrdinal n_local_elem = mesh.n_local_elem(); elements.resize(n_local_elem); Teuchos::ArrayRCP<GlobalOrdinal> connectivity(n_nodes_per_elem*n_local_elem); Teuchos::ArrayRCP<double> elem_centroid_coordinates(n_local_elem*from_dim); // Fill in the elements and connectivity { GlobalOrdinal i = 0; MeshBase::const_element_iterator end = mesh.local_elements_end(); for (MeshBase::const_element_iterator it = mesh.local_elements_begin(); it != end; ++it) { const Elem & elem = *(*it); elements[i] = elem.id(); for (GlobalOrdinal j=0; j<n_nodes_per_elem; j++) connectivity[(j*n_local_elem)+i] = elem.node(j); { Point centroid = elem.centroid(); for (GlobalOrdinal j=0; j<from_dim; j++) elem_centroid_coordinates[(j*n_local_elem) + i] = centroid(j) + offset(j); } i++; } } Teuchos::ArrayRCP<int> permutation_list(n_nodes_per_elem); for (GlobalOrdinal i = 0; i < n_nodes_per_elem; ++i ) permutation_list[i] = i; /* Moose::out<<"n_nodes_per_elem: "<<n_nodes_per_elem<<std::endl; Moose::out<<"Dim: "<<dim<<std::endl; Moose::err<<"Vertices size: "<<vertices.size()<<std::endl; { Moose::err<<libMesh::processor_id()<<" Vertices: "; for (unsigned int i=0; i<vertices.size(); i++) Moose::err<<vertices[i]<<" "; Moose::err<<std::endl; } Moose::err<<"Coordinates size: "<<coordinates.size()<<std::endl; { Moose::err<<libMesh::processor_id()<<" Coordinates: "; for (unsigned int i=0; i<coordinates.size(); i++) Moose::err<<coordinates[i]<<" "; Moose::err<<std::endl; } Moose::err<<"Connectivity size: "<<connectivity.size()<<std::endl; { Moose::err<<libMesh::processor_id()<<" Connectivity: "; for (unsigned int i=0; i<connectivity.size(); i++) Moose::err<<connectivity[i]<<" "; Moose::err<<std::endl; } Moose::err<<"Permutation_List size: "<<permutation_list.size()<<std::endl; { Moose::err<<libMesh::processor_id()<<" Permutation_List: "; for (unsigned int i=0; i<permutation_list.size(); i++) Moose::err<<permutation_list[i]<<" "; Moose::err<<std::endl; } */ Teuchos::RCP<MeshContainerType> mesh_container = Teuchos::rcp( new MeshContainerType(dim, vertices, coordinates, element_topology, n_nodes_per_elem, elements, connectivity, permutation_list) ); // We only have 1 element topology in this grid so we make just one mesh block Teuchos::ArrayRCP<Teuchos::RCP<MeshContainerType> > mesh_blocks(1); mesh_blocks[0] = mesh_container; // Create the MeshManager mesh_manager = Teuchos::rcp(new DataTransferKit::MeshManager<MeshContainerType>(mesh_blocks, comm, dim) ); // Pack the coordinates into a field, this will be the positions we'll ask for other systems fields at if (from_dim == dim) target_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(mesh_container, comm)); else { Teuchos::ArrayRCP<GlobalOrdinal> empty_elements(0); Teuchos::ArrayRCP<GlobalOrdinal> empty_connectivity(0); Teuchos::RCP<MeshContainerType> coords_only_mesh_container = Teuchos::rcp( new MeshContainerType(from_dim, vertices, target_coordinates, element_topology, n_nodes_per_elem, empty_elements, empty_connectivity, permutation_list) ); target_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(coords_only_mesh_container, comm)); } { Teuchos::ArrayRCP<GlobalOrdinal> empty_elements(0); Teuchos::ArrayRCP<GlobalOrdinal> empty_connectivity(0); Teuchos::RCP<MeshContainerType> centroid_coords_only_mesh_container = Teuchos::rcp( new MeshContainerType(from_dim, elements, elem_centroid_coordinates, element_topology, n_nodes_per_elem, empty_elements, empty_connectivity, permutation_list) ); elem_centroid_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(centroid_coords_only_mesh_container, comm)); } // Swap back Moose::swapLibMeshComm(old_comm); }
/* * \brief This constructor will pull the mesh data DTK needs out of Moab, * partition it for the example, and build a DataTransferKit::MeshContainer * object from the local data in the partition. You can directly write the * traits interface yourself, but this is probably the easiest way to get * started (although potentially inefficient). */ MoabMesh::MoabMesh( const RCP_Comm& comm, const std::string& filename, const moab::EntityType& block_topology, const int partitioning_type ) : d_comm( comm ) { // Compute the node dimension. int node_dim = 0; if ( block_topology == moab::MBTRI ) { node_dim = 2; } else if ( block_topology == moab::MBQUAD ) { node_dim = 2; } else if ( block_topology == moab::MBTET ) { node_dim = 3; } else if ( block_topology == moab::MBHEX ) { node_dim = 3; } else if ( block_topology == moab::MBPYRAMID ) { node_dim = 3; } else { node_dim = 0; } // Create a moab instance. moab::ErrorCode error; d_moab = Teuchos::rcp( new moab::Core() ); std::cout<<"Filename: "<<filename<<std::endl; // Load the mesh. d_moab->load_mesh( &filename[0] ); moab::EntityHandle root_set = d_moab->get_root_set(); // Extract the elements with this block's topology. std::vector<moab::EntityHandle> global_elements; error = d_moab->get_entities_by_type( root_set, block_topology, global_elements ); assert( error == moab::MB_SUCCESS ); std::cout<<"Global elements: "<<global_elements.size()<<std::endl; // Partition the mesh. int comm_size = d_comm->getSize(); int comm_rank = d_comm->getRank(); // Get the number of nodes in an element. std::vector<moab::EntityHandle> elem_vertices; error = d_moab->get_adjacencies( &global_elements[0], 1, 0, false, elem_vertices ); assert( error == moab::MB_SUCCESS ); int nodes_per_element = elem_vertices.size(); // Get the global element coordinates. std::vector<double> global_coords; error = d_moab->get_vertex_coordinates( global_coords ); assert( error == moab::MB_SUCCESS ); // Get the global max and min values for the coordinates. This problem is // symmetric. double min = *(std::min_element( global_coords.begin(), global_coords.end() ) ); double max = *(std::max_element( global_coords.begin(), global_coords.end() ) ); double width = max - min; Teuchos::Array<moab::EntityHandle> elements; elem_vertices.resize( nodes_per_element ); std::vector<double> elem_coords( 3*nodes_per_element ); std::vector<moab::EntityHandle>::const_iterator global_elem_iterator; for ( global_elem_iterator = global_elements.begin(); global_elem_iterator != global_elements.end(); ++global_elem_iterator ) { // Get the individual element vertices. error = d_moab->get_adjacencies( &*global_elem_iterator, 1, 0, false, elem_vertices ); assert( error == moab::MB_SUCCESS ); // Get the invidivual element coordinates. error = d_moab->get_coords( &elem_vertices[0], elem_vertices.size(), &elem_coords[0] ); assert( error == moab::MB_SUCCESS ); // Partition in x direction. if ( partitioning_type == 0 ) { for ( int i = 0; i < comm_size; ++i ) { if ( elem_coords[0] >= min + width*(comm_rank)/comm_size - 1e-6 && elem_coords[0] <= min + width*(comm_rank+1)/comm_size + 1e-6 ) { elements.push_back( *global_elem_iterator ); } } } // Partition in y direction. else if ( partitioning_type == 1 ) { for ( int i = 0; i < comm_size; ++i ) { if ( elem_coords[1] >= min + width*(comm_rank)/comm_size - 1e-6 && elem_coords[1] <= min + width*(comm_rank+1)/comm_size + 1e-6 ) { elements.push_back( *global_elem_iterator ); } } } else { throw std::logic_error( "Partitioning type not supported." ); } } Teuchos::ArrayRCP<moab::EntityHandle> elements_arcp( elements.size() ); std::copy( elements.begin(), elements.end(), elements_arcp.begin() ); elements.clear(); d_comm->barrier(); // Get the nodes. std::vector<moab::EntityHandle> vertices; error = d_moab->get_connectivity( &elements_arcp[0], elements_arcp.size(), vertices ); assert( error == moab::MB_SUCCESS ); d_vertices = Teuchos::ArrayRCP<moab::EntityHandle>( vertices.size() ); std::copy( vertices.begin(), vertices.end(), d_vertices.begin() ); vertices.clear(); // Get the node coordinates. Teuchos::ArrayRCP<double> coords( node_dim * d_vertices.size() ); std::vector<double> interleaved_coords( 3*d_vertices.size() ); error = d_moab->get_coords( &d_vertices[0], d_vertices.size(), &interleaved_coords[0] ); assert( error == moab::MB_SUCCESS ); for ( int n = 0; n < (int) d_vertices.size(); ++n ) { for ( int d = 0; d < (int) node_dim; ++d ) { coords[ d*d_vertices.size() + n ] = interleaved_coords[ n*3 + d ]; } } interleaved_coords.clear(); // Get the connectivity. int connectivity_size = elements_arcp.size() * nodes_per_element; Teuchos::ArrayRCP<moab::EntityHandle> connectivity( connectivity_size ); std::vector<moab::EntityHandle> elem_conn; for ( int i = 0; i < (int) elements_arcp.size(); ++i ) { error = d_moab->get_connectivity( &elements_arcp[i], 1, elem_conn ); assert( error == moab::MB_SUCCESS ); assert( elem_conn.size() == Teuchos::as<std::vector<moab::EntityHandle>::size_type>(nodes_per_element) ); for ( int n = 0; n < (int) elem_conn.size(); ++n ) { connectivity[ n*elements_arcp.size() + i ] = elem_conn[n]; } } // Get the permutation vector. Teuchos::ArrayRCP<int> permutation_list( nodes_per_element ); for ( int i = 0; i < (int) nodes_per_element; ++i ) { permutation_list[i] = i; } // Create the mesh container. d_mesh_container = Teuchos::rcp( new Container( node_dim, d_vertices, coords, getTopology(block_topology), nodes_per_element, elements_arcp, connectivity, permutation_list ) ); }
DTKAdapter::DTKAdapter(Teuchos::RCP<const Teuchos::Comm<int> > in_comm, EquationSystems & in_es): comm(in_comm), es(in_es), mesh(in_es.get_mesh()), dim(mesh.mesh_dimension()) { std::set<unsigned int> semi_local_nodes; get_semi_local_nodes(semi_local_nodes); num_local_nodes = semi_local_nodes.size(); vertices.resize(num_local_nodes); Teuchos::ArrayRCP<double> coordinates(num_local_nodes * dim); // Fill in the vertices and coordinates { unsigned int i = 0; for(std::set<unsigned int>::iterator it = semi_local_nodes.begin(); it != semi_local_nodes.end(); ++it) { const Node & node = mesh.node(*it); vertices[i] = node.id(); for(unsigned int j=0; j<dim; j++) coordinates[(j*num_local_nodes) + i] = node(j); i++; } } // Currently assuming all elements are the same! DataTransferKit::DTK_ElementTopology element_topology = get_element_topology(mesh.elem(0)); unsigned int n_nodes_per_elem = mesh.elem(0)->n_nodes(); unsigned int n_local_elem = mesh.n_local_elem(); Teuchos::ArrayRCP<int> elements(n_local_elem); Teuchos::ArrayRCP<int> connectivity(n_nodes_per_elem*n_local_elem); // Fill in the elements and connectivity { unsigned int i = 0; MeshBase::const_element_iterator end = mesh.local_elements_end(); for(MeshBase::const_element_iterator it = mesh.local_elements_begin(); it != end; ++it) { const Elem & elem = *(*it); elements[i] = elem.id(); for(unsigned int j=0; j<n_nodes_per_elem; j++) connectivity[(j*n_local_elem)+i] = elem.node(j); i++; } } Teuchos::ArrayRCP<int> permutation_list(n_nodes_per_elem); for (unsigned int i = 0; i < n_nodes_per_elem; ++i ) permutation_list[i] = i; /* if(this->processor_id() == 1) sleep(1); libMesh::out<<"n_nodes_per_elem: "<<n_nodes_per_elem<<std::endl; libMesh::out<<"Dim: "<<dim<<std::endl; libMesh::err<<"Vertices size: "<<vertices.size()<<std::endl; { libMesh::err<<this->processor_id()<<" Vertices: "; for(unsigned int i=0; i<vertices.size(); i++) libMesh::err<<vertices[i]<<" "; libMesh::err<<std::endl; } libMesh::err<<"Coordinates size: "<<coordinates.size()<<std::endl; { libMesh::err<<this->processor_id()<<" Coordinates: "; for(unsigned int i=0; i<coordinates.size(); i++) libMesh::err<<coordinates[i]<<" "; libMesh::err<<std::endl; } libMesh::err<<"Connectivity size: "<<connectivity.size()<<std::endl; { libMesh::err<<this->processor_id()<<" Connectivity: "; for(unsigned int i=0; i<connectivity.size(); i++) libMesh::err<<connectivity[i]<<" "; libMesh::err<<std::endl; } libMesh::err<<"Permutation_List size: "<<permutation_list.size()<<std::endl; { libMesh::err<<this->processor_id()<<" Permutation_List: "; for(unsigned int i=0; i<permutation_list.size(); i++) libMesh::err<<permutation_list[i]<<" "; libMesh::err<<std::endl; } */ Teuchos::RCP<MeshContainerType> mesh_container = Teuchos::rcp( new MeshContainerType(dim, vertices, coordinates, element_topology, n_nodes_per_elem, elements, connectivity, permutation_list) ); // We only have 1 element topology in this grid so we make just one mesh block Teuchos::ArrayRCP<Teuchos::RCP<MeshContainerType> > mesh_blocks(1); mesh_blocks[0] = mesh_container; // Create the MeshManager mesh_manager = Teuchos::rcp(new DataTransferKit::MeshManager<MeshContainerType>(mesh_blocks, comm, dim) ); // Pack the coordinates into a field, this will be the positions we'll ask for other systems fields at target_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(mesh_container, comm)); }
/*! * \brief Get the wave mesh. */ Teuchos::RCP<DataTransferKit::MeshManager<WaveAdapter::MeshType> > WaveAdapter::getMesh( const RCP_Wave& wave ) { // Get the process rank. int my_rank = wave->get_comm()->getRank(); // Set the vertex dimension - this is a 1D problem. int vertex_dimension = 1; // Compute globally unique vertex ids. Teuchos::RCP<std::vector<double> > grid = wave->get_grid(); Teuchos::ArrayRCP<int> vertices( grid->size() ); for ( int i = 0; i < (int) vertices.size(); ++i ) { vertices[i] = i + my_rank*vertices.size(); } // Get the grid vertex coordinates. Teuchos::ArrayRCP<double> coordinates( &(*grid)[0], 0, grid->size(), false ); // Set the grid topology - this is 1D so we are using line segments. DataTransferKit::DTK_ElementTopology element_topology = DataTransferKit::DTK_LINE_SEGMENT; // Each line segment will be constructed by 2 vertices. int vertices_per_element = 2; // Compute globally unique element ids. Teuchos::ArrayRCP<int> elements( grid->size() - 1 ); for ( int i = 0; i < (int) elements.size(); ++i ) { elements[i] = i + my_rank*elements.size(); } // Generate element connectivity. The global vertex ids are used to // describe the construction of each line segment. Teuchos::ArrayRCP<int> connectivity( vertices_per_element*elements.size() ); for ( int i = 0; i < (int) elements.size(); ++i ) { connectivity[i] = vertices[i]; connectivity[ elements.size() + i ] = vertices[i+1]; } // Define the permutation list. Here our line segments are ordered the // same as DTK canonical ordering so the list is an monotonically // increasing set of integers. Teuchos::ArrayRCP<int> permutation_list( vertices_per_element ); for ( int i = 0; i < vertices_per_element; ++i ) { permutation_list[i] = i; } // Build a DTK Mesh container with the data. The MeshType typedef is set // in the header file. Teuchos::RCP<MeshType> mesh_container = Teuchos::rcp( new MeshType( vertex_dimension, vertices, coordinates, element_topology, vertices_per_element, elements, connectivity, permutation_list ) ); // We only have 1 element topology in this grid so we make just one mesh // block. Teuchos::ArrayRCP<Teuchos::RCP<MeshType> > mesh_blocks( 1 ); mesh_blocks[0] = mesh_container; // Return a mesh manager. return Teuchos::rcp( new DataTransferKit::MeshManager<MeshType>( mesh_blocks, wave->get_comm(), 1 ) ); }