DTKInterpolationAdapter::DTKInterpolationAdapter(Teuchos::RCP<const Teuchos::MpiComm<int> > in_comm, EquationSystems & in_es, const Point & offset, unsigned int from_dim):
comm(in_comm),
es(in_es),
_offset(offset),
mesh(in_es.get_mesh()),
dim(mesh.mesh_dimension())
{
MPI_Comm old_comm = Moose::swapLibMeshComm(*comm->getRawMpiComm());
std::set<GlobalOrdinal> semi_local_nodes;
get_semi_local_nodes(semi_local_nodes);
num_local_nodes = semi_local_nodes.size();
vertices.resize(num_local_nodes);
Teuchos::ArrayRCP<double> coordinates(num_local_nodes * dim);
Teuchos::ArrayRCP<double> target_coordinates(num_local_nodes * from_dim);
// Fill in the vertices and coordinates
{
GlobalOrdinal i = 0;
for (std::set<GlobalOrdinal>::iterator it = semi_local_nodes.begin();
it != semi_local_nodes.end();
++it)
{
const Node & node = mesh.node(*it);
vertices[i] = node.id();
for (GlobalOrdinal j=0; j<dim; j++)
coordinates[(j*num_local_nodes) + i] = node(j) + offset(j);
for (GlobalOrdinal j=0; j<from_dim; j++)
target_coordinates[(j*num_local_nodes) + i] = node(j) + offset(j);
i++;
}
}
// Currently assuming all elements are the same!
DataTransferKit::DTK_ElementTopology element_topology = get_element_topology(mesh.elem(0));
GlobalOrdinal n_nodes_per_elem = mesh.elem(0)->n_nodes();
GlobalOrdinal n_local_elem = mesh.n_local_elem();
elements.resize(n_local_elem);
Teuchos::ArrayRCP<GlobalOrdinal> connectivity(n_nodes_per_elem*n_local_elem);
Teuchos::ArrayRCP<double> elem_centroid_coordinates(n_local_elem*from_dim);
// Fill in the elements and connectivity
{
GlobalOrdinal i = 0;
MeshBase::const_element_iterator end = mesh.local_elements_end();
for (MeshBase::const_element_iterator it = mesh.local_elements_begin();
it != end;
++it)
{
const Elem & elem = *(*it);
elements[i] = elem.id();
for (GlobalOrdinal j=0; j<n_nodes_per_elem; j++)
connectivity[(j*n_local_elem)+i] = elem.node(j);
{
Point centroid = elem.centroid();
for (GlobalOrdinal j=0; j<from_dim; j++)
elem_centroid_coordinates[(j*n_local_elem) + i] = centroid(j) + offset(j);
}
i++;
}
}
Teuchos::ArrayRCP<int> permutation_list(n_nodes_per_elem);
for (GlobalOrdinal i = 0; i < n_nodes_per_elem; ++i )
permutation_list[i] = i;
/*
Moose::out<<"n_nodes_per_elem: "<<n_nodes_per_elem<<std::endl;
Moose::out<<"Dim: "<<dim<<std::endl;
Moose::err<<"Vertices size: "<<vertices.size()<<std::endl;
{
Moose::err<<libMesh::processor_id()<<" Vertices: ";
for (unsigned int i=0; i<vertices.size(); i++)
Moose::err<<vertices[i]<<" ";
Moose::err<<std::endl;
}
Moose::err<<"Coordinates size: "<<coordinates.size()<<std::endl;
{
Moose::err<<libMesh::processor_id()<<" Coordinates: ";
for (unsigned int i=0; i<coordinates.size(); i++)
Moose::err<<coordinates[i]<<" ";
Moose::err<<std::endl;
}
Moose::err<<"Connectivity size: "<<connectivity.size()<<std::endl;
{
Moose::err<<libMesh::processor_id()<<" Connectivity: ";
for (unsigned int i=0; i<connectivity.size(); i++)
Moose::err<<connectivity[i]<<" ";
Moose::err<<std::endl;
}
Moose::err<<"Permutation_List size: "<<permutation_list.size()<<std::endl;
{
Moose::err<<libMesh::processor_id()<<" Permutation_List: ";
for (unsigned int i=0; i<permutation_list.size(); i++)
Moose::err<<permutation_list[i]<<" ";
Moose::err<<std::endl;
}
*/
Teuchos::RCP<MeshContainerType> mesh_container = Teuchos::rcp(
new MeshContainerType(dim, vertices, coordinates,
element_topology, n_nodes_per_elem,
elements, connectivity, permutation_list) );
// We only have 1 element topology in this grid so we make just one mesh block
Teuchos::ArrayRCP<Teuchos::RCP<MeshContainerType> > mesh_blocks(1);
mesh_blocks[0] = mesh_container;
// Create the MeshManager
mesh_manager = Teuchos::rcp(new DataTransferKit::MeshManager<MeshContainerType>(mesh_blocks, comm, dim) );
// Pack the coordinates into a field, this will be the positions we'll ask for other systems fields at
if (from_dim == dim)
target_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(mesh_container, comm));
else
{
Teuchos::ArrayRCP<GlobalOrdinal> empty_elements(0);
Teuchos::ArrayRCP<GlobalOrdinal> empty_connectivity(0);
Teuchos::RCP<MeshContainerType> coords_only_mesh_container = Teuchos::rcp(
new MeshContainerType(from_dim, vertices, target_coordinates,
element_topology, n_nodes_per_elem,
empty_elements, empty_connectivity, permutation_list) );
target_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(coords_only_mesh_container, comm));
}
{
Teuchos::ArrayRCP<GlobalOrdinal> empty_elements(0);
Teuchos::ArrayRCP<GlobalOrdinal> empty_connectivity(0);
Teuchos::RCP<MeshContainerType> centroid_coords_only_mesh_container = Teuchos::rcp(
new MeshContainerType(from_dim, elements, elem_centroid_coordinates,
element_topology, n_nodes_per_elem,
empty_elements, empty_connectivity, permutation_list) );
elem_centroid_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(centroid_coords_only_mesh_container, comm));
}
// Swap back
Moose::swapLibMeshComm(old_comm);
}
/*
* \brief This constructor will pull the mesh data DTK needs out of Moab,
* partition it for the example, and build a DataTransferKit::MeshContainer
* object from the local data in the partition. You can directly write the
* traits interface yourself, but this is probably the easiest way to get
* started (although potentially inefficient).
*/
MoabMesh::MoabMesh( const RCP_Comm& comm,
const std::string& filename,
const moab::EntityType& block_topology,
const int partitioning_type )
: d_comm( comm )
{
// Compute the node dimension.
int node_dim = 0;
if ( block_topology == moab::MBTRI )
{
node_dim = 2;
}
else if ( block_topology == moab::MBQUAD )
{
node_dim = 2;
}
else if ( block_topology == moab::MBTET )
{
node_dim = 3;
}
else if ( block_topology == moab::MBHEX )
{
node_dim = 3;
}
else if ( block_topology == moab::MBPYRAMID )
{
node_dim = 3;
}
else
{
node_dim = 0;
}
// Create a moab instance.
moab::ErrorCode error;
d_moab = Teuchos::rcp( new moab::Core() );
std::cout<<"Filename: "<<filename<<std::endl;
// Load the mesh.
d_moab->load_mesh( &filename[0] );
moab::EntityHandle root_set = d_moab->get_root_set();
// Extract the elements with this block's topology.
std::vector<moab::EntityHandle> global_elements;
error = d_moab->get_entities_by_type(
root_set, block_topology, global_elements );
assert( error == moab::MB_SUCCESS );
std::cout<<"Global elements: "<<global_elements.size()<<std::endl;
// Partition the mesh.
int comm_size = d_comm->getSize();
int comm_rank = d_comm->getRank();
// Get the number of nodes in an element.
std::vector<moab::EntityHandle> elem_vertices;
error = d_moab->get_adjacencies( &global_elements[0],
1,
0,
false,
elem_vertices );
assert( error == moab::MB_SUCCESS );
int nodes_per_element = elem_vertices.size();
// Get the global element coordinates.
std::vector<double> global_coords;
error = d_moab->get_vertex_coordinates( global_coords );
assert( error == moab::MB_SUCCESS );
// Get the global max and min values for the coordinates. This problem is
// symmetric.
double min = *(std::min_element( global_coords.begin(),
global_coords.end() ) );
double max = *(std::max_element( global_coords.begin(),
global_coords.end() ) );
double width = max - min;
Teuchos::Array<moab::EntityHandle> elements;
elem_vertices.resize( nodes_per_element );
std::vector<double> elem_coords( 3*nodes_per_element );
std::vector<moab::EntityHandle>::const_iterator global_elem_iterator;
for ( global_elem_iterator = global_elements.begin();
global_elem_iterator != global_elements.end();
++global_elem_iterator )
{
// Get the individual element vertices.
error = d_moab->get_adjacencies( &*global_elem_iterator,
1,
0,
false,
elem_vertices );
assert( error == moab::MB_SUCCESS );
// Get the invidivual element coordinates.
error = d_moab->get_coords( &elem_vertices[0],
elem_vertices.size(),
&elem_coords[0] );
assert( error == moab::MB_SUCCESS );
// Partition in x direction.
if ( partitioning_type == 0 )
{
for ( int i = 0; i < comm_size; ++i )
{
if ( elem_coords[0] >= min + width*(comm_rank)/comm_size - 1e-6 &&
elem_coords[0] <= min + width*(comm_rank+1)/comm_size + 1e-6 )
{
elements.push_back( *global_elem_iterator );
}
}
}
// Partition in y direction.
else if ( partitioning_type == 1 )
{
for ( int i = 0; i < comm_size; ++i )
{
if ( elem_coords[1] >= min + width*(comm_rank)/comm_size - 1e-6 &&
elem_coords[1] <= min + width*(comm_rank+1)/comm_size + 1e-6 )
{
elements.push_back( *global_elem_iterator );
}
}
}
else
{
throw std::logic_error( "Partitioning type not supported." );
}
}
Teuchos::ArrayRCP<moab::EntityHandle> elements_arcp( elements.size() );
std::copy( elements.begin(), elements.end(), elements_arcp.begin() );
elements.clear();
d_comm->barrier();
// Get the nodes.
std::vector<moab::EntityHandle> vertices;
error = d_moab->get_connectivity( &elements_arcp[0],
elements_arcp.size(),
vertices );
assert( error == moab::MB_SUCCESS );
d_vertices = Teuchos::ArrayRCP<moab::EntityHandle>( vertices.size() );
std::copy( vertices.begin(), vertices.end(), d_vertices.begin() );
vertices.clear();
// Get the node coordinates.
Teuchos::ArrayRCP<double> coords( node_dim * d_vertices.size() );
std::vector<double> interleaved_coords( 3*d_vertices.size() );
error = d_moab->get_coords( &d_vertices[0], d_vertices.size(),
&interleaved_coords[0] );
assert( error == moab::MB_SUCCESS );
for ( int n = 0; n < (int) d_vertices.size(); ++n )
{
for ( int d = 0; d < (int) node_dim; ++d )
{
coords[ d*d_vertices.size() + n ] =
interleaved_coords[ n*3 + d ];
}
}
interleaved_coords.clear();
// Get the connectivity.
int connectivity_size = elements_arcp.size() * nodes_per_element;
Teuchos::ArrayRCP<moab::EntityHandle> connectivity( connectivity_size );
std::vector<moab::EntityHandle> elem_conn;
for ( int i = 0; i < (int) elements_arcp.size(); ++i )
{
error = d_moab->get_connectivity( &elements_arcp[i], 1, elem_conn );
assert( error == moab::MB_SUCCESS );
assert( elem_conn.size() ==
Teuchos::as<std::vector<moab::EntityHandle>::size_type>(nodes_per_element) );
for ( int n = 0; n < (int) elem_conn.size(); ++n )
{
connectivity[ n*elements_arcp.size() + i ] = elem_conn[n];
}
}
// Get the permutation vector.
Teuchos::ArrayRCP<int> permutation_list( nodes_per_element );
for ( int i = 0; i < (int) nodes_per_element; ++i )
{
permutation_list[i] = i;
}
// Create the mesh container.
d_mesh_container = Teuchos::rcp(
new Container( node_dim,
d_vertices,
coords,
getTopology(block_topology),
nodes_per_element,
elements_arcp,
connectivity,
permutation_list ) );
}
DTKAdapter::DTKAdapter(Teuchos::RCP<const Teuchos::Comm<int> > in_comm, EquationSystems & in_es):
comm(in_comm),
es(in_es),
mesh(in_es.get_mesh()),
dim(mesh.mesh_dimension())
{
std::set<unsigned int> semi_local_nodes;
get_semi_local_nodes(semi_local_nodes);
num_local_nodes = semi_local_nodes.size();
vertices.resize(num_local_nodes);
Teuchos::ArrayRCP<double> coordinates(num_local_nodes * dim);
// Fill in the vertices and coordinates
{
unsigned int i = 0;
for(std::set<unsigned int>::iterator it = semi_local_nodes.begin();
it != semi_local_nodes.end();
++it)
{
const Node & node = mesh.node(*it);
vertices[i] = node.id();
for(unsigned int j=0; j<dim; j++)
coordinates[(j*num_local_nodes) + i] = node(j);
i++;
}
}
// Currently assuming all elements are the same!
DataTransferKit::DTK_ElementTopology element_topology = get_element_topology(mesh.elem(0));
unsigned int n_nodes_per_elem = mesh.elem(0)->n_nodes();
unsigned int n_local_elem = mesh.n_local_elem();
Teuchos::ArrayRCP<int> elements(n_local_elem);
Teuchos::ArrayRCP<int> connectivity(n_nodes_per_elem*n_local_elem);
// Fill in the elements and connectivity
{
unsigned int i = 0;
MeshBase::const_element_iterator end = mesh.local_elements_end();
for(MeshBase::const_element_iterator it = mesh.local_elements_begin();
it != end;
++it)
{
const Elem & elem = *(*it);
elements[i] = elem.id();
for(unsigned int j=0; j<n_nodes_per_elem; j++)
connectivity[(j*n_local_elem)+i] = elem.node(j);
i++;
}
}
Teuchos::ArrayRCP<int> permutation_list(n_nodes_per_elem);
for (unsigned int i = 0; i < n_nodes_per_elem; ++i )
permutation_list[i] = i;
/*
if(this->processor_id() == 1)
sleep(1);
libMesh::out<<"n_nodes_per_elem: "<<n_nodes_per_elem<<std::endl;
libMesh::out<<"Dim: "<<dim<<std::endl;
libMesh::err<<"Vertices size: "<<vertices.size()<<std::endl;
{
libMesh::err<<this->processor_id()<<" Vertices: ";
for(unsigned int i=0; i<vertices.size(); i++)
libMesh::err<<vertices[i]<<" ";
libMesh::err<<std::endl;
}
libMesh::err<<"Coordinates size: "<<coordinates.size()<<std::endl;
{
libMesh::err<<this->processor_id()<<" Coordinates: ";
for(unsigned int i=0; i<coordinates.size(); i++)
libMesh::err<<coordinates[i]<<" ";
libMesh::err<<std::endl;
}
libMesh::err<<"Connectivity size: "<<connectivity.size()<<std::endl;
{
libMesh::err<<this->processor_id()<<" Connectivity: ";
for(unsigned int i=0; i<connectivity.size(); i++)
libMesh::err<<connectivity[i]<<" ";
libMesh::err<<std::endl;
}
libMesh::err<<"Permutation_List size: "<<permutation_list.size()<<std::endl;
{
libMesh::err<<this->processor_id()<<" Permutation_List: ";
for(unsigned int i=0; i<permutation_list.size(); i++)
libMesh::err<<permutation_list[i]<<" ";
libMesh::err<<std::endl;
}
*/
Teuchos::RCP<MeshContainerType> mesh_container = Teuchos::rcp(
new MeshContainerType(dim, vertices, coordinates,
element_topology, n_nodes_per_elem,
elements, connectivity, permutation_list) );
// We only have 1 element topology in this grid so we make just one mesh block
Teuchos::ArrayRCP<Teuchos::RCP<MeshContainerType> > mesh_blocks(1);
mesh_blocks[0] = mesh_container;
// Create the MeshManager
mesh_manager = Teuchos::rcp(new DataTransferKit::MeshManager<MeshContainerType>(mesh_blocks, comm, dim) );
// Pack the coordinates into a field, this will be the positions we'll ask for other systems fields at
target_coords = Teuchos::rcp(new DataTransferKit::FieldManager<MeshContainerType>(mesh_container, comm));
}
/*!
* \brief Get the wave mesh.
*/
Teuchos::RCP<DataTransferKit::MeshManager<WaveAdapter::MeshType> >
WaveAdapter::getMesh( const RCP_Wave& wave )
{
// Get the process rank.
int my_rank = wave->get_comm()->getRank();
// Set the vertex dimension - this is a 1D problem.
int vertex_dimension = 1;
// Compute globally unique vertex ids.
Teuchos::RCP<std::vector<double> > grid = wave->get_grid();
Teuchos::ArrayRCP<int> vertices( grid->size() );
for ( int i = 0; i < (int) vertices.size(); ++i )
{
vertices[i] = i + my_rank*vertices.size();
}
// Get the grid vertex coordinates.
Teuchos::ArrayRCP<double> coordinates( &(*grid)[0], 0, grid->size(), false );
// Set the grid topology - this is 1D so we are using line segments.
DataTransferKit::DTK_ElementTopology element_topology =
DataTransferKit::DTK_LINE_SEGMENT;
// Each line segment will be constructed by 2 vertices.
int vertices_per_element = 2;
// Compute globally unique element ids.
Teuchos::ArrayRCP<int> elements( grid->size() - 1 );
for ( int i = 0; i < (int) elements.size(); ++i )
{
elements[i] = i + my_rank*elements.size();
}
// Generate element connectivity. The global vertex ids are used to
// describe the construction of each line segment.
Teuchos::ArrayRCP<int> connectivity( vertices_per_element*elements.size() );
for ( int i = 0; i < (int) elements.size(); ++i )
{
connectivity[i] = vertices[i];
connectivity[ elements.size() + i ] = vertices[i+1];
}
// Define the permutation list. Here our line segments are ordered the
// same as DTK canonical ordering so the list is an monotonically
// increasing set of integers.
Teuchos::ArrayRCP<int> permutation_list( vertices_per_element );
for ( int i = 0; i < vertices_per_element; ++i )
{
permutation_list[i] = i;
}
// Build a DTK Mesh container with the data. The MeshType typedef is set
// in the header file.
Teuchos::RCP<MeshType> mesh_container = Teuchos::rcp(
new MeshType( vertex_dimension, vertices, coordinates,
element_topology, vertices_per_element,
elements, connectivity, permutation_list ) );
// We only have 1 element topology in this grid so we make just one mesh
// block.
Teuchos::ArrayRCP<Teuchos::RCP<MeshType> > mesh_blocks( 1 );
mesh_blocks[0] = mesh_container;
// Return a mesh manager.
return Teuchos::rcp( new DataTransferKit::MeshManager<MeshType>(
mesh_blocks, wave->get_comm(), 1 ) );
}
