bool use_case_3_driver( MPI_Comm comm ,
const std::string& mesh_options )
{
if ( 0 == stk::parallel_machine_rank( comm ) ) {
std::cout << "stk_linsys use case 3" << std::endl
<< " Number Processes = " << stk::parallel_machine_size( comm )
<< std::endl ;
}
//--------------------------------------------------------------------
{
//------------------------------------------------------------------
// Declare the mesh meta data: element blocks and associated fields
stk::mesh::fem::FEMMetaData fem_meta;
fem_meta.FEM_initialize(SpatialDim, stk::mesh::fem::entity_rank_names(SpatialDim) ) ;
stk::mesh::MetaData & mesh_meta_data = stk::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
const stk::mesh::EntityRank element_rank = fem_meta.element_rank();
//--------------------------------
// Element-block declarations typically occur when reading the
// mesh-file meta-data, and thus won't usually appear in application code.
// Declaring the element blocks and associating an element traits
// with each element block.
stk::mesh::Part & universal = fem_meta.universal_part();
stk::mesh::Part & block_hex = fem_meta.declare_part("block_1", element_rank);
stk::mesh::Part & block_quad_shell = fem_meta.declare_part("block_2", element_rank);
stk::mesh::fem::CellTopology hex_top(shards::getCellTopologyData<shards::Hexahedron<> >());
stk::mesh::fem::CellTopology qshell_top(shards::getCellTopologyData<shards::ShellQuadrilateral<> >());
stk::mesh::fem::set_cell_topology( block_hex, hex_top );
stk::mesh::fem::set_cell_topology( block_quad_shell, qshell_top );
//--------------------------------
// Declaring fields of specified types on all nodes:
VectorFieldType & coordinates_field =
stk::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "coordinates" ) ,
stk::mesh::fem::FEMMetaData::NODE_RANK , universal , SpatialDim );
VectorFieldType & displacements_field =
stk::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "displacements" ) ,
stk::mesh::fem::FEMMetaData::NODE_RANK , universal , SpatialDim );
//--------------------------------
// Put a scalar "pressure" field on all elements, just to use in demonstrating
// DOF mappings below:
// ScalarFieldType & pressure_field =
stk::mesh::put_field(
fem_meta.declare_field< ScalarFieldType >("pressure"),
element_rank, universal);
//--------------------------------
// rotation_field only exists on the shell-nodes:
VectorFieldType & rotation_field =
stk::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "rotation" ),
stk::mesh::fem::FEMMetaData::NODE_RANK , block_quad_shell , SpatialDim );
//--------------------------------
// Commit (finalize) the meta data. Is now ready to be used
// in the creation and management of mesh bulk data.
fem_meta.commit();
//------------------------------------------------------------------
// stk::mesh::BulkData bulk data conforming to the meta data.
stk::mesh::BulkData mesh_bulk_data( mesh_meta_data , comm );
// In a typical app, the mesh would be read from file at this point.
// But in this use-case, we generate the mesh and initialize
// field data to use-case defined values.
use_case_3_generate_mesh(
mesh_options ,
mesh_bulk_data ,
coordinates_field ,
block_hex ,
block_quad_shell );
use_case_3_initialize_data(
mesh_bulk_data ,
coordinates_field ,
displacements_field ,
rotation_field );
mesh_bulk_data.modification_end();
//set owner-processors to lowest-sharing (stk::mesh defaults to
//highest-sharing) If highest-sharing owns, then it isn't correct for the
//way the fei library sets ownership of shared nodes for vectors etc.
stk::mesh::set_owners<stk::mesh::LowestRankSharingProcOwns>( mesh_bulk_data );
//------------------------------------------------------------------
const unsigned myProc = mesh_bulk_data.parallel_rank();
stk::mesh::Selector select_owned = fem_meta.locally_owned_part();
fei::SharedPtr<fei::Factory> feifactory(new Factory_Trilinos(comm));
stk::linsys::LinearSystem ls(comm, feifactory);
if (myProc == 0) {
std::cout << "Adding element-node connectivities for displacements field for all locally-owned "
<< "elements..." << std::endl;
}
stk::linsys::add_connectivities(ls, element_rank,
stk::mesh::fem::FEMMetaData::NODE_RANK,
displacements_field, select_owned, mesh_bulk_data);
ls.synchronize_mappings_and_structure();
ls.create_fei_LinearSystem();
fei::SharedPtr<fei::MatrixGraph> matgraph = ls.get_fei_MatrixGraph();
fei::SharedPtr<fei::Matrix> matrix = ls.get_fei_LinearSystem()->getMatrix();
fei::SharedPtr<fei::Vector> rhs = ls.get_fei_LinearSystem()->getRHS();
{
const std::vector<stk::mesh::Bucket*>& mesh_buckets = mesh_bulk_data.buckets(element_rank);
std::vector<stk::mesh::Bucket*> part_buckets;
stk::mesh::get_buckets(select_owned, mesh_buckets, part_buckets);
stk::linsys::DofMapper& dof_mapper = ls.get_DofMapper();
int field_id = dof_mapper.get_field_id(displacements_field);
stk::mesh::Entity& first_entity = *(part_buckets[0]->begin());
stk::mesh::PairIterRelation rel = first_entity.relations(stk::mesh::fem::FEMMetaData::NODE_RANK);
int num_nodes_per_elem = rel.second - rel.first;
int pattern_id = matgraph->definePattern(num_nodes_per_elem, stk::mesh::fem::FEMMetaData::NODE_RANK, field_id);
std::vector<int> node_ids(num_nodes_per_elem);
const int field_size = dof_mapper.get_fei_VectorSpace()->getFieldSize(field_id);
const int matsize = num_nodes_per_elem*field_size*num_nodes_per_elem*field_size;
const int vecsize = num_nodes_per_elem*field_size;
std::vector<double> elem_matrix_1d(matsize, 0);
std::vector<double*> elem_matrix_2d(vecsize);
std::vector<double> elem_vector(vecsize, 0);
for(size_t i=0; i<elem_matrix_2d.size(); ++i) {
elem_matrix_2d[i] = &elem_matrix_1d[i*vecsize];
}
//fill the dummy elem-matrix that we will use below for every element:
for(size_t i=0; i<elem_matrix_2d.size(); ++i) {
double* row = elem_matrix_2d[i];
if (i>=1) row[i-1] = -1;
row[i] = 2;
if (i<elem_matrix_2d.size()-1) row[i+1] = -1;
elem_vector[i] = 1;
}
std::vector<int> eqn_indices(vecsize);
for(size_t i=0; i<part_buckets.size(); ++i) {
stk::mesh::Bucket::iterator
b_iter = part_buckets[i]->begin(),
b_end = part_buckets[i]->end();
for(; b_iter != b_end; ++b_iter) {
stk::mesh::Entity& elem = *b_iter;
rel = elem.relations(stk::mesh::fem::FEMMetaData::NODE_RANK);
for(int j=0; rel.first != rel.second; ++rel.first, ++j) {
node_ids[j] = rel.first->entity()->identifier();
}
matgraph->getPatternIndices(pattern_id, &node_ids[0], eqn_indices);
matrix->sumIn(vecsize, &eqn_indices[0], vecsize, &eqn_indices[0],
&elem_matrix_2d[0]);
rhs->sumIn(vecsize, &eqn_indices[0], &elem_vector[0]);
}
}
}
ls.finalize_assembly();
matrix->writeToFile("A.mtx");
rhs->writeToFile("rhs.vec");
}
return true;
}
bool use_case_5_driver( MPI_Comm comm ,
const std::string& mesh_options,
const std::string& solver_params )
{
if ( 0 == stk::parallel_machine_rank( comm ) ) {
std::cout << "stk_linsys use case 5" << std::endl
<< " Number Processes = " << stk::parallel_machine_size( comm )
<< std::endl ;
}
//--------------------------------------------------------------------
{
//------------------------------------------------------------------
// Declare the mesh meta data: element blocks and associated fields
stk::mesh::fem::FEMMetaData fem_meta(SpatialDim, stk::mesh::fem::entity_rank_names(SpatialDim) ) ;
Ioss::Init::Initializer init_db;
stk::mesh::MetaData & mesh_meta_data = stk::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
{
const stk::mesh::fem::FEMMetaData &fmd = fem_meta.get ( mesh_meta_data );
std::cout <<fmd.is_FEM_initialized()<<endl;
}
const stk::mesh::EntityRank element_rank = fem_meta.element_rank();
//--------------------------------
// Element-block declarations typically occur when reading the
// mesh-file meta-data, and thus won't usually appear in application code.
// Declaring the element blocks and associating an element traits
// with each element block.
stk::mesh::Part & universal = fem_meta.universal_part();
stk::mesh::Part & block_hex = fem_meta.declare_part("block_1", element_rank);
stk::mesh::Part & block_quad_shell = fem_meta.declare_part("block_2", element_rank);
stk::mesh::fem::CellTopology hex_top(shards::getCellTopologyData<shards::Hexahedron<> >());
stk::mesh::fem::CellTopology qshell_top(shards::getCellTopologyData<shards::ShellQuadrilateral<> >());
stk::mesh::fem::set_cell_topology( block_hex, hex_top );
stk::mesh::fem::set_cell_topology( block_quad_shell, qshell_top );
stk::io::put_io_part_attribute(block_hex);
stk::io::put_io_part_attribute(block_quad_shell);
//--------------------------------
// Declaring fields of specified types on all nodes:
VectorFieldType & coordinates_field =
stk::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "coordinates" ) ,
stk::mesh::fem::FEMMetaData::NODE_RANK , universal , SpatialDim );
VectorFieldType & displacements_field =
stk::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "displacements" ) ,
stk::mesh::fem::FEMMetaData::NODE_RANK , universal , SpatialDim );
//--------------------------------
// rotation_field only exists on the shell-nodes:
VectorFieldType & rotation_field =
stk::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "rotation" ),
stk::mesh::fem::FEMMetaData::NODE_RANK , block_quad_shell , SpatialDim );
stk::mesh::Part& bcpart = fem_meta.declare_part("bcpart");
// Define the transient fields that will be output.
stk::io::set_field_role(displacements_field, Ioss::Field::TRANSIENT);
//--------------------------------
// Commit (finalize) the meta data. Is now ready to be used
// in the creation and management of mesh bulk data.
fem_meta.commit();
//------------------------------------------------------------------
// stk::mesh::BulkData bulk data conforming to the meta data.
stk::mesh::BulkData mesh_bulk_data( mesh_meta_data , comm );
// In a typical app, the mesh would be read from file at this point.
// But in this use-case, we generate the mesh and initialize
// field data to use-case defined values.
use_case_5_generate_mesh(
mesh_options ,
mesh_bulk_data ,
coordinates_field ,
block_hex ,
block_quad_shell );
use_case_5_initialize_data(
mesh_bulk_data ,
coordinates_field ,
displacements_field ,
rotation_field );
//Add a node to our boundary-condition part 'bcpart'.
//let's choose the first locally-owned node. (This will produce a
//different boundary-condition for different numbers of processors...
//A more realistic case would simply pick a specific set of nodes
//regardless of which processors they are on.)
mesh_bulk_data.modification_begin();
std::vector<stk::mesh::Entity*> local_nodes;
stk::mesh::Selector select_owned(fem_meta.locally_owned_part());
stk::mesh::get_selected_entities(select_owned,
mesh_bulk_data.buckets(stk::mesh::fem::FEMMetaData::NODE_RANK),
local_nodes);
if (local_nodes.size() > 0) {
stk::mesh::PartVector partvector;
partvector.push_back(&bcpart);
mesh_bulk_data.change_entity_parts(*local_nodes[0], partvector);
}
mesh_bulk_data.modification_end();
//set owner-processors to lowest-sharing (stk::mesh defaults to
//highest-sharing) If highest-sharing owns, then it isn't correct for the
//way the fei library sets ownership of shared nodes for vectors etc.
stk::mesh::set_owners<stk::mesh::LowestRankSharingProcOwns>( mesh_bulk_data );
//Note: set_owners should throw an error if not done inside a modification_begin/end block.
//------------------------------------------------------------------
const unsigned myProc = mesh_bulk_data.parallel_rank();
//Now begin the use-case:
//Create a fei::Factory of type Factory_Trilinos, which will produce
//fei::Matrix and fei::Vector objects with run-time-type compatible with Trilinos.
fei::SharedPtr<fei::Factory> feifactory(new Factory_Trilinos(comm));
stk::linsys::LinearSystem ls(comm, feifactory);
if (myProc == 0) {
std::cout << "Adding element-node connectivities for displacements field for all locally-owned "
<< "elements..." << std::endl;
}
//Add connectivities for our mesh to the linsys::LinearSystem object. This
//will enable us to generate a matrix-graph:
stk::linsys::add_connectivities(ls, element_rank,
stk::mesh::fem::FEMMetaData::NODE_RANK,
displacements_field, select_owned, mesh_bulk_data);
ls.synchronize_mappings_and_structure();
ls.create_fei_LinearSystem();
fei::SharedPtr<fei::MatrixGraph> matgraph = ls.get_fei_MatrixGraph();
fei::SharedPtr<fei::Matrix> matrix = ls.get_fei_LinearSystem()->getMatrix();
fei::SharedPtr<fei::Vector> rhs = ls.get_fei_LinearSystem()->getRHS();
fei::SharedPtr<fei::Vector> solution = ls.get_fei_LinearSystem()->getSolutionVector();
//Now we'll run through the mesh and load up dense element-matrices and element-vectors
//to assemble into the global sparse linear-system:
{
const std::vector<stk::mesh::Bucket*>& mesh_buckets = mesh_bulk_data.buckets(element_rank);
std::vector<stk::mesh::Bucket*> part_buckets;
stk::mesh::get_buckets(select_owned, mesh_buckets, part_buckets);
stk::linsys::DofMapper& dof_mapper = ls.get_DofMapper();
int field_id = dof_mapper.get_field_id(displacements_field);
stk::mesh::Entity& first_entity = *(part_buckets[0]->begin());
stk::mesh::PairIterRelation rel = first_entity.relations(stk::mesh::fem::FEMMetaData::NODE_RANK);
int num_nodes_per_elem = rel.second - rel.first;
int pattern_id = matgraph->definePattern(num_nodes_per_elem, stk::mesh::fem::FEMMetaData::NODE_RANK, field_id);
std::vector<int> node_ids(num_nodes_per_elem);
const int field_size = dof_mapper.get_fei_VectorSpace()->getFieldSize(field_id);
const int matsize = num_nodes_per_elem*field_size*num_nodes_per_elem*field_size;
const int vecsize = num_nodes_per_elem*field_size;
std::vector<double> elem_matrix_1d(matsize, 0);
std::vector<double*> elem_matrix_2d(vecsize);
std::vector<double> elem_vector(vecsize, 0);
for(size_t i=0; i<elem_matrix_2d.size(); ++i) {
elem_matrix_2d[i] = &elem_matrix_1d[i*vecsize];
}
//fill our dummy elem-matrix:
//This dummy matrix will be the same for every element. A real application
//would form a different elem-matrix for each element.
for(size_t i=0; i<elem_matrix_2d.size(); ++i) {
double* row = elem_matrix_2d[i];
if (i>=1) row[i-1] = -1;
row[i] = 2;
if (i<elem_matrix_2d.size()-1) row[i+1] = -1;
elem_vector[i] = 1;
}
std::vector<int> eqn_indices(vecsize);
for(size_t i=0; i<part_buckets.size(); ++i) {
stk::mesh::Bucket::iterator
b_iter = part_buckets[i]->begin(),
b_end = part_buckets[i]->end();
for(; b_iter != b_end; ++b_iter) {
stk::mesh::Entity& elem = *b_iter;
rel = elem.relations(stk::mesh::fem::FEMMetaData::NODE_RANK);
for(int j=0; rel.first != rel.second; ++rel.first, ++j) {
node_ids[j] = rel.first->entity()->identifier();
}
matgraph->getPatternIndices(pattern_id, &node_ids[0], eqn_indices);
matrix->sumIn(vecsize, &eqn_indices[0], vecsize, &eqn_indices[0],
&elem_matrix_2d[0]);
rhs->sumIn(vecsize, &eqn_indices[0], &elem_vector[0]);
}
}
stk::linsys::dirichlet_bc(ls, mesh_bulk_data, bcpart, stk::mesh::fem::FEMMetaData::NODE_RANK,
displacements_field, 0, 3.14159265);
ls.finalize_assembly();
//Read solver-parameters out of a file. In a real application this would
//be done during a parsing phase, *not* here in the assembly code.
Teuchos::ParameterList params;
if (solver_params != "") {
Teuchos::ParameterXMLFileReader param_file(solver_params);
params = param_file.getParameters();
}
//Launch the linear-solver:
int status = 0, ret;
ret = ls.solve(status, params);
if (ret != 0) {
throw std::runtime_error("Error in the linear solver.");
}
//Copy the contents of the solution-vector back into our mesh-data:
copy_vector_to_mesh( *solution, dof_mapper, mesh_bulk_data);
}
//This following section writes mesh data out to an exodus file:
{
const std::string out_filename("mesh.e");
stk::io::MeshData mesh;
stk::io::create_output_mesh(out_filename, comm, mesh_bulk_data, mesh);
stk::io::define_output_fields(mesh, fem_meta);
// Write the model to the mesh file (topology, coordinates, attributes, etc)
stk::io::process_output_request(mesh, mesh_bulk_data, 0.0);
}
//Write out our assembled linear-system to files:
matrix->writeToFile("A.mtx");
rhs->writeToFile("rhs.vec");
solution->writeToFile("solution.vec");
}
return true;
}
bool use_case_1_driver( MPI_Comm comm ,
const std::string& mesh_options )
{
if ( 0 == stk_classic::parallel_machine_rank( comm ) ) {
std::cout << "stk_linsys use case 1" << std::endl
<< " Number Processes = " << stk_classic::parallel_machine_size( comm )
<< std::endl ;
}
//--------------------------------------------------------------------
{
//------------------------------------------------------------------
// Declare the mesh meta data: element blocks and associated fields
stk_classic::mesh::fem::FEMMetaData fem_meta;
fem_meta.FEM_initialize(SpatialDim, stk_classic::mesh::fem::entity_rank_names(SpatialDim) ) ;
stk_classic::mesh::MetaData & mesh_meta_data = stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
const stk_classic::mesh::EntityRank element_rank = fem_meta.element_rank();
//--------------------------------
// Element-block declarations typically occur when reading the
// mesh-file meta-data, and thus won't usually appear in application code.
// Declaring the element blocks and associating an element traits
// with each element block.
stk_classic::mesh::Part & universal = fem_meta.universal_part();
stk_classic::mesh::Part & block_hex = fem_meta.declare_part("block_1", element_rank);
stk_classic::mesh::Part & block_quad_shell = fem_meta.declare_part("block_2", element_rank);
stk_classic::mesh::fem::CellTopology hex_top(shards::getCellTopologyData<shards::Hexahedron<> >());
stk_classic::mesh::fem::CellTopology qshell_top(shards::getCellTopologyData<shards::ShellQuadrilateral<> >());
stk_classic::mesh::fem::set_cell_topology( block_hex, hex_top );
stk_classic::mesh::fem::set_cell_topology( block_quad_shell, qshell_top );
//--------------------------------
// Declaring fields of specified types on all nodes:
VectorFieldType & coordinates_field =
stk_classic::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "coordinates" ) ,
stk_classic::mesh::fem::FEMMetaData::NODE_RANK , universal , SpatialDim );
VectorFieldType & displacements_field =
stk_classic::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "displacements" ) ,
stk_classic::mesh::fem::FEMMetaData::NODE_RANK , universal , SpatialDim );
//--------------------------------
// Put a scalar "pressure" field on all elements, just to use in demonstrating
// DOF mappings below:
ScalarFieldType & pressure_field =
stk_classic::mesh::put_field(
fem_meta.declare_field< ScalarFieldType >("pressure"),
element_rank, universal);
//--------------------------------
// rotation_field only exists on the shell-nodes:
VectorFieldType & rotation_field =
stk_classic::mesh::put_field(
fem_meta.declare_field< VectorFieldType >( "rotation" ),
stk_classic::mesh::fem::FEMMetaData::NODE_RANK , block_quad_shell , SpatialDim );
//--------------------------------
// Commit (finalize) the meta data. Is now ready to be used
// in the creation and management of mesh bulk data.
fem_meta.commit();
//------------------------------------------------------------------
// stk_classic::mesh::BulkData bulk data conforming to the meta data.
stk_classic::mesh::BulkData mesh_bulk_data( mesh_meta_data , comm );
// In a typical app, the mesh would be read from file at this point.
// But in this use-case, we generate the mesh and initialize
// field data to use-case defined values.
use_case_1_generate_mesh(
mesh_options ,
mesh_bulk_data ,
coordinates_field ,
block_hex ,
block_quad_shell );
use_case_1_initialize_data(
mesh_bulk_data ,
coordinates_field ,
displacements_field ,
rotation_field );
mesh_bulk_data.modification_end();
//------------------------------------------------------------------
const unsigned myProc = mesh_bulk_data.parallel_rank();
stk_classic::linsys::DofMapper dof_mapper(comm);
if (myProc == 0) {
std::cout << "Adding DOF mappings for displacements field for all locally-used "
<< "(owned and shared) nodes..." << std::endl;
}
const stk_classic::mesh::Selector select_used =
fem_meta.locally_owned_part() |
fem_meta.globally_shared_part();
dof_mapper.add_dof_mappings(mesh_bulk_data, select_used,
stk_classic::mesh::fem::FEMMetaData::NODE_RANK, displacements_field);
if (myProc == 0) {
std::cout << "Adding DOF mappings for pressure field for all locally-owned "
<< " elements..." << std::endl;
}
stk_classic::mesh::Selector select_owned = fem_meta.locally_owned_part();
dof_mapper.add_dof_mappings(mesh_bulk_data, select_owned,
element_rank, pressure_field);
dof_mapper.finalize();
if (myProc == 0) {
std::cout << "Global Number of Indices: "
<< dof_mapper.get_fei_VectorSpace()->getGlobalNumIndices() << std::endl;
}
std::vector<stk_classic::mesh::Entity*> nodes;
stk_classic::mesh::get_entities(mesh_bulk_data, stk_classic::mesh::fem::FEMMetaData::NODE_RANK, nodes);
std::vector<stk_classic::mesh::Entity*> elems;
stk_classic::mesh::get_entities(mesh_bulk_data, element_rank, elems);
int global_index = 0;
for(size_t i=0; i<nodes.size(); i+=1000) {
//is the i-th node in the locally-used part? If not, continue.
if (! select_used( nodes[i]->bucket())) continue;
global_index = dof_mapper.get_global_index(stk_classic::mesh::fem::FEMMetaData::NODE_RANK, nodes[i]->identifier(), displacements_field);
std::cout << "Proc " << myProc << ", global index for node " << nodes[i]->identifier()
<< ", field '"<<displacements_field.name()<<"' is: " << global_index << std::endl;
}
for(size_t i=0; i<elems.size(); i+=1000) {
//is the i-th elem in the locally-owned part? If not, continue.
if (!elems[i]->bucket().member(fem_meta.locally_owned_part())) continue;
global_index = dof_mapper.get_global_index(element_rank, elems[i]->identifier(), pressure_field);
std::cout << "Proc " << myProc << ", global index for element " << elems[i]->identifier()
<< ", field '"<<pressure_field.name()<<"' is: " << global_index << std::endl;
}
if (mesh_options == "10x10x10+shell:y") {
return dof_mapper.get_fei_VectorSpace()->getGlobalNumIndices() == 5093;
}
return true;
}
}