Exemple #1
0
void HexMeshBuilder::remove_element(unsigned xCoord, unsigned yCoord, unsigned zCoord)
{
    stk::mesh::EntityId elemId = generate_three_dim_elem_id(xCoord, yCoord, zCoord);
    stk::mesh::Entity elem = bulk_data().get_entity(stk::topology::ELEM_RANK, elemId);
    if (bulk_data().is_valid(elem))
        bulk_data().destroy_entity(elem);
}
Exemple #2
0
void HexMeshBuilder::label_node_coordinates(const ElemCoordTriple& elemCoords)
{
    for (unsigned nodeIndex = 0; nodeIndex < 8; nodeIndex++)
    {
       stk::mesh::Entity node = bulk_data().get_entity(stk::topology::NODE_RANK,
                                                      elemCoords.nodeIds[nodeIndex]);
       switch (nodeIndex)
       {
           case 0:
              label_coordinates(node, elemCoords.x-1, elemCoords.y-1, elemCoords.z-1);
              break;
           case 1:
              label_coordinates(node, elemCoords.x, elemCoords.y-1, elemCoords.z-1);
              break;
           case 2:
              label_coordinates(node, elemCoords.x, elemCoords.y, elemCoords.z-1);
              break;
           case 3:
              label_coordinates(node, elemCoords.x-1, elemCoords.y, elemCoords.z-1);
              break;
           case 4:
              label_coordinates(node, elemCoords.x-1, elemCoords.y-1, elemCoords.z);
              break;
           case 5:
              label_coordinates(node, elemCoords.x, elemCoords.y-1, elemCoords.z);
              break;
           case 6:
              label_coordinates(node, elemCoords.x, elemCoords.y, elemCoords.z);
              break;
           case 7:
              label_coordinates(node, elemCoords.x-1, elemCoords.y, elemCoords.z);
              break;
       }
    }
}
Exemple #3
0
STKUNIT_UNIT_TEST(UnitTestField, testFieldWithSelectorAnd)
{
  stk_classic::ParallelMachine pm = MPI_COMM_SELF ;
  std::ostringstream oss; // to test printing of things w/out spamming cout

  typedef stk_classic::mesh::Field<double,shards::ArrayDimension>           rank_one_field ;
  // specifications for test field

  const std::string name0("test_field_0");

  const int spatial_dimension = 3;
  stk_classic::mesh::fem::FEMMetaData meta_data( spatial_dimension );
  stk_classic::mesh::BulkData bulk_data( stk_classic::mesh::fem::FEMMetaData::get_meta_data(meta_data) , pm );

  rank_one_field  & f0 = meta_data.declare_field< rank_one_field >( name0 );

  stk_classic::mesh::EntityRank elem_rank = meta_data.element_rank();
  stk_classic::mesh::Part & elements = meta_data.declare_part("Elements", elem_rank);
  stk_classic::mesh::Part & hex8s = meta_data.declare_part("Hex8", elem_rank );
  stk_classic::mesh::Part & tet4s = meta_data.declare_part("Tet4", elem_rank );

  stk_classic::mesh::Selector elem_hex_selector = elements & hex8s;
  stk_classic::mesh::Selector elem_tet_selector = elements & tet4s;
  std::cout <<"elem_hex_selector: "<< elem_hex_selector << std::endl;
  std::cout <<"elem_tet_selector: "<< elem_tet_selector << std::endl;

  stk_classic::mesh::put_field( f0 , elem_rank , elem_hex_selector, 8u );
  stk_classic::mesh::put_field( f0 , elem_rank , elem_tet_selector, 4u );

  stk_classic::mesh::print( oss , "  " , f0 );

  meta_data.commit();

  bulk_data.modification_begin();

  // Declare 10 elements on each part

  stk_classic::mesh::PartVector parts;
  parts.push_back(&elements);
  parts.push_back(&hex8s);

  for ( unsigned i = 1 ; i < 11 ; ++i ) {
    bulk_data.declare_entity( elem_rank , i , parts );
  }

  parts.clear();
  parts.push_back(&elements);
  parts.push_back(&tet4s);

  for ( unsigned i = 11 ; i < 21 ; ++i ) {
    bulk_data.declare_entity( elem_rank , i , parts );
  }

  stk_classic::mesh::BucketVector f0_buckets;
  stk_classic::mesh::get_buckets(elem_hex_selector, bulk_data.buckets(elem_rank), f0_buckets);

  BOOST_FOREACH(stk_classic::mesh::Bucket* b, f0_buckets) {
    unsigned f0_size = b->field_data_size(f0);
    STKUNIT_ASSERT_EQUAL(64u, f0_size);
  }
Exemple #4
0
void QuadMeshBuilder::label_node_coordinates(const ElemCoordPair& elemCoords)
{
    for (unsigned nodeIndex = 0; nodeIndex < 4; nodeIndex++)
    {
       stk::mesh::Entity node = bulk_data().get_entity(stk::topology::NODE_RANK,
                                                      elemCoords.nodeIds[nodeIndex]);
       if (0 == nodeIndex)
          label_coordinates(node, elemCoords.x-1, elemCoords.y-1);
       else if (1 == nodeIndex)
          label_coordinates(node, elemCoords.x, elemCoords.y-1);
       else if (2 == nodeIndex)
          label_coordinates(node, elemCoords.x, elemCoords.y);
       else if (3 == nodeIndex)
          label_coordinates(node, elemCoords.x-1, elemCoords.y);
    }
}
bool test_change_owner_3( stk_classic::ParallelMachine pm )
{
  const int p_rank = stk_classic::parallel_machine_rank( pm );
  const unsigned p_size = stk_classic::parallel_machine_size( pm );

  stk_classic::mesh::fem::FEMMetaData fem_meta_data( spatial_dimension );
  const stk_classic::mesh::EntityRank element_rank = fem_meta_data.element_rank();

  VectorField * coordinates_field =
    & put_field(
        fem_meta_data.declare_field<VectorField>("coordinates"),
        NODE_RANK,
        fem_meta_data.universal_part() ,
        3
        );

  stk_classic::mesh::Part & quad_part  = stk_classic::mesh::fem::declare_part<Quad4>( fem_meta_data, "quad");

  fem_meta_data.commit();

  stk_classic::mesh::BulkData bulk_data( stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta_data),
                                 pm,
                                 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk_classic::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk_classic::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk_classic::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk_classic::mesh::Entity * n = bulk_data.get_entity( NODE_RANK, nid );
        double * const coord = stk_classic::mesh::field_data( *coordinates_field , *n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size>1 ) {

    std::vector<stk_classic::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  2 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  6 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 10 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 2 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 5 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 1 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    // output to debug

    ep.clear();

    if ( p_rank==0 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  1 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  5 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 1 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 4 ), 1 ) );
    }
    else if ( p_rank==1 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 10 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 0 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 0 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 0 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();
  }

  return true ;
}
STKUNIT_UNIT_TEST(UnitTestLinsysFunctions, test1)
{
    static const size_t spatial_dimension = 3;

    MPI_Barrier( MPI_COMM_WORLD );
    MPI_Comm comm = MPI_COMM_WORLD;
    //First create and fill MetaData and BulkData objects:

    const unsigned bucket_size = 100; //for a real application mesh, bucket_size would be much bigger...

    stk::mesh::fem::FEMMetaData fem_meta;
    stk::mesh::fem::FEMMetaData fem_meta2;
    fem_meta.FEM_initialize(spatial_dimension);
    fem_meta2.FEM_initialize(spatial_dimension);

    stk::mesh::MetaData & meta_data = stk::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
    stk::mesh::MetaData & meta_data2 = stk::mesh::fem::FEMMetaData::get_meta_data(fem_meta2);

    const stk::mesh::EntityRank element_rank = fem_meta.element_rank();

    stk::mesh::BulkData bulk_data( meta_data, comm, bucket_size );
    stk::mesh::BulkData bulk_data2( meta_data2, comm, bucket_size );

    //create a boundary-condition part for testing later:
    stk::mesh::Part& bcpart = fem_meta.declare_part("bcpart");

    fill_utest_mesh_meta_data( fem_meta );

    bool use_temperature=false;
    fill_utest_mesh_meta_data( fem_meta2, use_temperature );

    fill_utest_mesh_bulk_data( bulk_data );
    fill_utest_mesh_bulk_data( bulk_data2 );

    //set owner-processors to lowest-sharing (stk::mesh defaults to
    //highest-sharing) If highest-sharing owns, then it isn't correct for the
    //way the fei library sets ownership of shared nodes for vectors etc.
    stk::mesh::set_owners<stk::mesh::LowestRankSharingProcOwns>( bulk_data );

    //put a node in our boundary-condition part. arbitrarily choose the
    //first locally-owned node:

    bulk_data.modification_begin();

    std::vector<stk::mesh::Entity*> local_nodes;
    stk::mesh::Selector select_owned(meta_data.locally_owned_part());
    stk::mesh::get_selected_entities(select_owned,
                                     bulk_data.buckets(NODE_RANK),
                                     local_nodes);

    stk::mesh::EntityId bc_node_id = 0;

    if (local_nodes.size() > 0) {
        stk::mesh::PartVector partvector;
        partvector.push_back(&bcpart);
        bulk_data.change_entity_parts(*local_nodes[0], partvector);
        bc_node_id = stk::linsys::impl::entityid_to_int(local_nodes[0]->identifier());
    }

    bulk_data.modification_end();

    stk::mesh::Selector selector = ( meta_data.locally_owned_part() | meta_data.globally_shared_part() ) & *meta_data.get_part("block_1");
    std::vector<unsigned> count;
    stk::mesh::count_entities(selector, bulk_data, count);

    STKUNIT_ASSERT_EQUAL( count[element_rank], (unsigned)4 );
    STKUNIT_ASSERT_EQUAL( count[NODE_RANK],     (unsigned)20 );

    ScalarField* temperature_field = meta_data.get_field<ScalarField>("temperature");

    //Create a fei Factory and stk::linsys::LinearSystem object:

    fei::SharedPtr<fei::Factory> factory(new Factory_Trilinos(comm));

    stk::linsys::LinearSystem ls(comm, factory);

    stk::linsys::add_connectivities(ls, element_rank, NODE_RANK,
                                    *temperature_field, selector, bulk_data);

    fei::SharedPtr<fei::MatrixGraph> matgraph = ls.get_fei_MatrixGraph();
    int num_blocks = matgraph->getNumConnectivityBlocks();

    STKUNIT_ASSERT_EQUAL( num_blocks, (int)1 );

    ls.synchronize_mappings_and_structure();
    ls.create_fei_LinearSystem();

    //put 0 throughout the matrix and 3 throughout the rhs:
    fei::SharedPtr<fei::Matrix> mat = ls.get_fei_LinearSystem()->getMatrix();
    ls.get_fei_LinearSystem()->getMatrix()->putScalar(0);
    ls.get_fei_LinearSystem()->getRHS()->putScalar(3.0);

    //put 10 on the matrix diagonal to ensure it will be easy to solve later.
    fei::SharedPtr<fei::VectorSpace> vspace = ls.get_fei_LinearSystem()->getRHS()->getVectorSpace();
    int numLocalRows = vspace->getNumIndices_Owned();
    std::vector<int> local_rows(numLocalRows);
    vspace->getIndices_Owned(numLocalRows, &local_rows[0], numLocalRows);

    for(size_t i=0; i<local_rows.size(); ++i) {
        int col = local_rows[i];
        double coef = 10;
        double* coefPtr = &coef;
        mat->sumIn(1, &local_rows[i], 1, &col, &coefPtr);
    }

    //now we'll impose a dirichlet bc on our one-node bcpart:
    stk::linsys::dirichlet_bc(ls, bulk_data, bcpart, NODE_RANK,
                              *temperature_field, 0, 9.0);

    ls.finalize_assembly();

    //now confirm that the rhs value for the equation corresponding to our
    //bc node is 9.0:

    fei::SharedPtr<fei::Vector> rhsvec = ls.get_fei_LinearSystem()->getRHS();
    double rhs_bc_val = 0;
    int bc_eqn_index = ls.get_DofMapper().get_global_index(NODE_RANK,
                       bc_node_id, *temperature_field);
    rhsvec->copyOut(1, &bc_eqn_index, &rhs_bc_val);

    bool bc_val_is_correct = std::abs(rhs_bc_val - 9.0) < 1.e-13;
    STKUNIT_ASSERT( bc_val_is_correct );

    stk::linsys::copy_vector_to_mesh( *rhsvec, ls.get_DofMapper(), bulk_data);

    stk::mesh::Entity* bc_node = bulk_data.get_entity(NODE_RANK, local_nodes[0]->identifier());

    stk::mesh::FieldTraits<ScalarField>::data_type* bc_node_data = stk::mesh::field_data(*temperature_field, *bc_node);

    bool bc_node_data_is_correct = std::abs(bc_node_data[0] - 9.0) < 1.e-13;
    STKUNIT_ASSERT( bc_node_data_is_correct );

    //now make sure we get a throw if we use the wrong bulk-data (that doesn't have the
    //temperature field defined)
    STKUNIT_ASSERT_THROW(stk::linsys::copy_vector_to_mesh( *rhsvec, ls.get_DofMapper(), bulk_data2), std::runtime_error);

    //obtain and zero the solution vector
    fei::SharedPtr<fei::Vector> solnvec = ls.get_fei_LinearSystem()->getSolutionVector();
    solnvec->putScalar(0);

    //copy the vector of zeros into the mesh:
    stk::linsys::copy_vector_to_mesh( *solnvec, ls.get_DofMapper(), bulk_data);

    //assert that our bc node's data is now zero.
    bc_node_data_is_correct = std::abs(bc_node_data[0] - 0) < 1.e-13;
    STKUNIT_ASSERT( bc_node_data_is_correct );

    //call the linear-system solve function.
    //(note that when we add options to the solve method, we'll need to enhance this
    //testing to exercise various specific solves.)
    Teuchos::ParameterList params;

    int status = 0;
    ls.solve(status, params);

    //copy the solution-vector into the mesh:
    stk::linsys::copy_vector_to_mesh( *solnvec, ls.get_DofMapper(), bulk_data);

    //now assert that the value 9 (bc value) produced by the solve is in this
    //node's data.
    //note that we use a loose tolerance, because the default solver tolerance
    //is (I think) only 1.e-6.
    bc_node_data_is_correct = std::abs(bc_node_data[0] - 9.0) < 1.e-6;
    STKUNIT_ASSERT( bc_node_data_is_correct );
    STKUNIT_ASSERT(bc_node_data_is_correct);
}
bool test_change_owner_with_constraint( stk_classic::ParallelMachine pm )
{
  bool success = true ;

  const unsigned p_rank = stk_classic::parallel_machine_rank( pm );
  const unsigned p_size = stk_classic::parallel_machine_size( pm );

  if ( p_size != 2 ) { return success ; }

  std::vector<std::string> rank_names = stk_classic::mesh::fem::entity_rank_names(spatial_dimension);
  const stk_classic::mesh::EntityRank constraint_rank = rank_names.size();
  rank_names.push_back("Constraint");
  stk_classic::mesh::fem::FEMMetaData fem_meta_data( spatial_dimension, rank_names );
  const stk_classic::mesh::EntityRank element_rank = fem_meta_data.element_rank();

  VectorField * coordinates_field =
    & put_field(
        fem_meta_data.declare_field<VectorField>("coordinates"),
        NODE_RANK,
        fem_meta_data.universal_part() ,
        3
        );

  stk_classic::mesh::Part & owned_part = fem_meta_data.locally_owned_part();
  stk_classic::mesh::Part & quad_part  = stk_classic::mesh::fem::declare_part<Quad4>( fem_meta_data, "quad");

  fem_meta_data.commit();

  stk_classic::mesh::BulkData bulk_data( stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta_data),
                                 pm,
                                 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk_classic::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk_classic::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk_classic::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk_classic::mesh::Entity * n = bulk_data.get_entity( NODE_RANK, nid );
        double * const coord = stk_classic::mesh::field_data( *coordinates_field , *n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size>1 )
  {
    std::vector<stk_classic::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 1 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    bulk_data.modification_begin();

    if ( p_rank==1 )
    {
      // create constraint

      stk_classic::mesh::Entity * n10 = bulk_data.get_entity( NODE_RANK, 10 );
      stk_classic::mesh::Entity * n11 = bulk_data.get_entity( NODE_RANK, 11 );
      stk_classic::mesh::Entity * n12 = bulk_data.get_entity( NODE_RANK, 12 );

      stk_classic::mesh::PartVector add;
      add.push_back( &owned_part );
      const stk_classic::mesh::EntityId c_entity_id = 1;
      stk_classic::mesh::Entity & c = bulk_data.declare_entity( constraint_rank, c_entity_id, add );
      bulk_data.declare_relation( c , *n10 , 0 );
      bulk_data.declare_relation( c , *n11 , 1 );
      bulk_data.declare_relation( c , *n12 , 2 );
    }

    bulk_data.modification_end();

    stk_classic::mesh::Entity * n10 = bulk_data.get_entity( NODE_RANK, 10 );

    if ( p_rank==0 or p_rank==1 )
    {
      ThrowErrorMsgIf( !stk_classic::mesh::in_shared( *n10 ), "NODE[10] not shared" );
    }

    bulk_data.modification_begin();

    if ( p_rank==1 )
    {
      // destroy constraint

      stk_classic::mesh::Entity * c1 = bulk_data.get_entity( constraint_rank, 1 );

      ThrowErrorMsgIf( !bulk_data.destroy_entity( c1 ),
                       "failed to destroy constraint" );
    }

    bulk_data.modification_end();

    if ( p_rank==0 or p_rank==1 )
    {
      ThrowErrorMsgIf( stk_classic::mesh::in_shared( *n10 ), "NODE[10] shared" );
    }
  }

  return success ;
}
/// This function shows the basic calls needed to perform definition
/// and input of the mesh model and definition and periodic output
/// of a results database. The function is given the mesh filename
/// and the output filename and goes through all steps of
/// associating the filename with an Ioss::DatabaseIO object of the
/// correct type ("exodusII" in this example); creating an
/// Ioss::Region and then defining an stk::mesh corresponding to
/// this mesh.  The function also provides an example of how
/// specific element blocks existing in the mesh database could be
/// omitted from the analysis model.
///
/// The example then shows how to define a results database
/// corresponding to the analysis model and periodically output the
/// results in an execute loop.
///
/// A true application would have to provide additional
/// functionality and robustness, but the example shows how the
/// basic functionality can be provided by an application.
///
/// Note that the paradigm illustrated here is different than the
/// mesh input and output paradigm provided in the current
/// framework.  In this case, the application is responsible for the
/// majority of the IO behavior and the toolkit only provides some
/// helper functions to bridge between the Ioss and the stk::mesh.
/// It is hoped that this paradigm will result in more functionality
/// for the application with less complication and overhead.
void io_example( const std::string& in_filename,
                 const std::string& out_filename,
                 const std::string& decomp_method)
{
    // Initialize IO system.  Registers all element types and storage
    // types and the exodusII default database type.
    Ioss::Init::Initializer init_db;

    std::cout << "========================================================================\n"
              << " Copy input mesh to output mesh.                                        \n"
              << "========================================================================\n";

    std::string dbtype("exodusII");
    Ioss::PropertyManager properties;
    if (!decomp_method.empty()) {
        properties.add(Ioss::Property("DECOMPOSITION_METHOD", Ioss::Utils::uppercase(decomp_method)));
    }
    Ioss::DatabaseIO *dbi = Ioss::IOFactory::create(dbtype, in_filename, Ioss::READ_MODEL,
                            MPI_COMM_WORLD, properties);
    if (dbi == NULL || !dbi->ok()) {
        std::cerr  << "ERROR: Could not open database '" << in_filename
                   << "' of type '" << dbtype << "'\n";
        std::exit(EXIT_FAILURE);
    }

    std::cout << "Reading input file:   " << in_filename << "\n";
    // NOTE: 'in_region' owns 'dbi' pointer at this time...
    Ioss::Region in_region(dbi, "input_model");

    // SUBSETTING PARSING/PREPROCESSING...
    // Just an example of how application could control whether an
    // entity is subsetted or not...


#if 0
    // Example command line in current code corresponding to behavior below:
    std::cout << "\nWhen processing file multi-block.g for use case 2, the blocks below will be omitted:\n";
    std::cout << "\tOMIT BLOCK Cblock Eblock I1 I2\n\n";
    Ioss::ElementBlock *eb = in_region.get_element_block("cblock");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));

    eb = in_region.get_element_block("eblock");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));

    eb = in_region.get_element_block("i1");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));

    eb = in_region.get_element_block("i2");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));
#endif

#if 0
    // Example for subsetting -- omit "odd" blocks
    if (entity->type() == Ioss::ELEMENTBLOCK) {
        int id = entity->get_property("id").get_int();
        if (id % 2) {
            entity->property_add(Ioss::Property(std::string("omitted"), 1));
            std::cout << "Skipping " << entity->type_string() << ": "  << entity->name() << "\n";
        }
    }
#endif

    //----------------------------------
    // Process Entity Types. Subsetting is possible.

    static size_t spatial_dimension = in_region.get_property("spatial_dimension").get_int();

    stk::mesh::MetaData fem_meta_data( spatial_dimension );
    process_elementblocks(in_region, fem_meta_data);
    process_nodeblocks(in_region,    fem_meta_data);
    process_sidesets(in_region,      fem_meta_data);
    process_nodesets(in_region,      fem_meta_data);

    //----------------------------------
    // Done populating meta data, commit and create bulk data
    fem_meta_data.commit();

    //----------------------------------
    // Process Bulkdata for all Entity Types. Subsetting is possible.
    stk::mesh::BulkData bulk_data(fem_meta_data, MPI_COMM_WORLD);

    bulk_data.modification_begin();
    process_elementblocks(in_region, bulk_data);
    process_nodeblocks(in_region,    bulk_data);
    process_sidesets(in_region,      bulk_data);
    process_nodesets(in_region,      bulk_data);
    bulk_data.modification_end();

    //----------------------------------
    // OUTPUT...Create the output "mesh" portion

    std::cout << "Creating output file: " << out_filename << "\n";
    Ioss::DatabaseIO *dbo = Ioss::IOFactory::create(dbtype, out_filename,
                            Ioss::WRITE_RESULTS,
                            MPI_COMM_WORLD);
    if (dbo == NULL || !dbo->ok()) {
        std::cerr << "ERROR: Could not open results database '" << out_filename
                  << "' of type '" << dbtype << "'\n";
        std::exit(EXIT_FAILURE);
    }

#if 0
    {
        // Code to test the remove_io_part_attribute functionality.
        // Hook this up to a command line option at some point to test nightly...
        const stk::mesh::PartVector & all_parts = fem_meta_data.get_parts();
        for ( stk::mesh::PartVector::const_iterator ip = all_parts.begin(); ip != all_parts.end(); ++ip ) {
            stk::mesh::Part * const part = *ip;
            const stk::mesh::EntityRank part_rank = part->primary_entity_rank();

            if (stk::io::is_part_io_part(*part) && part_rank == 2) {
                std::cout << "Removing part attribute from " << part->name() << "\n";
                stk::io::remove_io_part_attribute(*part);
            }
        }
    }
#endif

    // NOTE: 'out_region' owns 'dbo' pointer at this time...
    Ioss::Region out_region(dbo, "results_output");

    stk::io::define_output_db(out_region, bulk_data, &in_region);
    stk::io::write_output_db(out_region,  bulk_data);

    // ------------------------------------------------------------------------
    /** \todo REFACTOR A real app would register a subset of the
     * fields on the mesh database as fields that the app would want
     * read at one or all or specified steps.  In this example, all
     * fields existing on the input mesh database are defined on the
     * parts in the stk::mesh.
     *
     * The real app would also only register a subset of the stk::mesh
     * fields as output fields and would probably have a mapping from
     * the internally used name to some name picked by the user. In
     * this example, all Ioss::Field::TRANSIENT fields defined on the stk::mesh are
     * output to the results database and the internal stk::mesh field
     * name is used as the name on the database....
     */

    out_region.begin_mode(Ioss::STATE_DEFINE_TRANSIENT);

    // Special processing for nodeblock (all nodes in model)...
    stk::io::ioss_add_fields(fem_meta_data.universal_part(), stk::topology::NODE_RANK,
                             out_region.get_node_blocks()[0],
                             Ioss::Field::TRANSIENT);

    const stk::mesh::PartVector & all_parts = fem_meta_data.get_parts();
    for ( stk::mesh::PartVector::const_iterator
            ip = all_parts.begin(); ip != all_parts.end(); ++ip ) {

        stk::mesh::Part * const part = *ip;

        const stk::mesh::EntityRank part_rank = part->primary_entity_rank();

        // Check whether this part should be output to results database.
        if (stk::io::is_part_io_part(*part)) {
            // Get Ioss::GroupingEntity corresponding to this part...
            Ioss::GroupingEntity *entity = out_region.get_entity(part->name());
            if (entity != NULL) {
                if (entity->type() == Ioss::SIDESET) {
                    Ioss::SideSet *sset = dynamic_cast<Ioss::SideSet*>(entity);
                    assert(sset != NULL);
                    int block_count = sset->block_count();
                    for (int i=0; i < block_count; i++) {
                        Ioss::SideBlock *fb = sset->get_block(i);
                        stk::io::ioss_add_fields(*part, part_rank,
                                                 fb, Ioss::Field::TRANSIENT);
                    }
                } else {
                    stk::io::ioss_add_fields(*part, part_rank,
                                             entity, Ioss::Field::TRANSIENT);
                }
            } else {
                /// \todo IMPLEMENT handle error... Possibly an assert since
                /// I think the corresponding entity should always exist...
            }
        }
    }
    out_region.end_mode(Ioss::STATE_DEFINE_TRANSIENT);
    // ------------------------------------------------------------------------

    // Read and Write transient fields...
    out_region.begin_mode(Ioss::STATE_TRANSIENT);
    int timestep_count = in_region.get_property("state_count").get_int();
    for (int step = 1; step <= timestep_count; step++) {
        double time = in_region.get_state_time(step);

        // Read data from the io input mesh database into stk::mesh fields...
        process_input_request(in_region, bulk_data, step);

        // execute()

        // Write data from the stk::mesh fields out to the output database.a
        int out_step = out_region.add_state(time);
        process_output_request(out_region, bulk_data, out_step);
    }
    out_region.end_mode(Ioss::STATE_TRANSIENT);
}
STKUNIT_UNIT_TEST(UnitTestLinsysFunctions, test3)
{
    static const size_t spatial_dimension = 3;

    MPI_Barrier( MPI_COMM_WORLD );
    MPI_Comm comm = MPI_COMM_WORLD;
    //First create and fill MetaData and BulkData objects:

    const unsigned bucket_size = 100; //for a real application mesh, bucket_size would be much bigger...

    stk::mesh::fem::FEMMetaData fem_meta;
    fem_meta.FEM_initialize(spatial_dimension);
    stk::mesh::MetaData & meta_data = stk::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
    stk::mesh::BulkData bulk_data( meta_data, comm, bucket_size );

    fill_utest_mesh_meta_data( fem_meta );

    fill_utest_mesh_bulk_data( bulk_data );

    //set owner-processors to lowest-sharing (stk::mesh defaults to
    //highest-sharing) If highest-sharing owns, then it isn't correct for the
    //way the fei library sets ownership of shared nodes for vectors etc.
    stk::mesh::set_owners<stk::mesh::LowestRankSharingProcOwns>( bulk_data );

    stk::mesh::Selector selector = ( meta_data.locally_owned_part() | meta_data.globally_shared_part() ) & *meta_data.get_part("block_1");
    std::vector<unsigned> count;
    stk::mesh::count_entities(selector, bulk_data, count);
    const stk::mesh::EntityRank element_rank = fem_meta.element_rank();

    STKUNIT_ASSERT_EQUAL( count[element_rank], (unsigned)4 );
    STKUNIT_ASSERT_EQUAL( count[NODE_RANK],     (unsigned)20 );

    ScalarField* temperature_field = meta_data.get_field<ScalarField>("temperature");

    //Create a fei Factory and stk::linsys::LinearSystem object:

    fei::SharedPtr<fei::Factory> factory(new Factory_Trilinos(comm));

    stk::linsys::LinearSystem ls(comm, factory);

    stk::linsys::add_connectivities(ls, element_rank, NODE_RANK,
                                    *temperature_field, selector, bulk_data);

    fei::SharedPtr<fei::MatrixGraph> matgraph = ls.get_fei_MatrixGraph();
    int num_blocks = matgraph->getNumConnectivityBlocks();

    STKUNIT_ASSERT_EQUAL( num_blocks, (int)1 );

    ls.synchronize_mappings_and_structure();
    ls.create_fei_LinearSystem();

    //put 3 throughout the matrix and 3 throughout the rhs:
    fei::SharedPtr<fei::Matrix> mat = ls.get_fei_LinearSystem()->getMatrix();
    mat->putScalar(3.0);
    ls.get_fei_LinearSystem()->getRHS()->putScalar(3.0);

    fei::SharedPtr<fei::Vector> rhsvec = ls.get_fei_LinearSystem()->getRHS();

    stk::linsys::scale_vector(2, *rhsvec);
    stk::linsys::scale_matrix(2, *mat);

    //now the rhs and matrix contain 6.

    //create another matrix and vector:
    fei::SharedPtr<fei::Matrix> mat2 = factory->createMatrix(matgraph);
    fei::SharedPtr<fei::Vector> vec2 = factory->createVector(matgraph);
    mat2->putScalar(3.0);
    vec2->putScalar(3.0);

    //add 3*mat to mat2
    stk::linsys::add_matrix_to_matrix(3.0, *mat, *mat2);

    //confirm that mat2 contains 21:
    bool result = confirm_matrix_values(*mat2, 21);
    STKUNIT_ASSERT(result);

    //add 3*rhsvec to vec2:
    stk::linsys::add_vector_to_vector(3.0, *rhsvec, *vec2);

    //confirm that vec2 contains 21:
    result = confirm_vector_values(*vec2, 21);
    STKUNIT_ASSERT(result);
}
bool test_change_owner_with_constraint( stk::ParallelMachine pm )
{
  bool success = true ;

  const unsigned p_rank = stk::parallel_machine_rank( pm );
  const unsigned p_size = stk::parallel_machine_size( pm );

  if ( p_size != 2 ) { return success ; }


  unsigned spatial_dimension = 2;
  std::vector<std::string> rank_names = stk::mesh::TopologicalMetaData::entity_rank_names(spatial_dimension);
  const stk::mesh::EntityRank constraint_rank = rank_names.size();
  rank_names.push_back("Constraint");
  stk::mesh::MetaData meta_data( rank_names );
  stk::mesh::TopologicalMetaData top_data( meta_data, spatial_dimension );

  VectorField * coordinates_field = 
    & put_field( 
        meta_data.declare_field<VectorField>("coordinates"),
        top_data.node_rank, 
        meta_data.universal_part() , 
        3 
        );

  stk::mesh::Part & owned_part = meta_data.locally_owned_part();
  stk::mesh::Part & quad_part  = top_data.declare_part<shards::Quadrilateral<4> >( "quad" );

  meta_data.commit();

  stk::mesh::BulkData bulk_data( meta_data, pm, 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk::mesh::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk::mesh::Entity * n = bulk_data.get_entity( top_data.node_rank, nid );
        double * const coord = stk::mesh::field_data( *coordinates_field , *n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size>1 )
  {
    std::vector<stk::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  3 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  4 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  7 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  8 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 11 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 12 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 15 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 16 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 3 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 6 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 9 ), 1 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    bulk_data.modification_begin();

    if ( p_rank==1 )
    {
      // create constraint

      stk::mesh::Entity * n10 = bulk_data.get_entity( top_data.node_rank, 10 );
      stk::mesh::Entity * n11 = bulk_data.get_entity( top_data.node_rank, 11 );
      stk::mesh::Entity * n12 = bulk_data.get_entity( top_data.node_rank, 12 );

      stk::mesh::PartVector add;
      add.push_back( &owned_part );
      const stk::mesh::EntityId c_entity_id = 1;
      stk::mesh::Entity & c = bulk_data.declare_entity( constraint_rank, c_entity_id, add );
      bulk_data.declare_relation( c , *n10 , 0 );
      bulk_data.declare_relation( c , *n11 , 1 );
      bulk_data.declare_relation( c , *n12 , 2 );
    }

    bulk_data.modification_end();

    stk::mesh::Entity * n10 = bulk_data.get_entity( top_data.node_rank, 10 );

    if ( p_rank==0 or p_rank==1 )
    {
      if ( not stk::mesh::in_shared( *n10 ) )
        throw std::runtime_error( "NODE[10] not shared" );
    }

    bulk_data.modification_begin();

    if ( p_rank==1 )
    {
      // destroy constraint

      stk::mesh::Entity * c1 = bulk_data.get_entity( constraint_rank, 1 );

      if ( not bulk_data.destroy_entity( c1 ) )
      {
        throw std::runtime_error( "failed to destroy constraint" );
      }
    }

    bulk_data.modification_end();

    if ( p_rank==0 or p_rank==1 )
    {
      if ( stk::mesh::in_shared( *n10 ) )
        throw std::runtime_error( "NODE[10] shared" );
    }
  }

  return success ;
}
bool test_change_owner_3( stk::ParallelMachine pm )
{
  const int p_rank = stk::parallel_machine_rank( pm );
  const unsigned p_size = stk::parallel_machine_size( pm );

  const int spatial_dimension = 2;
  stk::mesh::MetaData meta_data( stk::mesh::TopologicalMetaData::entity_rank_names(spatial_dimension) );
  stk::mesh::TopologicalMetaData top_data( meta_data, spatial_dimension );

  VectorField * coordinates_field =
    & put_field( 
        meta_data.declare_field<VectorField>("coordinates"),
        top_data.node_rank, 
        meta_data.universal_part() , 
        3 
        );

  stk::mesh::Part & quad_part  = top_data.declare_part<shards::Quadrilateral<4> >( "quad" );

  meta_data.commit();

  stk::mesh::BulkData bulk_data( meta_data, pm, 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk::mesh::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk::mesh::Entity * n = bulk_data.get_entity( top_data.node_rank, nid );
        double * const coord = stk::mesh::field_data( *coordinates_field , *n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size>1 ) {

    std::vector<stk::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  2 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  3 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  4 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  6 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  7 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  8 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 10 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 11 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 12 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 14 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 15 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 16 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 2 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 5 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 8 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 3 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 6 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 9 ), 1 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    // output to debug

    ep.clear();

    if ( p_rank==0 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  1 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank,  5 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 1 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 4 ), 1 ) );
    }
    else if ( p_rank==1 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 10 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 11 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 12 ), 0 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 14 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 15 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.node_rank, 16 ), 0 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 8 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( top_data.element_rank, 9 ), 0 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();
  }

  return true ;
}
Exemple #12
0
void ThreeDimGameofLifeMesh::share_node_with_this_id_to_this_processor(unsigned nodeId,
        unsigned procNum)
{
    stk::mesh::Entity node = bulk_data()->get_entity(stk::topology::NODE_RANK, nodeId);
    bulk_data()->add_node_sharing(node, procNum);
}
bool test_change_owner_2( stk::ParallelMachine pm )
{
  const unsigned p_rank = stk::parallel_machine_rank( pm );
  const unsigned p_size = stk::parallel_machine_size( pm );

  if ( p_size != 3 ) { return true ; }

  stk::mesh::MetaData fem_meta_data( spatial_dimension );
  const stk::mesh::EntityRank element_rank = stk::topology::ELEMENT_RANK;

  VectorField * coordinates_field =
    & put_field(
        fem_meta_data.declare_field<VectorField>(stk::topology::NODE_RANK, "coordinates"),
        fem_meta_data.universal_part() ,
        3
        );

  stk::mesh::Part & quad_part  = fem_meta_data.declare_part_with_topology( "quad", stk::topology::QUAD_4);

  fem_meta_data.commit();

  stk::mesh::BulkData bulk_data( fem_meta_data, pm, 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk::mesh::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk::mesh::Entity n = bulk_data.get_entity( NODE_RANK, nid );
        double * const coord = stk::mesh::field_data( *coordinates_field , n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size==3 )
  {
    std::vector<stk::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  9 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 13 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 7 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 2 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 2 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    ep.clear();

    if ( p_rank==1 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  9 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 13 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 0 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 7 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 0 ) );
    }

    if ( p_rank==2 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 0 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 0 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();
  }

  return true ;
}
STKUNIT_UNIT_TEST(UnitTestZoltanSimple, testUnit)
{
#ifdef STK_HAS_MPI
  stk::ParallelMachine comm(MPI_COMM_WORLD);
#else
  stk::ParallelMachine comm(0);
#endif

  unsigned spatial_dimension = 2;
  std::vector<std::string> rank_names = stk::mesh::fem::entity_rank_names(spatial_dimension);
  const stk::mesh::EntityRank constraint_rank = rank_names.size();
  rank_names.push_back("Constraint");

  stk::mesh::fem::FEMMetaData fem_meta;
  fem_meta.FEM_initialize( spatial_dimension, rank_names );
  stk::mesh::MetaData & meta_data = stk::mesh::fem::FEMMetaData::get_meta_data(fem_meta);
  stk::mesh::BulkData bulk_data( meta_data , comm , 100 );
  const stk::mesh::EntityRank element_rank    = fem_meta.element_rank();

  stk::mesh::fem::CellTopology quad_top(shards::getCellTopologyData<shards::Quadrilateral<4> >());
  stk::mesh::Part & quad_part( fem_meta.declare_part("quad", quad_top ) );
  VectorField & coord_field( fem_meta.declare_field< VectorField >( "coordinates" ) );
  ScalarField & weight_field( fem_meta.declare_field< ScalarField >( "element_weights" ) );

  stk::mesh::put_field( coord_field , NODE_RANK , fem_meta.universal_part() );
  stk::mesh::put_field(weight_field , element_rank , fem_meta.universal_part() );

  fem_meta.commit();

  const unsigned p_size = bulk_data.parallel_size();
  const unsigned p_rank = bulk_data.parallel_rank();

  bulk_data.modification_begin();

  if ( p_rank == 0 ) {

    std::vector<std::vector<stk::mesh::Entity*> > quads(nx);
    for ( unsigned ix = 0 ; ix < nx ; ++ix ) quads[ix].resize(ny);

    const unsigned nnx = nx + 1 ;
    const unsigned nny = ny + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk::mesh::Entity &q = stk::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
        quads[ix][iy] = &q; 
      }
    }

    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk::mesh::Entity * e = bulk_data.get_entity( element_rank, elem );
        double * const e_weight = stk::mesh::field_data( weight_field , *e );
        *e_weight = 1.0;
      }
    }

    for ( unsigned iy = 0 ; iy <= ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix <= nx ; ++ix ) {
        stk::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk::mesh::Entity * n = bulk_data.get_entity( NODE_RANK, nid );
        double * const coord = stk::mesh::field_data( coord_field , *n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }

    {
      const unsigned iy_left  =  0; 
      const unsigned iy_right = ny; 
      stk::mesh::PartVector add(1, &fem_meta.locally_owned_part());
      for ( unsigned ix = 0 ; ix <= nx ; ++ix ) {
        stk::mesh::EntityId nid_left  = 1 + ix + iy_left  * nnx ;
        stk::mesh::EntityId nid_right = 1 + ix + iy_right * nnx ;
        stk::mesh::Entity * n_left  = bulk_data.get_entity( NODE_RANK, nid_left  );
        stk::mesh::Entity * n_right = bulk_data.get_entity( NODE_RANK, nid_right );
        const stk::mesh::EntityId constraint_entity_id =  1 + ix + nny * nnx;
        stk::mesh::Entity & c = bulk_data.declare_entity( constraint_rank, constraint_entity_id, add );
        bulk_data.declare_relation( c , *n_left  , 0 );
        bulk_data.declare_relation( c , *n_right , 1 );
      }
    }

  }

  // Only P0 has any nodes or elements
  if ( p_rank == 0 ) {
    STKUNIT_ASSERT( ! bulk_data.buckets( NODE_RANK ).empty() );
    STKUNIT_ASSERT( ! bulk_data.buckets( element_rank ).empty() );
  }
  else {
    STKUNIT_ASSERT( bulk_data.buckets( NODE_RANK ).empty() );
    STKUNIT_ASSERT( bulk_data.buckets( element_rank ).empty() );
  }


  bulk_data.modification_end();

  // create some sides and faces to rebalance.
  stk::mesh::PartVector add_parts;
  stk::mesh::create_adjacent_entities(bulk_data, add_parts);

  // Zoltan partition is specialized fomm a virtual base class, stk::rebalance::Partition.
  // Other specializations are possible.
  Teuchos::ParameterList emptyList;
  stk::rebalance::Zoltan zoltan_partition(comm, spatial_dimension, emptyList);

  {
    stk::mesh::Selector selector(fem_meta.universal_part());

    stk::rebalance::rebalance(bulk_data, selector, &coord_field, &weight_field, zoltan_partition);
  }

  const double imbalance_threshold = stk::rebalance::check_balance(bulk_data, &weight_field, element_rank);
  const bool do_rebal = 1.5 < imbalance_threshold;

  // Check that we satisfy our threshhold
  STKUNIT_ASSERT( !do_rebal );
  if( (2 == p_size) || (4 == p_size) )
  {
    STKUNIT_ASSERT_NEAR(imbalance_threshold, 1.0, 1.e-8);
  }
  else
  {
    STKUNIT_ASSERT_LE(imbalance_threshold, 1.5);
  }

  // And verify that all dependent entities are on the same proc as their parent element
  {
    stk::mesh::EntityVector entities;
    stk::mesh::Selector selector = fem_meta.locally_owned_part();

    get_selected_entities(selector, bulk_data.buckets(NODE_RANK), entities);
    bool result = stk::rebalance::verify_dependent_ownership(element_rank, entities);
    //get_selected_entities(selector, bulk_data.buckets(constraint_rank), entities);
    //result &= stk::rebalance::verify_dependent_ownership(element_rank, entities);
    STKUNIT_ASSERT( result );
  }
}
Exemple #15
0
bool use_case_blas_driver(MPI_Comm comm,
                        int num_threads,
                        int num_trials,
                        const std::string &working_directory,
                        const std::string &mesh_filename,
                        const std::string &mesh_type,
                        const std::string &thread_runner,
                        int bucket_size,
                        bool performance_test)
{
  bool output = !performance_test; // If running for performance measurements, turn off output

  if (stk::parallel_machine_rank(comm) == 0) {
    std::cout << " stk_mesh Use Case Blas - fill, axpby, dot, norm , begin" << std::endl ;
    std::cout << "Running '" << mesh_filename << "' case, num_trials = "
              << num_trials << std::endl;
  }


  const AlgorithmRunnerInterface* alg_runner = NULL ;
  if ( thread_runner.empty() ||
       thread_runner == std::string("NonThreaded") ) {
    alg_runner = stk::algorithm_runner_non_thread();
  }
  else if ( thread_runner == std::string("TPI") ) {
    alg_runner = stk::algorithm_runner_tpi(num_threads);
  }
  else if ( thread_runner == std::string("TBB") ) {
    alg_runner = stk::algorithm_runner_tbb(num_threads);
  }

  if (alg_runner != NULL) {
    if (stk::parallel_machine_rank(comm) == 0)
      std::cout << "Using " << thread_runner
                << " algorithm runner, num_threads = " << num_threads
                << std::endl;
  } else {
    std::cout << "ERROR, failed to obtain requested AlgorithmRunner '"
              << thread_runner << "'." << std::endl;
    return false;
  }

  //----------------------------------

  // Timing:
  //   [0] = stk::mesh::MetaData creation
  //   [1] = stk::mesh::BulkData creation
  //   [2] = Initialization
  //   [3] = fill and axpby
  //   [4] = dot and norm2

  double time_min[9] = { 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 };
  double time_max[9] = { 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 };
  double wtime = 0 ;

  //--------------------------------------------------------------------

  reset_malloc_stats();

  if ( 0 == stk::parallel_machine_rank( comm ) ) {
    std::cout << "stk_mesh performance use case BLAS" << std::endl
              << "  Number Processes = " << stk::parallel_machine_size( comm )
              << std::endl ;
    std::cout.flush();
  }

  //--------------------------------------------------------------------

  // Initialize IO system.  Registers all element types and storage
  // types and the exodusII default database type.
  Ioss::Init::Initializer init_db;

  {
    wtime = stk::wall_time();

    //------------------------------------------------------------------
    // Declare the mesh meta data: element blocks and associated fields

    stk::mesh::fem::FEMMetaData meta_data(  spatial_dimension );
    stk::io::MeshData mesh_data;
    std::string filename = working_directory + mesh_filename;
    stk::io::create_input_mesh(mesh_type, filename, comm,
			       meta_data, mesh_data);
    stk::io::define_input_fields(mesh_data, meta_data);

    Fields fields;
    use_case_14_declare_fields(fields, meta_data.get_meta_data(meta_data));

    //--------------------------------
    // Commit (finalize) the meta data.  Is now ready to be used
    // in the creation and management of mesh bulk data.

    meta_data.commit();

    //------------------------------------------------------------------

    time_max[0] = stk::wall_dtime( wtime );

    //------------------------------------------------------------------
    // stk::mesh::BulkData bulk data conforming to the meta data.
    stk::mesh::BulkData bulk_data(meta_data.get_meta_data(meta_data) , comm, bucket_size);
    stk::io::populate_bulk_data(bulk_data, mesh_data);

    //------------------------------------------------------------------
    // Create output mesh...  (input filename + ".out14")
    if (output) {
      filename = working_directory + mesh_filename + ".blas";
      stk::io::create_output_mesh(filename, comm, bulk_data, mesh_data);
      stk::io::define_output_fields(mesh_data, meta_data, true);
    }

    stk::app::use_case_14_initialize_nodal_data(bulk_data ,
                                                *fields.model_coordinates ,
                                                *fields.coordinates_field ,
                                                *fields.velocity_field,
                                                1.0 /*dt*/);

    time_max[1] = stk::wall_dtime( wtime );

    //------------------------------------------------------------------
    // Ready to run the algorithms:
    //------------------------------------------------------------------

    //------------------------------------------------------------------
    time_max[2] = stk::wall_dtime( wtime );
    //------------------------------------------------------------------

    wtime = stk::wall_time();

    double dot1 = 0;

    for(int n=0; n<num_trials; ++n) {
      //
      // Call BLAS algs.
      //

      wtime = stk::wall_time();

      fill( *alg_runner, bulk_data , stk::mesh::fem::FEMMetaData::NODE_RANK , *fields.velocity_field, 0.2 );

      fill( *alg_runner, bulk_data , stk::mesh::fem::FEMMetaData::NODE_RANK , *fields.fint_field, 1.0 );

      axpby( *alg_runner, bulk_data , stk::mesh::fem::FEMMetaData::NODE_RANK ,
             0.01, *fields.model_coordinates , 1.0 , *fields.coordinates_field );

      axpby( *alg_runner, bulk_data , stk::mesh::fem::FEMMetaData::NODE_RANK ,
             0.1, *fields.coordinates_field, 1.0 , *fields.velocity_field );

      time_max[3] += stk::wall_dtime( wtime );

      dot1 = dot( *alg_runner, bulk_data, stk::mesh::fem::FEMMetaData::NODE_RANK ,
                  *fields.velocity_field, *fields.coordinates_field );

      double dot2 = dot( *alg_runner, bulk_data, stk::mesh::fem::FEMMetaData::NODE_RANK,
                         *fields.velocity_field, *fields.fint_field );

      double norm_1 = norm2(*alg_runner, bulk_data, stk::mesh::fem::FEMMetaData::NODE_RANK, *fields.velocity_field );

      double norm_2 = norm2(*alg_runner, bulk_data, stk::mesh::fem::FEMMetaData::NODE_RANK, *fields.coordinates_field );

      if ( stk::parallel_machine_rank( comm ) == 0 ) {
        std::cout << "    " << dot1 << "  " << dot2 << "  " << norm_1 << "  " << norm_2 << std::endl;
      }

      time_max[4] += stk::wall_dtime( wtime );

      if (output) {
        stk::io::process_output_request(mesh_data, bulk_data, n);
      }

    }//end for(..num_trials...

    if ( stk::parallel_machine_rank( comm ) == 0 ) {
      //Try to make sure the number gets printed out just the way we want it,
      //so we can use it as a pass/fail check for a regression test...
      std::cout.precision(6);
      std::cout.setf(std::ios_base::scientific, std::ios_base::floatfield);
      std::cout << "Final dot1: " << dot1 << std::endl;
    }
    //------------------------------------------------------------------

#ifdef USE_GNU_MALLOC_HOOKS
    if (parallel_machine_rank(comm) == 0) {
      double net_alloc = alloc_MB() - freed_MB();
      std::cout << "Mesh creation:" << "\n   Total allocated: "
                << alloc_MB()<<"MB in "<<alloc_blks() << " blocks."
                << "\n   Total freed: " << freed_MB() << "MB in "
                << freed_blks() << " blocks."
                << "\n   Net allocated: "<<net_alloc << "MB."<<std::endl;
    }
#endif

    //------------------------------------------------------------------
  }

  time_max[8] = stk::wall_dtime( wtime );

  time_min[0] = time_max[0] ;
  time_min[1] = time_max[1] ;
  time_min[2] = time_max[2] ;
  time_min[3] = time_max[3] ;
  time_min[4] = time_max[4] ;
  time_min[5] = time_max[5] ;
  time_min[6] = time_max[6] ;
  time_min[7] = time_max[7] ;
  time_min[8] = time_max[8] ;

  stk::all_reduce( comm , stk::ReduceMax<9>( time_max ) & stk::ReduceMin<9>( time_min ) );

  time_max[3] /= num_trials ;
  time_max[4] /= num_trials ;
  time_max[5] /= num_trials ;
  time_max[6] /= num_trials ;

  time_min[3] /= num_trials ;
  time_min[4] /= num_trials ;
  time_min[5] /= num_trials ;
  time_min[6] /= num_trials ;

  //   [0] = stk::mesh::MetaData creation
  //   [1] = stk::mesh::BulkData creation
  //   [2] = Initialization
  //   [3] = Internal force

  if ( ! stk::parallel_machine_rank( comm ) ) {
    std::cout
      << "stk_mesh performance use case results:" << std::endl
      << "  Number of trials         = " << num_trials << std::endl
      << "  Meta-data setup          = " << time_min[0] << " : "
      << time_max[0] << " sec, min : max"
      << std::endl
      << "  Bulk-data generation     = " << time_min[1] << " : "
      << time_max[1] << " sec, min : max"
      << std::endl
      << "  Initialization           = " << time_min[2] << " : "
      << time_max[2] << " sec, min : max"
      << std::endl
      << "  fill & axpby (per-trial) = " << time_min[3] << " : "
      << time_max[3] << " sec, min : max"
      << std::endl
      << "  dot & norm2 (per-trial)  = " << time_min[4] << " : "
      << time_max[4] << " sec, min : max"
      << std::endl
      << "  Mesh destruction         = " << time_min[8] << " : "
      << time_max[8] << " sec, min : max"
      << std::endl
      << std::endl ;
  }

  return true;
}
void testDofMapper( MPI_Comm comm )
{
  //First create and fill MetaData and BulkData objects:

  const unsigned bucket_size = 100; //for a real application mesh, bucket_size would be much bigger...

  stk::mesh::MetaData meta_data( stk::mesh::fem_entity_rank_names() );
  stk::mesh::BulkData bulk_data( meta_data, comm, bucket_size );

  fill_utest_mesh_meta_data( meta_data );
  fill_utest_mesh_bulk_data( bulk_data );

  stk::mesh::Selector selector = meta_data.locally_owned_part() | meta_data.globally_shared_part() ;
  std::vector<unsigned> count;
  stk::mesh::count_entities(selector, bulk_data, count);

  STKUNIT_ASSERT_EQUAL( count[stk::mesh::Element], (unsigned)4 );
  STKUNIT_ASSERT_EQUAL( count[stk::mesh::Node],    (unsigned)20 );

  std::vector<stk::mesh::Entity*> nodes;
  stk::mesh::get_entities(bulk_data, stk::mesh::Node, nodes);

  stk::mesh::ScalarField* temperature_field =
      meta_data.get_field<stk::mesh::ScalarField>("temperature");

  //Now we're ready to test the DofMapper:

  stk::linsys::DofMapper dof_mapper(comm);

  const stk::mesh::Selector select_used = meta_data.locally_owned_part() | meta_data.globally_shared_part();

  dof_mapper.add_dof_mappings(bulk_data, select_used,
                              stk::mesh::Node, *temperature_field);

  stk::mesh::EntityRank ent_type;
  stk::mesh::EntityId ent_id;
  const stk::mesh::FieldBase* field = NULL;
  int offset_into_field;
  int index = 0;
  //DofMapper::get_dof can't be called until after DofMapper::finalize() has
  //been called.
  //We'll call it now to verify that an exception is thrown:
  std::cout << "Testing error condition: " << std::endl;
  STKUNIT_ASSERT_THROW(dof_mapper.get_dof(index, ent_type, ent_id, field, offset_into_field), std::runtime_error );
  std::cout << "...Completed testing error condition." << std::endl;

  dof_mapper.finalize();

  //find a node that is in the locally-used part:
  size_t i_node = 0;
  while(! select_used( nodes[i_node]->bucket() ) && i_node<nodes.size()) {
    ++i_node;
  }

  //test the get_global_index function:
  stk::mesh::EntityId node_id = nodes[i_node]->identifier();
  index = dof_mapper.get_global_index(stk::mesh::Node, node_id, *temperature_field);
  STKUNIT_ASSERT_EQUAL( index, (int)(node_id-1) );

  std::cout << "Testing error condition: " << std::endl;
  //call DofMapper::get_global_index with a non-existent ID and verify that an
  //exception is thrown:
  STKUNIT_ASSERT_THROW(dof_mapper.get_global_index(stk::mesh::Node, (stk::mesh::EntityId)999999, *temperature_field), std::runtime_error);
  std::cout << "...Completed testing error condition." << std::endl;

  int numProcs = 1;
  numProcs = stk::parallel_machine_size( MPI_COMM_WORLD );

  fei::SharedPtr<fei::VectorSpace> fei_vspace = dof_mapper.get_fei_VectorSpace();
  int numIndices = fei_vspace->getGlobalNumIndices();
  STKUNIT_ASSERT_EQUAL( numIndices, (int)(numProcs*20 - (numProcs-1)*4) );

  dof_mapper.get_dof(index, ent_type, ent_id, field, offset_into_field);

  STKUNIT_ASSERT_EQUAL( ent_type, nodes[i_node]->entity_rank() );
  STKUNIT_ASSERT_EQUAL( ent_id,   nodes[i_node]->identifier() );
  STKUNIT_ASSERT_EQUAL( field->name() == temperature_field->name(), true );
  STKUNIT_ASSERT_EQUAL( offset_into_field, (int)0 );
}