Example #1
0
bool test_change_owner_3( stk_classic::ParallelMachine pm )
{
  const int p_rank = stk_classic::parallel_machine_rank( pm );
  const unsigned p_size = stk_classic::parallel_machine_size( pm );

  stk_classic::mesh::fem::FEMMetaData fem_meta_data( spatial_dimension );
  const stk_classic::mesh::EntityRank element_rank = fem_meta_data.element_rank();

  VectorField * coordinates_field =
    & put_field(
        fem_meta_data.declare_field<VectorField>("coordinates"),
        NODE_RANK,
        fem_meta_data.universal_part() ,
        3
        );

  stk_classic::mesh::Part & quad_part  = stk_classic::mesh::fem::declare_part<Quad4>( fem_meta_data, "quad");

  fem_meta_data.commit();

  stk_classic::mesh::BulkData bulk_data( stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta_data),
                                 pm,
                                 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk_classic::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk_classic::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk_classic::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk_classic::mesh::Entity * n = bulk_data.get_entity( NODE_RANK, nid );
        double * const coord = stk_classic::mesh::field_data( *coordinates_field , *n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size>1 ) {

    std::vector<stk_classic::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  2 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  6 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 10 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 2 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 5 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 1 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    // output to debug

    ep.clear();

    if ( p_rank==0 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  1 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  5 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 1 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 4 ), 1 ) );
    }
    else if ( p_rank==1 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 10 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 0 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 0 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 0 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 0 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();
  }

  return true ;
}
/// This function shows the basic calls needed to perform definition
/// and input of the mesh model and definition and periodic output
/// of a results database. The function is given the mesh filename
/// and the output filename and goes through all steps of
/// associating the filename with an Ioss::DatabaseIO object of the
/// correct type ("exodusII" in this example); creating an
/// Ioss::Region and then defining an stk::mesh corresponding to
/// this mesh.  The function also provides an example of how
/// specific element blocks existing in the mesh database could be
/// omitted from the analysis model.
///
/// The example then shows how to define a results database
/// corresponding to the analysis model and periodically output the
/// results in an execute loop.
///
/// A true application would have to provide additional
/// functionality and robustness, but the example shows how the
/// basic functionality can be provided by an application.
///
/// Note that the paradigm illustrated here is different than the
/// mesh input and output paradigm provided in the current
/// framework.  In this case, the application is responsible for the
/// majority of the IO behavior and the toolkit only provides some
/// helper functions to bridge between the Ioss and the stk::mesh.
/// It is hoped that this paradigm will result in more functionality
/// for the application with less complication and overhead.
void io_example( const std::string& in_filename,
                 const std::string& out_filename,
                 const std::string& decomp_method)
{
    // Initialize IO system.  Registers all element types and storage
    // types and the exodusII default database type.
    Ioss::Init::Initializer init_db;

    std::cout << "========================================================================\n"
              << " Copy input mesh to output mesh.                                        \n"
              << "========================================================================\n";

    std::string dbtype("exodusII");
    Ioss::PropertyManager properties;
    if (!decomp_method.empty()) {
        properties.add(Ioss::Property("DECOMPOSITION_METHOD", Ioss::Utils::uppercase(decomp_method)));
    }
    Ioss::DatabaseIO *dbi = Ioss::IOFactory::create(dbtype, in_filename, Ioss::READ_MODEL,
                            MPI_COMM_WORLD, properties);
    if (dbi == NULL || !dbi->ok()) {
        std::cerr  << "ERROR: Could not open database '" << in_filename
                   << "' of type '" << dbtype << "'\n";
        std::exit(EXIT_FAILURE);
    }

    std::cout << "Reading input file:   " << in_filename << "\n";
    // NOTE: 'in_region' owns 'dbi' pointer at this time...
    Ioss::Region in_region(dbi, "input_model");

    // SUBSETTING PARSING/PREPROCESSING...
    // Just an example of how application could control whether an
    // entity is subsetted or not...


#if 0
    // Example command line in current code corresponding to behavior below:
    std::cout << "\nWhen processing file multi-block.g for use case 2, the blocks below will be omitted:\n";
    std::cout << "\tOMIT BLOCK Cblock Eblock I1 I2\n\n";
    Ioss::ElementBlock *eb = in_region.get_element_block("cblock");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));

    eb = in_region.get_element_block("eblock");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));

    eb = in_region.get_element_block("i1");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));

    eb = in_region.get_element_block("i2");
    if (eb != NULL)
        eb->property_add(Ioss::Property(std::string("omitted"), 1));
#endif

#if 0
    // Example for subsetting -- omit "odd" blocks
    if (entity->type() == Ioss::ELEMENTBLOCK) {
        int id = entity->get_property("id").get_int();
        if (id % 2) {
            entity->property_add(Ioss::Property(std::string("omitted"), 1));
            std::cout << "Skipping " << entity->type_string() << ": "  << entity->name() << "\n";
        }
    }
#endif

    //----------------------------------
    // Process Entity Types. Subsetting is possible.

    static size_t spatial_dimension = in_region.get_property("spatial_dimension").get_int();

    stk::mesh::MetaData fem_meta_data( spatial_dimension );
    process_elementblocks(in_region, fem_meta_data);
    process_nodeblocks(in_region,    fem_meta_data);
    process_sidesets(in_region,      fem_meta_data);
    process_nodesets(in_region,      fem_meta_data);

    //----------------------------------
    // Done populating meta data, commit and create bulk data
    fem_meta_data.commit();

    //----------------------------------
    // Process Bulkdata for all Entity Types. Subsetting is possible.
    stk::mesh::BulkData bulk_data(fem_meta_data, MPI_COMM_WORLD);

    bulk_data.modification_begin();
    process_elementblocks(in_region, bulk_data);
    process_nodeblocks(in_region,    bulk_data);
    process_sidesets(in_region,      bulk_data);
    process_nodesets(in_region,      bulk_data);
    bulk_data.modification_end();

    //----------------------------------
    // OUTPUT...Create the output "mesh" portion

    std::cout << "Creating output file: " << out_filename << "\n";
    Ioss::DatabaseIO *dbo = Ioss::IOFactory::create(dbtype, out_filename,
                            Ioss::WRITE_RESULTS,
                            MPI_COMM_WORLD);
    if (dbo == NULL || !dbo->ok()) {
        std::cerr << "ERROR: Could not open results database '" << out_filename
                  << "' of type '" << dbtype << "'\n";
        std::exit(EXIT_FAILURE);
    }

#if 0
    {
        // Code to test the remove_io_part_attribute functionality.
        // Hook this up to a command line option at some point to test nightly...
        const stk::mesh::PartVector & all_parts = fem_meta_data.get_parts();
        for ( stk::mesh::PartVector::const_iterator ip = all_parts.begin(); ip != all_parts.end(); ++ip ) {
            stk::mesh::Part * const part = *ip;
            const stk::mesh::EntityRank part_rank = part->primary_entity_rank();

            if (stk::io::is_part_io_part(*part) && part_rank == 2) {
                std::cout << "Removing part attribute from " << part->name() << "\n";
                stk::io::remove_io_part_attribute(*part);
            }
        }
    }
#endif

    // NOTE: 'out_region' owns 'dbo' pointer at this time...
    Ioss::Region out_region(dbo, "results_output");

    stk::io::define_output_db(out_region, bulk_data, &in_region);
    stk::io::write_output_db(out_region,  bulk_data);

    // ------------------------------------------------------------------------
    /** \todo REFACTOR A real app would register a subset of the
     * fields on the mesh database as fields that the app would want
     * read at one or all or specified steps.  In this example, all
     * fields existing on the input mesh database are defined on the
     * parts in the stk::mesh.
     *
     * The real app would also only register a subset of the stk::mesh
     * fields as output fields and would probably have a mapping from
     * the internally used name to some name picked by the user. In
     * this example, all Ioss::Field::TRANSIENT fields defined on the stk::mesh are
     * output to the results database and the internal stk::mesh field
     * name is used as the name on the database....
     */

    out_region.begin_mode(Ioss::STATE_DEFINE_TRANSIENT);

    // Special processing for nodeblock (all nodes in model)...
    stk::io::ioss_add_fields(fem_meta_data.universal_part(), stk::topology::NODE_RANK,
                             out_region.get_node_blocks()[0],
                             Ioss::Field::TRANSIENT);

    const stk::mesh::PartVector & all_parts = fem_meta_data.get_parts();
    for ( stk::mesh::PartVector::const_iterator
            ip = all_parts.begin(); ip != all_parts.end(); ++ip ) {

        stk::mesh::Part * const part = *ip;

        const stk::mesh::EntityRank part_rank = part->primary_entity_rank();

        // Check whether this part should be output to results database.
        if (stk::io::is_part_io_part(*part)) {
            // Get Ioss::GroupingEntity corresponding to this part...
            Ioss::GroupingEntity *entity = out_region.get_entity(part->name());
            if (entity != NULL) {
                if (entity->type() == Ioss::SIDESET) {
                    Ioss::SideSet *sset = dynamic_cast<Ioss::SideSet*>(entity);
                    assert(sset != NULL);
                    int block_count = sset->block_count();
                    for (int i=0; i < block_count; i++) {
                        Ioss::SideBlock *fb = sset->get_block(i);
                        stk::io::ioss_add_fields(*part, part_rank,
                                                 fb, Ioss::Field::TRANSIENT);
                    }
                } else {
                    stk::io::ioss_add_fields(*part, part_rank,
                                             entity, Ioss::Field::TRANSIENT);
                }
            } else {
                /// \todo IMPLEMENT handle error... Possibly an assert since
                /// I think the corresponding entity should always exist...
            }
        }
    }
    out_region.end_mode(Ioss::STATE_DEFINE_TRANSIENT);
    // ------------------------------------------------------------------------

    // Read and Write transient fields...
    out_region.begin_mode(Ioss::STATE_TRANSIENT);
    int timestep_count = in_region.get_property("state_count").get_int();
    for (int step = 1; step <= timestep_count; step++) {
        double time = in_region.get_state_time(step);

        // Read data from the io input mesh database into stk::mesh fields...
        process_input_request(in_region, bulk_data, step);

        // execute()

        // Write data from the stk::mesh fields out to the output database.a
        int out_step = out_region.add_state(time);
        process_output_request(out_region, bulk_data, out_step);
    }
    out_region.end_mode(Ioss::STATE_TRANSIENT);
}
Example #3
0
bool test_change_owner_with_constraint( stk_classic::ParallelMachine pm )
{
  bool success = true ;

  const unsigned p_rank = stk_classic::parallel_machine_rank( pm );
  const unsigned p_size = stk_classic::parallel_machine_size( pm );

  if ( p_size != 2 ) { return success ; }

  std::vector<std::string> rank_names = stk_classic::mesh::fem::entity_rank_names(spatial_dimension);
  const stk_classic::mesh::EntityRank constraint_rank = rank_names.size();
  rank_names.push_back("Constraint");
  stk_classic::mesh::fem::FEMMetaData fem_meta_data( spatial_dimension, rank_names );
  const stk_classic::mesh::EntityRank element_rank = fem_meta_data.element_rank();

  VectorField * coordinates_field =
    & put_field(
        fem_meta_data.declare_field<VectorField>("coordinates"),
        NODE_RANK,
        fem_meta_data.universal_part() ,
        3
        );

  stk_classic::mesh::Part & owned_part = fem_meta_data.locally_owned_part();
  stk_classic::mesh::Part & quad_part  = stk_classic::mesh::fem::declare_part<Quad4>( fem_meta_data, "quad");

  fem_meta_data.commit();

  stk_classic::mesh::BulkData bulk_data( stk_classic::mesh::fem::FEMMetaData::get_meta_data(fem_meta_data),
                                 pm,
                                 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk_classic::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk_classic::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk_classic::mesh::fem::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk_classic::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk_classic::mesh::Entity * n = bulk_data.get_entity( NODE_RANK, nid );
        double * const coord = stk_classic::mesh::field_data( *coordinates_field , *n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size>1 )
  {
    std::vector<stk_classic::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 1 ) );

      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 1 ) );
      ep.push_back( stk_classic::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 1 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    bulk_data.modification_begin();

    if ( p_rank==1 )
    {
      // create constraint

      stk_classic::mesh::Entity * n10 = bulk_data.get_entity( NODE_RANK, 10 );
      stk_classic::mesh::Entity * n11 = bulk_data.get_entity( NODE_RANK, 11 );
      stk_classic::mesh::Entity * n12 = bulk_data.get_entity( NODE_RANK, 12 );

      stk_classic::mesh::PartVector add;
      add.push_back( &owned_part );
      const stk_classic::mesh::EntityId c_entity_id = 1;
      stk_classic::mesh::Entity & c = bulk_data.declare_entity( constraint_rank, c_entity_id, add );
      bulk_data.declare_relation( c , *n10 , 0 );
      bulk_data.declare_relation( c , *n11 , 1 );
      bulk_data.declare_relation( c , *n12 , 2 );
    }

    bulk_data.modification_end();

    stk_classic::mesh::Entity * n10 = bulk_data.get_entity( NODE_RANK, 10 );

    if ( p_rank==0 or p_rank==1 )
    {
      ThrowErrorMsgIf( !stk_classic::mesh::in_shared( *n10 ), "NODE[10] not shared" );
    }

    bulk_data.modification_begin();

    if ( p_rank==1 )
    {
      // destroy constraint

      stk_classic::mesh::Entity * c1 = bulk_data.get_entity( constraint_rank, 1 );

      ThrowErrorMsgIf( !bulk_data.destroy_entity( c1 ),
                       "failed to destroy constraint" );
    }

    bulk_data.modification_end();

    if ( p_rank==0 or p_rank==1 )
    {
      ThrowErrorMsgIf( stk_classic::mesh::in_shared( *n10 ), "NODE[10] shared" );
    }
  }

  return success ;
}
Example #4
0
bool test_change_owner_2( stk::ParallelMachine pm )
{
  const unsigned p_rank = stk::parallel_machine_rank( pm );
  const unsigned p_size = stk::parallel_machine_size( pm );

  if ( p_size != 3 ) { return true ; }

  stk::mesh::MetaData fem_meta_data( spatial_dimension );
  const stk::mesh::EntityRank element_rank = stk::topology::ELEMENT_RANK;

  VectorField * coordinates_field =
    & put_field(
        fem_meta_data.declare_field<VectorField>(stk::topology::NODE_RANK, "coordinates"),
        fem_meta_data.universal_part() ,
        3
        );

  stk::mesh::Part & quad_part  = fem_meta_data.declare_part_with_topology( "quad", stk::topology::QUAD_4);

  fem_meta_data.commit();

  stk::mesh::BulkData bulk_data( fem_meta_data, pm, 100 );
  bulk_data.modification_begin();

  unsigned nx = 3;
  unsigned ny = 3;

  if ( p_rank==0 )
  {
    const unsigned nnx = nx + 1 ;
    for ( unsigned iy = 0 ; iy < ny ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx ; ++ix ) {
        stk::mesh::EntityId elem = 1 + ix + iy * nx ;
        stk::mesh::EntityId nodes[4] ;
        nodes[0] = 1 + ix + iy * nnx ;
        nodes[1] = 2 + ix + iy * nnx ;
        nodes[2] = 2 + ix + ( iy + 1 ) * nnx ;
        nodes[3] = 1 + ix + ( iy + 1 ) * nnx ;

        stk::mesh::declare_element( bulk_data , quad_part , elem , nodes );
      }
    }

    for ( unsigned iy = 0 ; iy < ny+1 ; ++iy ) {
      for ( unsigned ix = 0 ; ix < nx+1 ; ++ix ) {
        stk::mesh::EntityId nid = 1 + ix + iy * nnx ;
        stk::mesh::Entity n = bulk_data.get_entity( NODE_RANK, nid );
        double * const coord = stk::mesh::field_data( *coordinates_field , n );
        coord[0] = .1*ix;
        coord[1] = .1*iy;
        coord[2] = 0;
      }
    }
  }

  bulk_data.modification_end();

  if ( p_size==3 )
  {
    std::vector<stk::mesh::EntityProc> ep;

    if ( p_rank==0 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  9 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 13 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 7 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 1 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 1 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 2 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 2 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 2 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();

    ep.clear();

    if ( p_rank==1 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  9 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 13 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 14 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 15 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 16 ), 0 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 7 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 8 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 9 ), 0 ) );
    }

    if ( p_rank==2 )
    {
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  3 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  4 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  7 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK,  8 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 11 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( NODE_RANK, 12 ), 0 ) );

      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 3 ), 0 ) );
      ep.push_back( stk::mesh::EntityProc( bulk_data.get_entity( element_rank, 6 ), 0 ) );
    }

    bulk_data.modification_begin();
    bulk_data.change_entity_owner( ep );
    bulk_data.modification_end();
  }

  return true ;
}