Beispiel #1
0
inline
void setupKeyholeMesh2D_case2(stk::mesh::BulkData& bulk)
{
//
//   proc 0      proc 1
//            |
//            | block_2 block_3
//            |
// block_1    |         12---11
//            |         | 4  |
//    4----3  | 3----6  6----10
//    | 1  |  | |  2 |
//    1----2  | 2----5  5----9
//            |         | 3  |
//            |         7----8
//            |
//
//nodes 5 and 6 are ghosts (aura) on proc 0,
//and should be members of block_2 and block_3 on proc 0
//if edges are added, the edge between nodes 5 and 6 should
//be a member of block_2 not block_3.
//

  stk::mesh::MetaData& meta = bulk.mesh_meta_data();

  stk::mesh::Part& block_1 = meta.declare_part_with_topology("block_1", stk::topology::QUAD_4_2D);
  stk::mesh::Part& block_2 = meta.declare_part_with_topology("block_2", stk::topology::QUAD_4_2D);
  stk::mesh::Part& block_3 = meta.declare_part_with_topology("block_3", stk::topology::QUAD_4_2D);
  meta.commit();

  bulk.modification_begin();

  stk::mesh::EntityIdVector elem1_nodes {1, 2, 3, 4};
  stk::mesh::EntityIdVector elem2_nodes {2, 5, 6, 3};
  stk::mesh::EntityIdVector elem3_nodes {7, 8, 9, 5};
  stk::mesh::EntityIdVector elem4_nodes {6, 10, 11, 12};

  stk::mesh::EntityId elemId = 1;
  if (bulk.parallel_rank() == 0) {
    stk::mesh::declare_element(bulk, block_1, elemId, elem1_nodes);
    stk::mesh::Entity node2 = bulk.get_entity(stk::topology::NODE_RANK, 2);
    stk::mesh::Entity node3 = bulk.get_entity(stk::topology::NODE_RANK, 3);
    bulk.add_node_sharing(node2, 1);
    bulk.add_node_sharing(node3, 1);
  }
  else if (bulk.parallel_rank() == 1) {
    elemId = 2;
    stk::mesh::declare_element(bulk, block_2, elemId, elem2_nodes);
    elemId = 3;
    stk::mesh::declare_element(bulk, block_3, elemId, elem3_nodes);
    elemId = 4;
    stk::mesh::declare_element(bulk, block_3, elemId, elem4_nodes);
    stk::mesh::Entity node2 = bulk.get_entity(stk::topology::NODE_RANK, 2);
    stk::mesh::Entity node3 = bulk.get_entity(stk::topology::NODE_RANK, 3);
    bulk.add_node_sharing(node2, 0);
    bulk.add_node_sharing(node3, 0);
  }

  bulk.modification_end();
}
Beispiel #2
0
void setupKeyholeMesh2D_case1(stk::mesh::BulkData& bulk)
{
//
//   proc 0      proc 1
//            |
//            |  block_2 block_3
//            |
//  block_1   |  10---9  9----12
//            |  | 3  |  |  4  |
//    4----3  |  3----8  8----11
//    | 1  |  |
//    1----2  |  2----7
//            |  | 2  |
//            |  5----6
//            |
//
//shared nodes 2 and 3 should be members of block_1 and block_2 on both procs
//nodes 8 and 9 are ghosts on proc 0, and should be members of block_2 and block_3
//
//if edges are added, the edge between nodes 2 and 3 should be a member of block_1 not block_2.
//
//also, the edge between nodes 8 and 9 should be a member of block_2 and block_3 on both procs.

  stk::mesh::MetaData& meta = bulk.mesh_meta_data();

  stk::mesh::Part& block_1 = meta.declare_part_with_topology("block_1", stk::topology::QUAD_4_2D);
  stk::mesh::Part& block_2 = meta.declare_part_with_topology("block_2", stk::topology::QUAD_4_2D);
  stk::mesh::Part& block_3 = meta.declare_part_with_topology("block_3", stk::topology::QUAD_4_2D);
  meta.commit();

  bulk.modification_begin();

  const int nodesPerElem = 4;
  stk::mesh::EntityId elem1_nodes[nodesPerElem] = {1, 2, 3, 4};
  stk::mesh::EntityId elem2_nodes[nodesPerElem] = {5, 6, 7, 2};
  stk::mesh::EntityId elem3_nodes[nodesPerElem] = {3, 8, 9, 10};
  stk::mesh::EntityId elem4_nodes[nodesPerElem] = {8, 11, 12, 9};

  stk::mesh::EntityId elemId = 1;
  if (bulk.parallel_rank() == 0) {
    stk::mesh::declare_element(bulk, block_1, elemId, elem1_nodes);
  }
  else if (bulk.parallel_rank() == 1) {
    elemId = 2;
    stk::mesh::declare_element(bulk, block_2, elemId, elem2_nodes);
    elemId = 3;
    stk::mesh::declare_element(bulk, block_2, elemId, elem3_nodes);
    elemId = 4;
    stk::mesh::declare_element(bulk, block_3, elemId, elem4_nodes);
  }

  bulk.modification_end();
}
inline void createBoundingBoxesForElementsInElementBlocks(const stk::mesh::BulkData &bulk, FlaotBoxVector& domainBoxes)
{
    stk::mesh::EntityVector elements;
    stk::mesh::get_selected_entities(bulk.mesh_meta_data().locally_owned_part(), bulk.buckets(stk::topology::ELEM_RANK), elements);

    size_t numberBoundingBoxes = elements.size();
    domainBoxes.resize(numberBoundingBoxes);

    stk::mesh::FieldBase const * coords = bulk.mesh_meta_data().coordinate_field();

    std::vector<double> boxCoordinates(6);

    for(size_t i=0;i<elements.size();++i)
    {
        unsigned num_nodes = bulk.num_nodes(elements[i]);
        std::vector<double> coordinates(3*num_nodes,0);
        const stk::mesh::Entity* nodes = bulk.begin_nodes(elements[i]);
        for(unsigned j=0;j<num_nodes;++j)
        {
            double* data = static_cast<double*>(stk::mesh::field_data(*coords, nodes[j]));
            coordinates[3*j] = data[0];
            coordinates[3*j+1] = data[1];
            coordinates[3*j+2] = data[2];
        }
        findBoundingBoxCoordinates(coordinates, boxCoordinates);
        unsigned id = bulk.identifier(elements[i]);
        Ident domainBoxId(id, bulk.parallel_rank());
        domainBoxes[i] = std::make_pair(FloatBox(boxCoordinates[0], boxCoordinates[1], boxCoordinates[2],
                                                boxCoordinates[3], boxCoordinates[4], boxCoordinates[5]),
                                                domainBoxId);

    }
}
Beispiel #4
0
inline
void setupKeyholeMesh3D_case2(stk::mesh::BulkData& bulk)
{
    ThrowRequire(bulk.parallel_size() == 3);
    stk::io::fill_mesh("generated:3x1x3", bulk);

    stk::mesh::EntityProcVec elementProcChanges;
    if (bulk.parallel_rank() == 1) {
        elementProcChanges.push_back(stk::mesh::EntityProc(bulk.get_entity(stk::topology::ELEM_RANK,4),2));
        elementProcChanges.push_back(stk::mesh::EntityProc(bulk.get_entity(stk::topology::ELEM_RANK,6),2));
    }
    bulk.change_entity_owner(elementProcChanges);
    bulk.modification_begin();
    if (bulk.parallel_rank() == 1) {
        stk::mesh::Entity local_element5 = bulk.get_entity(stk::topology::ELEM_RANK,5);
        const bool delete_success = bulk.destroy_entity(local_element5);
        ThrowRequire(delete_success);
    }
    bulk.modification_end();
}
void print_entity_proc_map( stk::diag::Writer & writer ,
                            const stk::mesh::BulkData & mesh )
{
  const stk::mesh::MetaData & meta = mesh.mesh_meta_data();
  const std::vector<stk::mesh::Entity*> & comm = mesh.entity_comm();
  const std::vector<stk::mesh::Ghosting*> & ghost = mesh.ghostings();

  size_t counter = 0 ;

  for ( size_t ig = 0 ; ig < ghost.size() ; ++ig ) {

    const stk::mesh::Ghosting & g = * ghost[ig] ;

    writer << "P" << mesh.parallel_rank()
           << " " << g.name() << " Communication:" << std::endl ;

    for ( std::vector<stk::mesh::Entity*>::const_iterator
          i = comm.begin() ; i != comm.end() ; ++i ) {

      const stk::mesh::Entity & entity = **i ;

      std::vector<unsigned> procs ;

      stk::mesh::comm_procs( g , entity , procs );

      if ( ! procs.empty() ) {
        writer << "[" << counter << "] "
               << meta.entity_rank_name( entity.entity_rank() )
               << "[" << entity.identifier() << " " ;
        if ( entity.owner_rank() != mesh.parallel_rank() ) {
          writer << "not_" ;
        }
        writer << "owned ] {" ;
        for ( size_t j = 0 ; j < procs.size() ; ++j ) {
          writer << " " << procs[j] ;
        }
        writer << " }" << std::endl ;
      }
    }
  }
}
Beispiel #6
0
std::string get_non_unique_key_messages(const stk::mesh::BulkData& bulkData, const std::vector<stk::mesh::EntityKeyProc> &badKeyProcs)
{
    std::ostringstream os;
    for(const stk::mesh::EntityKeyProc& keyProc : badKeyProcs)
    {
        stk::mesh::Entity entity = bulkData.get_entity(keyProc.first);
        os << "[" << bulkData.parallel_rank() << "] Key " << keyProc.first <<
                get_topology(bulkData.bucket(entity).topology()) << "is also present (inappropriately) on processor " <<
                keyProc.second << "." << std::endl;
    }
    return os.str();
}
Beispiel #7
0
// element ids / proc_id:
// |-------|-------|-------|
// |       |       |       |
// |  1/0  |  4/2  |  7/2  |
// |       |       |       |
// |-------|-------|-------|
// |       |       |       |
// |  2/0  |  5/1  |  8/2  |
// |       |       |       |
// |-------|-------|-------|
// |       |       |       |
// |  3/0  |  6/2  |  9/2  |
// |       |       |       |
// |-------|-------|-------|
inline
void setupKeyholeMesh3D_case1(stk::mesh::BulkData& bulk)
{
    ThrowRequire(bulk.parallel_size() == 3);
    stk::io::fill_mesh("generated:3x1x3", bulk);

    stk::mesh::EntityProcVec elementProcChanges;
    if (bulk.parallel_rank() == 1) {
        elementProcChanges.push_back(stk::mesh::EntityProc(bulk.get_entity(stk::topology::ELEM_RANK,4u),2));
        elementProcChanges.push_back(stk::mesh::EntityProc(bulk.get_entity(stk::topology::ELEM_RANK,6u),2));
    }
    bulk.change_entity_owner(elementProcChanges);
}
inline void add_nodes_to_move(stk::mesh::BulkData& bulk,
                       stk::mesh::Entity elem,
                       int dest_proc,
                       std::vector<stk::mesh::EntityProc>& entities_to_move)
{
    const stk::mesh::Entity* nodes = bulk.begin_nodes(elem);
    for(unsigned i = 0; i < bulk.num_nodes(elem); ++i)
    {
        if(bulk.parallel_owner_rank(nodes[i]) == bulk.parallel_rank())
        {
            entities_to_move.push_back(stk::mesh::EntityProc(nodes[i], dest_proc));
        }
    }
}
Beispiel #9
0
std::string get_message_for_split_coincident_elements(const stk::mesh::BulkData& bulkData, const std::map<stk::mesh::EntityId, std::pair<stk::mesh::EntityId, int> > & splitCoincidentElements)
{
    std::ostringstream out;
    for(const auto& item : splitCoincidentElements) {
        stk::mesh::Entity element = bulkData.get_entity(stk::topology::ELEM_RANK,item.first);
        const stk::mesh::PartVector& elementParts = bulkData.bucket(element).supersets();
        std::string blockNames;
        blockNames = "{";
        for (const stk::mesh::Part* part : elementParts) {
            if (stk::mesh::impl::is_element_block(*part)) {
                blockNames += " " + part->name();
            }
        }
        blockNames += " }";
        out << "[" << bulkData.parallel_rank() << "] Element " << item.first << " (" << bulkData.bucket(element).topology() << ") in blocks " << blockNames << " is coincident with element " << item.second.first << " on processor " << item.second.second << std::endl;
    }
    return out.str();
}
Beispiel #10
0
std::vector<stk::mesh::EntityKeyProc> get_non_unique_keys(const stk::mesh::BulkData& bulkData, const stk::parallel::DistributedIndex& distributedIndex,
        const stk::parallel::DistributedIndex::KeyTypeVector& localKeys)
{
    stk::parallel::DistributedIndex::KeyProcVector sharedKeyProcs;
    distributedIndex.query_to_usage(localKeys, sharedKeyProcs);

    std::vector<stk::mesh::EntityKeyProc> badKeys;
    for (const stk::parallel::DistributedIndex::KeyProc& sharedKeyProc : sharedKeyProcs)
    {
        stk::mesh::EntityKey key( static_cast<stk::mesh::EntityKey::entity_key_t>(sharedKeyProc.first) );
        if ( bulkData.parallel_rank() != sharedKeyProc.second )
        {
            if(!bulkData.in_shared(key, sharedKeyProc.second))
                badKeys.push_back({key, sharedKeyProc.second});
        }
    }
    return badKeys;
}
void fill_sharing_data(stk::mesh::BulkData& bulkData, stk::mesh::ElemElemGraph &graph, const stk::mesh::EntityVector& sidesThatNeedFixing, std::vector<SideSharingData>& sideSharingDataThisProc, std::vector<stk::mesh::impl::IdViaSidePair>& idAndSides)
{
    // Element 1, side 5: face 15
    // Element 2, side 3: face 23
    // Are these faces the same? Yes: delete face 23, then connect face 15 to element 2 with negative permutation

    const stk::mesh::PartOrdinal sharedOrd = bulkData.mesh_meta_data().globally_shared_part().mesh_meta_data_ordinal();
    for(size_t i=0;i<sidesThatNeedFixing.size();++i)
    {
        stk::mesh::impl::ElementViaSidePair elementAndSide = get_element_and_side_ordinal(bulkData, sidesThatNeedFixing[i]);
        stk::mesh::impl::LocalId localElemId = graph.get_local_element_id(elementAndSide.element);
        for(const stk::mesh::GraphEdge& edge : graph.get_edges_for_element(localElemId))
        {
            if(edge.side1() == elementAndSide.side && edge.elem2() < 0)
            {
                const stk::mesh::impl::ParallelInfo &pInfo = graph.get_parallel_info_for_graph_edge(edge);
                const stk::mesh::Entity* nodes = bulkData.begin_nodes(sidesThatNeedFixing[i]);
                unsigned numNodes = bulkData.num_nodes(sidesThatNeedFixing[i]);

                SideSharingData localTemp({bulkData.identifier(elementAndSide.element), elementAndSide.side},
                                          sidesThatNeedFixing[i],
                                          pInfo.get_proc_rank_of_neighbor(),
                                          std::min(bulkData.parallel_rank(),pInfo.get_proc_rank_of_neighbor()),
                                          bulkData.identifier(sidesThatNeedFixing[i]));

                localTemp.sideNodes.resize(numNodes);
                for(unsigned j=0; j<numNodes; ++j) {
                    localTemp.sideNodes[j] = bulkData.identifier(nodes[j]);
                }

                fill_part_ordinals_besides_owned_and_shared(bulkData.bucket(sidesThatNeedFixing[i]), sharedOrd, localTemp.partOrdinals);

                sideSharingDataThisProc.push_back(localTemp);

                stk::mesh::EntityId localId = -edge.elem2();
                idAndSides.push_back({localId, edge.side2()});
            }
        }
    }
}
void make_small_hybrid_mesh(stk::mesh::MetaData &meta, stk::mesh::BulkData &mesh,
                            bool user_attempt_no_induce = false, bool user_parts_force_no_induce = true)
{
    stk::ParallelMachine pm = MPI_COMM_WORLD;
    int p_size = stk::parallel_machine_size(pm);

    if(p_size > 2)
    {
        return;
    }

    const unsigned p_rank = mesh.parallel_rank();

    stk::mesh::Part * hexPart = &meta.get_topology_root_part(stk::topology::HEX_8);
    stk::mesh::Part * pyrPart = &meta.get_topology_root_part(stk::topology::PYRAMID_5);
    stk::mesh::Part * tetPart = &meta.get_topology_root_part(stk::topology::TET_4);

    if (user_attempt_no_induce)
    {
        hexPart = &meta.declare_part_with_topology("my_hex_part",stk::topology::HEX_8, user_parts_force_no_induce);
        pyrPart = &meta.declare_part_with_topology("my_pyr_part",stk::topology::PYRAMID_5, user_parts_force_no_induce);
        tetPart = &meta.declare_part_with_topology("my_tet_part",stk::topology::TET_4, user_parts_force_no_induce);

        EXPECT_EQ(user_parts_force_no_induce, hexPart->force_no_induce());
        EXPECT_EQ(user_parts_force_no_induce, pyrPart->force_no_induce());
        EXPECT_EQ(user_parts_force_no_induce, tetPart->force_no_induce());
    }

    meta.commit();

    const size_t numHex = 1;
    stk::mesh::EntityIdVector hexNodeIDs[] {
        { 1, 2, 3, 4, 5, 6, 7, 8 }
    };
    stk::mesh::EntityId hexElemIDs[] = { 1 };

    const size_t numPyr = 1;
    stk::mesh::EntityIdVector pyrNodeIDs[] {
        { 5, 6, 7, 8, 9 }
    };
    stk::mesh::EntityId pyrElemIDs[] = { 2 };

    const size_t numTet = 4;
    stk::mesh::EntityIdVector tetNodeIDs[] {
        { 7, 8, 9, 12 },
        { 6, 9, 10, 7 },
        { 7, 9, 10, 12 },
        { 7, 12, 10, 11 }
    };
    stk::mesh::EntityId tetElemIDs[] = { 3, 4, 5, 6 };

    // list of triplets: (owner-proc, shared-nodeID, sharing-proc)
    std::vector< std::vector<unsigned> > shared_nodeIDs_and_procs
    {
        { 0, 5, 1 },  // proc 0
        { 0, 6, 1 },
        { 0, 7, 1 },
        { 0, 8, 1 },
        { 1, 5, 0 },  // proc 1
        { 1, 6, 0 },
        { 1, 7, 0 },
        { 1, 8, 0 }
    };

    mesh.modification_begin();

    if (0 == p_rank) {
        for (size_t i = 0; i < numHex; ++i) {
          stk::mesh::declare_element(mesh, *hexPart, hexElemIDs[i], hexNodeIDs[i]);
        }
    }
    if ( (1 == p_rank) || (1 == p_size) )  { // setup the pyramids/tets for either np 2 or serial
        for (size_t i = 0; i < numPyr; ++i) {
          stk::mesh::declare_element(mesh, *pyrPart, pyrElemIDs[i], pyrNodeIDs[i]);
        }
        for (size_t i = 0; i < numTet; ++i) {
          stk::mesh::declare_element(mesh, *tetPart, tetElemIDs[i], tetNodeIDs[i]);
        }
    }

    if (p_size > 1)
    {
        for (size_t nodeIdx = 0, end = shared_nodeIDs_and_procs.size(); nodeIdx < end; ++nodeIdx) {
            if (p_rank == shared_nodeIDs_and_procs[nodeIdx][0]) {
                stk::mesh::EntityId nodeID = shared_nodeIDs_and_procs[nodeIdx][1];
                int sharingProc = shared_nodeIDs_and_procs[nodeIdx][2];
                stk::mesh::Entity node = mesh.get_entity(stk::topology::NODE_RANK, nodeID);
                mesh.add_node_sharing(node, sharingProc);
            }
        }
    }

    mesh.modification_end();
}
Beispiel #13
0
void fixup_ghosted_to_shared_nodes(stk::mesh::BulkData & bulk)
{
    stk::mesh::EntityVector ghosted_nodes_that_are_now_shared;
    find_ghosted_nodes_that_need_to_be_shared(bulk, ghosted_nodes_that_are_now_shared);

    stk::CommSparse comm(bulk.parallel());

    for (int phase=0;phase<2;++phase)
    {
        for (size_t i = 0; i < ghosted_nodes_that_are_now_shared.size(); ++i)
        {
            stk::mesh::Entity node = ghosted_nodes_that_are_now_shared[i];
            int proc = bulk.parallel_owner_rank(node);
            comm.send_buffer(proc).pack<stk::mesh::EntityKey>(bulk.entity_key(node));
        }
        if (phase == 0 )
        {
            comm.allocate_buffers();
        }
        else
        {
            comm.communicate();
        }
    }

    stk::mesh::EntityVector sharedNodes;
    for (int process=0;process<bulk.parallel_size();++process)
    {
        while(comm.recv_buffer(process).remaining())
        {
            stk::mesh::EntityKey key;
            comm.recv_buffer(process).unpack<stk::mesh::EntityKey>(key);

            stk::mesh::Entity entity = bulk.get_entity(key);
            if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) )
            {
                bulk.add_node_sharing(entity, process);
                sharedNodes.push_back(entity);
            }
        }
    }
/////////////////////////

    stk::CommSparse commSecondStage(bulk.parallel());
    for (int phase=0;phase<2;++phase)
    {
        for (size_t i=0;i<sharedNodes.size();++i)
        {
            std::vector<int> procs;
            stk::mesh::EntityKey key = bulk.entity_key(sharedNodes[i]);
            bulk.comm_shared_procs(key, procs);
            for (size_t j=0;j<procs.size();++j)
            {
                if ( procs[j] != bulk.parallel_rank() )
                {
                    commSecondStage.send_buffer(procs[j]).pack<int>(bulk.parallel_rank()).pack<stk::mesh::EntityKey>(key);
                    for (size_t k=0;k<procs.size();++k)
                    {
                        commSecondStage.send_buffer(procs[j]).pack<int>(procs[k]).pack<stk::mesh::EntityKey>(key);
                    }
                }
            }
        }
        if (phase == 0 )
        {
            commSecondStage.allocate_buffers();
        }
        else
        {
            commSecondStage.communicate();
        }
    }

    for (int proc_that_sent_message=0;proc_that_sent_message<bulk.parallel_size();++proc_that_sent_message)
    {
        if ( proc_that_sent_message == bulk.parallel_rank() ) continue;
        while(commSecondStage.recv_buffer(proc_that_sent_message).remaining())
        {
            stk::mesh::EntityKey key;
            int sharingProc;
            commSecondStage.recv_buffer(proc_that_sent_message).unpack<int>(sharingProc).unpack<stk::mesh::EntityKey>(key);
            if ( sharingProc != bulk.parallel_rank() )
            {
                stk::mesh::Entity entity = bulk.get_entity(key);
                if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) && !bulk.in_shared(key, sharingProc) )
                {
                    bulk.add_node_sharing(entity, sharingProc);
                }
            }
        }
    }
}
Beispiel #14
0
    /**
     * Check for nonpositive Jacobian
     */
    bool GeometryVerifier::isGeometryBad(stk::mesh::BulkData& bulk, bool printTable) //, stk::mesh::Part& mesh_part )
    {
      const stk::mesh::fem::FEMMetaData& meta = stk::mesh::fem::FEMMetaData::get(bulk);
      const unsigned p_rank = bulk.parallel_rank();

      unsigned foundBad=0;
      jac_data_map jac_data;

      stk::mesh::Field<double, stk::mesh::Cartesian> *coord_field =
        meta.get_field<stk::mesh::Field<double, stk::mesh::Cartesian> >("coordinates");

      mesh::Selector select_owned( meta.locally_owned_part() );
      const std::vector<mesh::Bucket*> & buckets = bulk.buckets( meta.element_rank() );

      //for ( std::vector<mesh::Bucket *>::const_iterator ik = buckets.begin() ; ik != buckets.end() ; ++ik )
      const stk::mesh::PartVector & all_parts = meta.get_parts();
      for ( stk::mesh::PartVector::const_iterator ip = all_parts.begin(); ip != all_parts.end(); ++ip ) 
        {
          stk::mesh::Part * part = *ip;

          if ( stk::mesh::is_auto_declared_part(*part) )
            continue;

          const CellTopologyData * const part_cell_topo_data = stk::percept::PerceptMesh::get_cell_topology(*part);
          //std::cout << "P[" << p_rank << "] part = " << part->name() << " part_cell_topo_data= " << part_cell_topo_data << " topo-name= "
          //          << (part_cell_topo_data ? part_cell_topo_data->name : "null") << std::endl;

          if (part_cell_topo_data)
            jac_data[part_cell_topo_data->name] = jacData();
        }

      for (unsigned ipass = 0; ipass < 1; ipass++)
        {
          for ( std::vector<mesh::Bucket *>::const_iterator ik = buckets.begin() ; ik != buckets.end() ; ++ik )
            {
              if ( select_owned( **ik ) ) {

              const mesh::Bucket & bucket = **ik ;

              // Number of elems in this bucket of elems and elem field data
              const unsigned number_elems = bucket.size();

              double * elem_node_data = field_data( *coord_field , bucket.begin() );
              //double * elem_centroid_data = field_data( elem_centroid_field , bucket.begin() );
              //double * const coord = field_data( m_coordinates_field , *node );

              // FIXME
              if (0) { elem_node_data[0]++;}

#if 1
              const CellTopologyData * const bucket_cell_topo_data = stk::percept::PerceptMesh::get_cell_topology(bucket);
              int bucket_shardsId = ShardsInterfaceTable::s_singleton.lookupShardsId(bucket_cell_topo_data->name);
#endif

              //if (0) { std::cout << bucket_cell_topo_data->name; }
              if (0) { std::cout << "bucket_shardsId= " << bucket_shardsId << " name= " << bucket_cell_topo_data->name <<  std::endl; }

              if (0) { std::cout << "number_elems= " << number_elems << std::endl;}

              CellTopology cell_topo(bucket_cell_topo_data);
              double volEqui = getEquiVol(cell_topo);
              unsigned numCells = number_elems;
              unsigned numNodes = cell_topo.getNodeCount();
              unsigned spaceDim = cell_topo.getDimension();
              //unsigned spatialDimMeta = stk::mesh::fem::FEMMetaData::get(bulk).spatial_dimension();

              // Rank-3 array with dimensions (C,N,D) for the node coordinates of 3 traingle cells
              FieldContainer<double> cellNodes(numCells, numNodes, spaceDim);
              PerceptMesh::fillCellNodes(bucket,  coord_field, cellNodes, spaceDim);

              FieldContainer<double> volume(numCells);

              // get min/max edge length
              FieldContainer<double> elem_min_edge_length(number_elems);
              FieldContainer<double> elem_max_edge_length(number_elems);
              PerceptMesh::findMinMaxEdgeLength(bucket, *coord_field, elem_min_edge_length, elem_max_edge_length);

              /// note: we're using cubature here instead of explicitly specifying some reference points
              ///  the idea is that we'll get a good estimate of the Jacobian's sign by testing it at all the
              ///  cubature points

              DefaultCubatureFactory<double> cubFactory;                                              // create cubature factory
              unsigned cubDegree = 2;                                                                      // set cubature degree, e.g. 2
              Teuchos::RCP<Cubature<double> > myCub;
              bool hasGoodTopo = true;
              try {
                myCub = cubFactory.create(cell_topo, cubDegree);         // create default cubature
              }
              catch(...)
                {
                  if (!p_rank)
                    std::cout << "WARNING: mesh contains elements that Intrepid doesn't support for quadrature, cell_topo= " << cell_topo.getName() << std::endl;
                  //continue;
                  hasGoodTopo = false;
                }

              FieldContainer<double> jacobian_det(numCells, 1);
              unsigned numCubPoints = 1;
              FieldContainer<double> jacobian(numCells, numCubPoints, spaceDim, spaceDim);

              if (hasGoodTopo)
                {
                  numCubPoints = myCub->getNumPoints();                                               // retrieve number of cubature points

                  FieldContainer<double> cub_points(numCubPoints, spaceDim);
                  FieldContainer<double> cub_weights(numCubPoints);

                  // Rank-4 array (C,P,D,D) for the Jacobian and its inverse and Rank-2 array (C,P) for its determinant
                  //FieldContainer<double> jacobian(numCells, numCubPoints, spaceDim, spaceDim);
                  jacobian.resize(numCells, numCubPoints, spaceDim, spaceDim);
                  FieldContainer<double> jacobian_inv(numCells, numCubPoints, spaceDim, spaceDim);
                  //FieldContainer<double> jacobian_det(numCells, numCubPoints);
                  jacobian_det.resize(numCells, numCubPoints);

                  myCub->getCubature(cub_points, cub_weights);                                          // retrieve cubature points and weights

                  // Methods to compute cell Jacobians, their inverses and their determinants

                  CellTools<double>::setJacobian(jacobian, cub_points, cellNodes, cell_topo);           // compute cell Jacobians
                  CellTools<double>::setJacobianInv(jacobian_inv, jacobian);                            // compute inverses of cell Jacobians
                  CellTools<double>::setJacobianDet(jacobian_det, jacobian);                            // compute determinants of cell Jacobians

                  FieldContainer<double> weightedMeasure(numCells, numCubPoints);

                  FieldContainer<double> onesLeft(numCells,  numCubPoints);
                  onesLeft.initialize(1.0);

                  // compute weighted measure
                  FunctionSpaceTools::computeCellMeasure<double>(weightedMeasure, jacobian_det, cub_weights);

                  // integrate to get volume
                  FunctionSpaceTools::integrate<double>(volume, onesLeft, weightedMeasure,  COMP_BLAS);
                }

              jacData& jdata = jac_data[cell_topo.getName()];
              jdata.numEle += numCells;

              for (unsigned iCell = 0; iCell < numCells; iCell++)
                {
                  mesh::Entity & elem = bucket[iCell];
                  double min_edge_length = elem_min_edge_length[iCell];
                  double max_edge_length = elem_max_edge_length[iCell];
                  double max_edge_lengthNotZero = (fabs(max_edge_length) < 1.e-20? 1.e-20 : max_edge_length);

                  double cellVolActual = volume(iCell);
                  double cellVol = cellVolActual/volEqui; // scaled so that equilateral cell has vol=1.0

                  for (unsigned iCubPt = 0; iCubPt < numCubPoints; iCubPt++)
                    {
                      double jacDet = jacobian_det(iCell, iCubPt);
                      if (hasGoodTopo && jacDet < m_badJacobian)
                        {
                          ++foundBad;
                        }

                      double cellVolNotZero = fabs(cellVol) < 1.e-20? 1.e-20 : cellVol;
                      double quality_measure_1 = (cellVolNotZero < 0? -1.0 : 1.0) * min_edge_length / pow(fabs(cellVolNotZero), 1./(double(spaceDim)));
                      if (0 && iCubPt==0)
                        {
                          std::cout << "quality_measure_1= " << quality_measure_1 << " cellVolNotZero= " << cellVolNotZero << " cellVolActual= "
                                    << cellVolActual << " volEqui= " << volEqui << " min_edge_length= " << min_edge_length
                                    << " max_edge_length= " << max_edge_length << std::endl;
                        }

                      double quality_measure_2 = min_edge_length / max_edge_lengthNotZero;

                      if (ipass == 0)
                        {
                          jdata.jac.registerValue(elem.identifier(), jacDet);
                          jdata.QM_1.registerValue(elem.identifier(),  quality_measure_1);
                          jdata.QM_2.registerValue(elem.identifier(),  quality_measure_2);
                        }
                    }
                }

              if (m_dump)
                {
                  for (unsigned iCell = 0; iCell < numCells; iCell++)
                    {
                      for (unsigned iCubPt = 0; iCubPt < numCubPoints; iCubPt++)
                        {
                          stk::PrintTable table;
                          std::ostringstream msg; msg << "Jacobian"<<" iCell= "<<iCell<<" iCubPt= "<<iCubPt << " Det= " << jacobian_det(iCell, iCubPt);
                          table.setTitle(msg.str());

                          for (unsigned id = 0; id < spaceDim; id++)
                            {
                              for (unsigned jd = 0; jd < spaceDim; jd++)
                                {
                                  table << jacobian(iCell, iCubPt, id, jd);
                                }
                              table << stk::end_row;
                            }
                          std::cout << "P["<< bulk.parallel_rank() << "] " << cell_topo.getName() << "\n" << table;
                        }
                    }
                }
              }

            } // buckets

          // setup the histogram ranges and counts

        } // ipass

      for (jac_data_map::iterator itMap = jac_data.begin(); itMap != jac_data.end(); itMap++)
        {
          itMap->second.jac.finish(bulk);
          itMap->second.QM_1.finish(bulk);
          itMap->second.QM_2.finish(bulk);
        }

      //  all_reduce( mesh.parallel() , ReduceMax<1>( & error_flag ) );

      stk::PrintTable table;
      if (0)
        {
          const unsigned rank = bulk.parallel_rank();
          std::string title = "Jacobian and Quality Table P["+toString(rank)+"]\n";
          table.setTitle(title.c_str());
        }
      table.setTitle("Jacobian and Quality Table\n");

      table << "|" << "Element Type" << "|"
            << "Min JacDet" << "|" << "Id" << "|"
            << "Max JacDet" << "|" << "Id" << "|"
            << "Ave JacDet" << "|"
            << "Sum JacDet" << "|"
            << "Min QM1" << "|" << "Id" << "|"
            << "Max QM1" << "|" << "Id" << "|"
            << "Ave QM1" << "|"
            << "Min QM2" << "|" << "Id" << "|"
            << "Max QM2" << "|" << "Id" << "|"
            << "Ave QM2" << "|"
            << stk::end_header;

      for (jac_data_map::iterator itMap = jac_data.begin(); itMap != jac_data.end(); itMap++)
        {
          if (1)
            {
              std::cout << "P[" << p_rank << "] nele = " << itMap->second.numEle << std::endl;
            }

          table << "|" << itMap->first << "|"
                << itMap->second.jac.min << "|"
                << itMap->second.jac.min_i << "|"
                << itMap->second.jac.max << "|"
                << itMap->second.jac.max_i << "|"
                << itMap->second.jac.ave << "|"
                << itMap->second.jac.sum << "|"
                << itMap->second.QM_1.min << "|"
                << itMap->second.QM_1.min_i << "|"
                << itMap->second.QM_1.max << "|"
                << itMap->second.QM_1.max_i << "|"
                << itMap->second.QM_1.ave << "|"
                << itMap->second.QM_2.min << "|"
                << itMap->second.QM_2.min_i << "|"
                << itMap->second.QM_2.max << "|"
                << itMap->second.QM_2.max_i << "|"
                << itMap->second.QM_2.ave << "|"
                << stk::end_row;
        }

      if (!p_rank && printTable)
        //if (printTable)
        {
          std::cout << "P[" << p_rank << "] Explanation: JacDet=det(element jacobian), QM1=min(element edge length)/(elemement vol)^(1/dim), QM2=min(element edge length)/max(element edge length)\n" 
                    << " NOTE: QM1 is normalized to 1 for ideally shaped elements, < 1 or > 1 values signify badly shaped elements\n"
                    << " NOTE: QM2 is small for badly shaped elements, normalized to 1 for ideally shaped elements\n"
                    << std::endl;
          std::cout << table;
        }

      return (foundBad > 0);
    }
inline
void setup2Block2HexMesh(stk::mesh::BulkData& bulk)
{
//
//   proc 0      proc 1
//             |
//    block_1  |  block_2
//             |
//      8----7 |    7----12
//     /    /| |   /    / |
//    5----6 3 |  6----11 10
//    | 1  |/  |  | 2  | /
//    1----2   |  2----9
//             |
//             |
//             |
//
//shared nodes 2, 3, 6, 7
//

  if (bulk.parallel_size() > 2) {
    return;
  }

  stk::mesh::MetaData& meta = bulk.mesh_meta_data();

  stk::topology hex = stk::topology::HEX_8;
  stk::mesh::Part& block_1 = meta.declare_part_with_topology("block_1", hex);
  stk::mesh::Part& block_2 = meta.declare_part_with_topology("block_2", hex);
  meta.commit();

  bulk.modification_begin();

  stk::mesh::EntityIdVector elem1_nodes {1, 2, 3, 4, 5, 6, 7, 8};
  stk::mesh::EntityIdVector elem2_nodes {2, 9, 10, 3, 6, 11, 12, 7};

  stk::mesh::EntityId elemId = 1;
  if (bulk.parallel_rank() == 0) {
    stk::mesh::declare_element(bulk, block_1, elemId, elem1_nodes);
  }
  if (bulk.parallel_rank() == 1 || bulk.parallel_size() == 1) {
    elemId = 2;
    stk::mesh::declare_element(bulk, block_2, elemId, elem2_nodes);
  }
  if(bulk.parallel_rank() == 0 && bulk.parallel_size() == 2)
  {
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 2), 1);
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 3), 1);
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 6), 1);
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 7), 1);
  }
  if(bulk.parallel_rank() == 1 && bulk.parallel_size() == 2)
  {
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 2), 0);
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 3), 0);
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 6), 0);
    bulk.add_node_sharing(bulk.get_entity(stk::topology::NODE_RANK , 7), 0);
  }

  bulk.modification_end();
}
Beispiel #16
0
void communicate_field_data( const stk::mesh::BulkData & mesh ,
                             const std::vector< const stk::mesh::FieldBase * > & fields )
{
  if ( fields.empty() ) { return; }

  const unsigned parallel_size = mesh.parallel_size();
  const unsigned parallel_rank = mesh.parallel_rank();

  // Sizing for send and receive
  
  const unsigned zero = 0 ;
  std::vector<unsigned> send_size( parallel_size , zero );
  std::vector<unsigned> recv_size( parallel_size , zero );
  std::vector<unsigned> procs ;
  
  for ( std::vector<stk::mesh::Entity*>::const_iterator
        i =  mesh.entity_comm().begin() ;
        i != mesh.entity_comm().end() ; ++i ) {
    stk::mesh::Entity & e = **i ;

    unsigned size = 0 ;
    for ( std::vector<const stk::mesh::FieldBase *>::const_iterator
          fi = fields.begin() ; fi != fields.end() ; ++fi ) {
      const stk::mesh::FieldBase & f = **fi ; 
      size += stk::mesh::field_data_size( f , e );
    }

    if ( size ) {
      if ( e.owner_rank() == parallel_rank ) {
        // owner sends
        stk::mesh::comm_procs( e , procs );
        for ( std::vector<unsigned>::iterator
              ip = procs.begin() ; ip != procs.end() ; ++ip ) {
          send_size[ *ip ] += size ;
        }
      }
      else { 
        // non-owner receives
        recv_size[ e.owner_rank() ] += size ;
      }
    }
  }

  // Allocate send and receive buffers:
  
  stk::CommAll sparse ;

  {
    const unsigned * const s_size = & send_size[0] ;
    const unsigned * const r_size = & recv_size[0] ;
    sparse.allocate_buffers( mesh.parallel(), parallel_size / 4 , s_size, r_size);
  }

  // Send packing:

  for ( std::vector<stk::mesh::Entity*>::const_iterator
        i =  mesh.entity_comm().begin() ;
        i != mesh.entity_comm().end() ; ++i ) {
    stk::mesh::Entity & e = **i ;

    if ( e.owner_rank() == parallel_rank ) {

      stk::mesh::comm_procs( e , procs );

      for ( std::vector<const stk::mesh::FieldBase *>::const_iterator
            fi = fields.begin() ; fi != fields.end() ; ++fi ) {
        const stk::mesh::FieldBase & f = **fi ;
        const unsigned size = stk::mesh::field_data_size( f , e );

        if ( size ) {
          unsigned char * ptr =
            reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , e ));

          for ( std::vector<unsigned>::iterator
                ip = procs.begin() ; ip != procs.end() ; ++ip ) {

            stk::CommBuffer & b = sparse.send_buffer( *ip );
            b.pack<unsigned char>( ptr , size );
          }
        }
      }
    }
  }

  // Communicate:

  sparse.communicate();

  // Unpack for recv:

  for ( std::vector<stk::mesh::Entity*>::const_iterator
        i =  mesh.entity_comm().begin() ;
        i != mesh.entity_comm().end() ; ++i ) {
    stk::mesh::Entity & e = **i ;
    if ( e.owner_rank() != parallel_rank ) {

      for ( std::vector<const stk::mesh::FieldBase *>::const_iterator
            fi = fields.begin() ; fi != fields.end() ; ++fi ) {
        const stk::mesh::FieldBase & f = **fi ;
        const unsigned size = stk::mesh::field_data_size( f , e );

        if ( size ) {
          unsigned char * ptr =
            reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , e ));

          stk::CommBuffer & b = sparse.recv_buffer( e.owner_rank() );
          b.unpack<unsigned char>( ptr , size );
        }
      }
    }
  }
}
void use_case_5_generate_mesh(
  const std::string& mesh_options ,
  stk::mesh::BulkData & mesh ,
  const VectorFieldType & node_coord ,
  stk::mesh::Part & hex_block ,
  stk::mesh::Part & quad_shell_block )
{
  mesh.modification_begin();

  const unsigned parallel_size = mesh.parallel_size();
  const unsigned parallel_rank = mesh.parallel_rank();

  double t = 0 ;
  size_t num_hex = 0 ;
  size_t num_shell = 0 ;
  size_t num_nodes = 0 ;
  size_t num_block = 0 ;
  int error_flag = 0 ;

  try {

    Iogn::GeneratedMesh gmesh( mesh_options, parallel_size, parallel_rank );

    num_nodes = gmesh.node_count_proc();
    num_block = gmesh.block_count();

    t = stk::wall_time();

    std::vector<int> node_map( num_nodes , 0 );

    gmesh.node_map( node_map );

    {

      for ( size_t i = 1 ; i <= num_block ; ++i ) {
        const size_t                        num_elem = gmesh.element_count_proc(i);
        const std::pair<std::string,int> top_info = gmesh.topology_type(i);

	std::vector<int> elem_map( num_elem , 0 );
        std::vector<int> elem_conn( num_elem * top_info.second );

	gmesh.element_map( i, elem_map );
        gmesh.connectivity( i , elem_conn );

        if ( top_info.second == 8 ) {

          for ( size_t j = 0 ; j < num_elem ; ++j ) {

            const int * const local_node_id = & elem_conn[ j * 8 ] ;

            const stk::mesh::EntityId node_id[8] = {
              local_node_id[0] ,
              local_node_id[1] ,
              local_node_id[2] ,
              local_node_id[3] ,
              local_node_id[4] ,
              local_node_id[5] ,
              local_node_id[6] ,
              local_node_id[7]
            };

            const stk::mesh::EntityId elem_id = elem_map[ j ];

            stk::mesh::fem::declare_element( mesh , hex_block , elem_id , node_id );

            ++num_hex ;
          }
        }
        else if ( top_info.second == 4 ) {

          for ( size_t j = 0 ; j < num_elem ; ++j ) {

            const int * const local_node_id = & elem_conn[ j * 4 ] ;

            const stk::mesh::EntityId node_id[4] = {
              local_node_id[0] ,
              local_node_id[1] ,
              local_node_id[2] ,
              local_node_id[3]
            };

            const stk::mesh::EntityId elem_id = elem_map[ j ];

            stk::mesh::fem::declare_element( mesh , quad_shell_block , elem_id , node_id );

            ++num_shell ;
          }
        }
      }
    }

    std::vector<double> node_coordinates( 3 * node_map.size() );

    gmesh.coordinates( node_coordinates );

    if ( 3 * node_map.size() != node_coordinates.size() ) {
      std::ostringstream msg ;
      msg << "  P" << mesh.parallel_rank()
          << ": ERROR, node_map.size() = "
          << node_map.size()
          << " , node_coordinates.size() / 3 = "
          << ( node_coordinates.size() / 3 );
      throw std::runtime_error( msg.str() );
    }

    for ( unsigned i = 0 ; i < node_map.size() ; ++i ) {
      const unsigned i3 = i * 3 ;

      stk::mesh::Entity * const node = mesh.get_entity( stk::mesh::fem::FEMMetaData::NODE_RANK , node_map[i] );

      if ( NULL == node ) {
        std::ostringstream msg ;
        msg << "  P:" << mesh.parallel_rank()
            << " ERROR, Node not found: "
            << node_map[i] << " = node_map[" << i << "]" ;
        throw std::runtime_error( msg.str() );
      }

      double * const data = field_data( node_coord , *node );
      data[0] = node_coordinates[ i3 + 0 ];
      data[1] = node_coordinates[ i3 + 1 ];
      data[2] = node_coordinates[ i3 + 2 ];
    }
  }
  catch ( const std::exception & X ) {
    std::cout << "  P:" << mesh.parallel_rank() << ": " << X.what()
              << std::endl ;
    std::cout.flush();
    error_flag = 1 ;
  }
  catch( ... ) {
    std::cout << "  P:" << mesh.parallel_rank()
              << " Caught unknown exception"
              << std::endl ;
    std::cout.flush();
    error_flag = 1 ;
  }

  stk::all_reduce( mesh.parallel() , stk::ReduceMax<1>( & error_flag ) );

  if ( error_flag ) {
    std::string msg( "Failed mesh generation" );
    throw std::runtime_error( msg );
  }

  mesh.modification_end();

  double dt = stk::wall_dtime( t );

  stk::all_reduce( mesh.parallel() , stk::ReduceMax<1>( & dt ) );

  std::cout << "  P" << mesh.parallel_rank()
            << ": Meshed Hex = " << num_hex
            << " , Shell = " << num_shell
            << " , Node = " << num_nodes
            << " in " << dt << " sec"
            << std::endl ;
  std::cout.flush();
}
Beispiel #18
0
void Gear::mesh( stk::mesh::BulkData & M )
{
  stk::mesh::EntityRank element_rank = stk::topology::ELEMENT_RANK;
  stk::mesh::EntityRank side_rank = m_mesh_meta_data.side_rank();

  M.modification_begin();

  m_mesh = & M ;

  const unsigned p_size = M.parallel_size();
  const unsigned p_rank = M.parallel_rank();

  std::vector<size_t> counts ;
  stk::mesh::comm_mesh_counts(M, counts);

  // max_id is no longer available from comm_mesh_stats.
  // If we assume uniform numbering from 1.., then max_id
  // should be equal to counts...
  const stk::mesh::EntityId node_id_base = counts[ stk::topology::NODE_RANK ] + 1 ;
  const stk::mesh::EntityId elem_id_base = counts[ element_rank ] + 1 ;

  const unsigned long elem_id_gear_max =
    m_angle_num * ( m_rad_num - 1 ) * ( m_z_num - 1 );

  std::vector<stk::mesh::Part*> elem_parts ;
  std::vector<stk::mesh::Part*> face_parts ;
  std::vector<stk::mesh::Part*> node_parts ;

  {
    stk::mesh::Part * const p_gear = & m_gear ;
    stk::mesh::Part * const p_surf = & m_surf ;

    elem_parts.push_back( p_gear );
    face_parts.push_back( p_surf );
  }

  for ( unsigned ia = 0 ; ia < m_angle_num ; ++ia ) {
    for ( unsigned ir = 0 ; ir < m_rad_num - 1 ; ++ir ) {
      for ( unsigned iz = 0 ; iz < m_z_num - 1 ; ++iz ) {

        stk::mesh::EntityId elem_id_gear = identifier( m_z_num-1 , m_rad_num-1 , iz , ir , ia );

        if ( ( ( elem_id_gear * p_size ) / elem_id_gear_max ) == p_rank ) {

          stk::mesh::EntityId elem_id = elem_id_base + elem_id_gear ;

          // Create the node and set the model_coordinates

          const size_t ia_1 = ( ia + 1 ) % m_angle_num ;
          const size_t ir_1 = ir + 1 ;
          const size_t iz_1 = iz + 1 ;

          stk::mesh::Entity node[8] ;

          node[0] = create_node( node_parts, node_id_base, iz  , ir  , ia_1 );
          node[1] = create_node( node_parts, node_id_base, iz_1, ir  , ia_1 );
          node[2] = create_node( node_parts, node_id_base, iz_1, ir  , ia   );
          node[3] = create_node( node_parts, node_id_base, iz  , ir  , ia   );
          node[4] = create_node( node_parts, node_id_base, iz  , ir_1, ia_1 );
          node[5] = create_node( node_parts, node_id_base, iz_1, ir_1, ia_1 );
          node[6] = create_node( node_parts, node_id_base, iz_1, ir_1, ia   );
          node[7] = create_node( node_parts, node_id_base, iz  , ir_1, ia   );
#if 0 /* VERIFY_CENTROID */

          // Centroid of the element for verification

          const double TWO_PI = 2.0 * acos( -1.0 );
          const double angle = m_ang_inc * (0.5 + ia);
          const double z = m_center[2] + m_z_min + m_z_inc * (0.5 + iz);

          double c[3] = { 0 , 0 , 0 };

          for ( size_t j = 0 ; j < 8 ; ++j ) {
            double * const coord_data = field_data( m_model_coord , *node[j] );
            c[0] += coord_data[0] ;
            c[1] += coord_data[1] ;
            c[2] += coord_data[2] ;
          }
          c[0] /= 8 ; c[1] /= 8 ; c[2] /= 8 ;
          c[0] -= m_center[0] ;
          c[1] -= m_center[1] ;

          double val_a = atan2( c[1] , c[0] );
          if ( val_a < 0 ) { val_a += TWO_PI ; }
          const double err_a = angle - val_a ;
          const double err_z = z - c[2] ;

          const double eps = 100 * std::numeric_limits<double>::epsilon();

          if ( err_z < - eps || eps < err_z ||
               err_a < - eps || eps < err_a ) {
            std::string msg ;
            msg.append("problem setup element centroid error" );
            throw std::logic_error( msg );
          }
#endif

          stk::mesh::Entity elem =
            M.declare_entity( element_rank, elem_id, elem_parts );

          for ( size_t j = 0 ; j < 8 ; ++j ) {
            M.declare_relation( elem , node[j] ,
                                static_cast<unsigned>(j) );
          }
        }
      }
    }
  }

  // Array of faces on the surface

  {
    const size_t ir = m_rad_num - 1 ;

    for ( size_t ia = 0 ; ia < m_angle_num ; ++ia ) {
      for ( size_t iz = 0 ; iz < m_z_num - 1 ; ++iz ) {

        stk::mesh::EntityId elem_id_gear =
          identifier( m_z_num-1 , m_rad_num-1 , iz , ir-1 , ia );

        if ( ( ( elem_id_gear * p_size ) / elem_id_gear_max ) == p_rank ) {

          stk::mesh::EntityId elem_id = elem_id_base + elem_id_gear ;

          unsigned face_ord = 5 ;
          stk::mesh::EntityId face_id = elem_id * 10 + face_ord + 1;

          stk::mesh::Entity node[4] ;

          const size_t ia_1 = ( ia + 1 ) % m_angle_num ;
          const size_t iz_1 = iz + 1 ;

          node[0] = create_node( node_parts, node_id_base, iz  , ir  , ia_1 );
          node[1] = create_node( node_parts, node_id_base, iz_1, ir  , ia_1 );
          node[2] = create_node( node_parts, node_id_base, iz_1, ir  , ia   );
          node[3] = create_node( node_parts, node_id_base, iz  , ir  , ia   );

          stk::mesh::Entity face =
            M.declare_entity( side_rank, face_id, face_parts );

          for ( size_t j = 0 ; j < 4 ; ++j ) {
            M.declare_relation( face , node[j] ,
                                static_cast<unsigned>(j) );
          }

          stk::mesh::Entity elem = M.get_entity(element_rank, elem_id);

          M.declare_relation( elem , face , face_ord );
        }
      }
    }
  }
  M.modification_begin();
}