stk::mesh::EntityVector fill_shared_entities_that_need_fixing(const stk::mesh::BulkData& bulkData) { stk::mesh::EntityVector sides; stk::mesh::get_selected_entities(bulkData.mesh_meta_data().locally_owned_part(), bulkData.buckets(bulkData.mesh_meta_data().side_rank()), sides); stk::mesh::EntityVector sidesThatNeedFixing; for(stk::mesh::Entity side : sides) if(bulkData.state(side) == stk::mesh::Created) { unsigned num_nodes = bulkData.num_nodes(side); const stk::mesh::Entity* nodes = bulkData.begin_nodes(side); std::vector<stk::mesh::EntityKey> nodeKeys(num_nodes); for(unsigned int i=0;i<num_nodes;++i) nodeKeys[i] = bulkData.entity_key(nodes[i]); std::vector<int> shared_procs; bulkData.shared_procs_intersection(nodeKeys, shared_procs); if(!shared_procs.empty()) sidesThatNeedFixing.push_back(side); } return sidesThatNeedFixing; }
void fixup_ghosted_to_shared_nodes(stk::mesh::BulkData & bulk) { stk::mesh::EntityVector ghosted_nodes_that_are_now_shared; find_ghosted_nodes_that_need_to_be_shared(bulk, ghosted_nodes_that_are_now_shared); stk::CommSparse comm(bulk.parallel()); for (int phase=0;phase<2;++phase) { for (size_t i = 0; i < ghosted_nodes_that_are_now_shared.size(); ++i) { stk::mesh::Entity node = ghosted_nodes_that_are_now_shared[i]; int proc = bulk.parallel_owner_rank(node); comm.send_buffer(proc).pack<stk::mesh::EntityKey>(bulk.entity_key(node)); } if (phase == 0 ) { comm.allocate_buffers(); } else { comm.communicate(); } } stk::mesh::EntityVector sharedNodes; for (int process=0;process<bulk.parallel_size();++process) { while(comm.recv_buffer(process).remaining()) { stk::mesh::EntityKey key; comm.recv_buffer(process).unpack<stk::mesh::EntityKey>(key); stk::mesh::Entity entity = bulk.get_entity(key); if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) ) { bulk.add_node_sharing(entity, process); sharedNodes.push_back(entity); } } } ///////////////////////// stk::CommSparse commSecondStage(bulk.parallel()); for (int phase=0;phase<2;++phase) { for (size_t i=0;i<sharedNodes.size();++i) { std::vector<int> procs; stk::mesh::EntityKey key = bulk.entity_key(sharedNodes[i]); bulk.comm_shared_procs(key, procs); for (size_t j=0;j<procs.size();++j) { if ( procs[j] != bulk.parallel_rank() ) { commSecondStage.send_buffer(procs[j]).pack<int>(bulk.parallel_rank()).pack<stk::mesh::EntityKey>(key); for (size_t k=0;k<procs.size();++k) { commSecondStage.send_buffer(procs[j]).pack<int>(procs[k]).pack<stk::mesh::EntityKey>(key); } } } } if (phase == 0 ) { commSecondStage.allocate_buffers(); } else { commSecondStage.communicate(); } } for (int proc_that_sent_message=0;proc_that_sent_message<bulk.parallel_size();++proc_that_sent_message) { if ( proc_that_sent_message == bulk.parallel_rank() ) continue; while(commSecondStage.recv_buffer(proc_that_sent_message).remaining()) { stk::mesh::EntityKey key; int sharingProc; commSecondStage.recv_buffer(proc_that_sent_message).unpack<int>(sharingProc).unpack<stk::mesh::EntityKey>(key); if ( sharingProc != bulk.parallel_rank() ) { stk::mesh::Entity entity = bulk.get_entity(key); if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) && !bulk.in_shared(key, sharingProc) ) { bulk.add_node_sharing(entity, sharingProc); } } } } }