stk::parallel::DistributedIndex::KeyTypeVector get_all_local_keys(const stk::mesh::BulkData & bulkData)
{
stk::parallel::DistributedIndex::KeyTypeVector localKeys;
for(stk::mesh::EntityRank rank = stk::topology::NODE_RANK;rank < bulkData.mesh_meta_data().entity_rank_count();++rank)
{
stk::mesh::EntityVector entities;
stk::mesh::get_selected_entities(get_owned_or_shared_selector(bulkData), bulkData.buckets(rank), entities);
for(stk::mesh::Entity entity: entities)
localKeys.push_back(bulkData.entity_key(entity));
}
return localKeys;
}
stk::mesh::EntityVector fill_shared_entities_that_need_fixing(const stk::mesh::BulkData& bulkData)
{
stk::mesh::EntityVector sides;
stk::mesh::get_selected_entities(bulkData.mesh_meta_data().locally_owned_part(), bulkData.buckets(bulkData.mesh_meta_data().side_rank()), sides);
stk::mesh::EntityVector sidesThatNeedFixing;
for(stk::mesh::Entity side : sides)
if(bulkData.state(side) == stk::mesh::Created)
{
unsigned num_nodes = bulkData.num_nodes(side);
const stk::mesh::Entity* nodes = bulkData.begin_nodes(side);
std::vector<stk::mesh::EntityKey> nodeKeys(num_nodes);
for(unsigned int i=0;i<num_nodes;++i)
nodeKeys[i] = bulkData.entity_key(nodes[i]);
std::vector<int> shared_procs;
bulkData.shared_procs_intersection(nodeKeys, shared_procs);
if(!shared_procs.empty())
sidesThatNeedFixing.push_back(side);
}
return sidesThatNeedFixing;
}
void find_ghosted_nodes_that_need_to_be_shared(const stk::mesh::BulkData & bulk, stk::mesh::EntityVector& ghosted_nodes_that_are_now_shared)
{
stk::mesh::EntityRank endRank = static_cast<stk::mesh::EntityRank>(bulk.mesh_meta_data().entity_rank_count());
if (endRank >= stk::topology::END_RANK)
{
endRank = stk::topology::END_RANK;
}
for (stk::mesh::EntityRank rank=stk::topology::EDGE_RANK; rank<endRank; ++rank)
{
const stk::mesh::BucketVector& entity_buckets = bulk.buckets(rank);
for(size_t i=0; i<entity_buckets.size(); ++i)
{
const stk::mesh::Bucket& bucket = *entity_buckets[i];
if ( bucket.owned() )
{
for(size_t n=0; n<bucket.size(); ++n)
{
const stk::mesh::Entity * nodes = bulk.begin_nodes(bucket[n]);
unsigned num_nodes = bulk.num_nodes(bucket[n]);
for (unsigned j=0;j<num_nodes;++j)
{
if (bulk.in_receive_ghost(bulk.entity_key(nodes[j])))
{
ghosted_nodes_that_are_now_shared.push_back(nodes[j]);
}
}
}
}
}
}
std::sort(ghosted_nodes_that_are_now_shared.begin(), ghosted_nodes_that_are_now_shared.end());
stk::mesh::EntityVector::iterator iter = std::unique(ghosted_nodes_that_are_now_shared.begin(), ghosted_nodes_that_are_now_shared.end());
ghosted_nodes_that_are_now_shared.erase(iter, ghosted_nodes_that_are_now_shared.end());
}
void fixup_ghosted_to_shared_nodes(stk::mesh::BulkData & bulk)
{
stk::mesh::EntityVector ghosted_nodes_that_are_now_shared;
find_ghosted_nodes_that_need_to_be_shared(bulk, ghosted_nodes_that_are_now_shared);
stk::CommSparse comm(bulk.parallel());
for (int phase=0;phase<2;++phase)
{
for (size_t i = 0; i < ghosted_nodes_that_are_now_shared.size(); ++i)
{
stk::mesh::Entity node = ghosted_nodes_that_are_now_shared[i];
int proc = bulk.parallel_owner_rank(node);
comm.send_buffer(proc).pack<stk::mesh::EntityKey>(bulk.entity_key(node));
}
if (phase == 0 )
{
comm.allocate_buffers();
}
else
{
comm.communicate();
}
}
stk::mesh::EntityVector sharedNodes;
for (int process=0;process<bulk.parallel_size();++process)
{
while(comm.recv_buffer(process).remaining())
{
stk::mesh::EntityKey key;
comm.recv_buffer(process).unpack<stk::mesh::EntityKey>(key);
stk::mesh::Entity entity = bulk.get_entity(key);
if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) )
{
bulk.add_node_sharing(entity, process);
sharedNodes.push_back(entity);
}
}
}
/////////////////////////
stk::CommSparse commSecondStage(bulk.parallel());
for (int phase=0;phase<2;++phase)
{
for (size_t i=0;i<sharedNodes.size();++i)
{
std::vector<int> procs;
stk::mesh::EntityKey key = bulk.entity_key(sharedNodes[i]);
bulk.comm_shared_procs(key, procs);
for (size_t j=0;j<procs.size();++j)
{
if ( procs[j] != bulk.parallel_rank() )
{
commSecondStage.send_buffer(procs[j]).pack<int>(bulk.parallel_rank()).pack<stk::mesh::EntityKey>(key);
for (size_t k=0;k<procs.size();++k)
{
commSecondStage.send_buffer(procs[j]).pack<int>(procs[k]).pack<stk::mesh::EntityKey>(key);
}
}
}
}
if (phase == 0 )
{
commSecondStage.allocate_buffers();
}
else
{
commSecondStage.communicate();
}
}
for (int proc_that_sent_message=0;proc_that_sent_message<bulk.parallel_size();++proc_that_sent_message)
{
if ( proc_that_sent_message == bulk.parallel_rank() ) continue;
while(commSecondStage.recv_buffer(proc_that_sent_message).remaining())
{
stk::mesh::EntityKey key;
int sharingProc;
commSecondStage.recv_buffer(proc_that_sent_message).unpack<int>(sharingProc).unpack<stk::mesh::EntityKey>(key);
if ( sharingProc != bulk.parallel_rank() )
{
stk::mesh::Entity entity = bulk.get_entity(key);
if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) && !bulk.in_shared(key, sharingProc) )
{
bulk.add_node_sharing(entity, sharingProc);
}
}
}
}
}