void unpack_remote_elem_side(stk::CommSparse& commSparse, int procId, stk::mesh::GraphEdge& recvGraphEdge)
{
stk::mesh::EntityId globalId;
commSparse.recv_buffer(procId).unpack<stk::mesh::EntityId>(globalId);
recvGraphEdge.elem2 = -globalId;
commSparse.recv_buffer(procId).unpack<int>(recvGraphEdge.side2);
}
void allocate_and_send(stk::CommSparse& comm, const std::vector<SideSharingData>& sideSharingDataThisProc, const std::vector<stk::mesh::impl::IdViaSidePair>& idAndSides)
{
pack_data(comm, sideSharingDataThisProc, idAndSides);
comm.allocate_buffers();
pack_data(comm, sideSharingDataThisProc, idAndSides);
comm.communicate();
}
void pack_graph_edge_and_chosen_side_id_to_proc(stk::CommSparse& commSparse, int otherProc, const stk::mesh::GraphEdge& graphEdge, stk::mesh::EntityId chosenSideId)
{
commSparse.send_buffer(otherProc).pack<stk::mesh::EntityId>(graphEdge.elem1);
commSparse.send_buffer(otherProc).pack<int>(graphEdge.side1);
commSparse.send_buffer(otherProc).pack<stk::mesh::EntityId>(graphEdge.elem2);
commSparse.send_buffer(otherProc).pack<int>(graphEdge.side2);
commSparse.send_buffer(otherProc).pack<stk::mesh::EntityId>(chosenSideId);
}
void NoGhostGameofLife::pack_num_active_neighbors_into_buffer(stk::CommSparse& buffer,
int numActive, stk::mesh::EntityKey
remoteElemKey)
{
int procNum = m_remoteElementKeyToOwningProcessor[remoteElemKey];
buffer.send_buffer(procNum).pack<stk::mesh::EntityKey>(remoteElemKey);
buffer.send_buffer(procNum).pack<int>(numActive);
}
void unpack_local_elem_side(stk::CommSparse& commSparse,
int procId,
const IdMapper& idMapper,
stk::mesh::GraphEdge& recvGraphEdge)
{
stk::mesh::EntityId globalId;
commSparse.recv_buffer(procId).unpack<stk::mesh::EntityId>(globalId);
recvGraphEdge.elem1 = idMapper.globalToLocal(globalId);
commSparse.recv_buffer(procId).unpack<int>(recvGraphEdge.side1);
}
void allocate_or_communicate(int iphase, stk::CommSparse& comm)
{
if (iphase == 0)
{
comm.allocate_buffers();
}
else
{
comm.communicate();
}
}
void send_data_to_other_procs(const std::vector<double>& data, stk::CommSparse& comm)
{
int myprocId = comm.parallel_rank();
for(int i=0;i<comm.parallel_size();++i)
{
if(i!=myprocId)
{
stk::pack_vector_to_proc(comm, data, i);
}
}
}
void pack_edge(stk::CommSparse &comm, const ElemElemGraph& graph, const stk::mesh::BulkData& bulkData, const stk::mesh::GraphEdge& edge, int other_proc)
{
stk::mesh::EntityId id1 = bulkData.identifier(graph.get_entity(edge.elem1()));
unsigned side1 = edge.side1();
stk::mesh::EntityId id2 = -edge.elem2();
unsigned side2 = edge.side2();
comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(id1);
comm.send_buffer(other_proc).pack<unsigned>(side1);
comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(id2);
comm.send_buffer(other_proc).pack<unsigned>(side2);
}
void NoGhostGameofLife::fill_buffer_with_local_neighbors_of_remote_keys(stk::CommSparse& buffer)
{
for (int phase = 0; phase < 2; phase++)
{
for (stk::mesh::EntityKey remoteElemKey : m_remoteElementKeys)
fill_buffer_with_local_neighbors_of_remote_element_key(remoteElemKey, buffer);
if (0 == phase)
buffer.allocate_buffers();
else
buffer.communicate();
}
}
void NoGhostGameofLife::send_num_active_neighbors_of_remote_elem_keys(stk::CommSparse& buffer)
{
for (int phase = 0 ; phase < 2; phase++)
{
for (stk::mesh::EntityKey remoteElemKey : m_remoteElementKeysToVisit)
pack_number_of_local_neighbors_of_remote_element_into_buffer(buffer, remoteElemKey);
if (0 == phase)
buffer.allocate_buffers();
else if (1 == phase)
buffer.communicate();
}
}
void NoGhostGameofLife::fill_buffer_with_local_neighbors_of_remote_element_key(stk::mesh::EntityKey
remoteKey,
stk::CommSparse&
buffer)
{
int procNum = m_remoteElementKeyToOwningProcessor[remoteKey];
size_t numNeighbors = m_remoteElementKeyToLocalNeighborElements[remoteKey].size();
buffer.send_buffer(procNum).pack<stk::mesh::EntityKey>(remoteKey);
buffer.send_buffer(procNum).pack<size_t>(numNeighbors);
for (stk::mesh::Entity localElem : m_remoteElementKeyToLocalNeighborElements[remoteKey])
buffer.send_buffer(procNum).pack<stk::mesh::EntityKey>(m_bulkData->entity_key(localElem));
}
void NoGhostGameofLife::fill_buffer_with_local_element_keys_and_remote_node_keys(stk::CommSparse&
buffer)
{
for (int phase = 0; phase < 2; phase++)
{
for (stk::mesh::Entity elem : m_elements)
fill_buffer_with_this_elements_info(elem, buffer);
if (0 == phase)
buffer.allocate_buffers();
else
buffer.communicate();
}
}
void NoGhostGameofLife::communicate_remote_element_keys_to_check(stk::CommSparse& buffer)
{
for (int phase = 0; phase < 2; phase++)
{
for (stk::mesh::EntityKey remoteElemKey : m_remoteElementKeysToVisit)
buffer.send_buffer(m_remoteElementKeyToOwningProcessor[remoteElemKey]).
pack<stk::mesh::EntityKey>(remoteElemKey);
if (0 == phase)
buffer.allocate_buffers();
else
buffer.communicate();
}
}
void NoGhostGameofLife::fill_buffer_with_map_info(stk::mesh::Entity elem, stk::CommSparse& buffer,
std::unordered_map<int,std::unordered_set
<stk::mesh::EntityKey, std::hash
<stk::mesh::EntityKey>>>& map)
{
for (std::pair< const int,std::unordered_set<stk::mesh::EntityKey,
std::hash<stk::mesh::EntityKey>>>& pair : map)
{
int remoteProc = pair.first;
buffer.send_buffer(remoteProc).pack<stk::mesh::EntityKey>(m_bulkData->entity_key(elem));
buffer.send_buffer(remoteProc).pack<size_t>(pair.second.size());
for (stk::mesh::EntityKey nodeKey : pair.second)
buffer.send_buffer(remoteProc).pack<stk::mesh::EntityKey>(nodeKey);
}
}
stk::mesh::GraphEdge unpack_edge(stk::CommSparse& comm, const stk::mesh::BulkData& bulkData, const ElemElemGraph& graph, int proc_id)
{
stk::mesh::EntityId id1 = 0, id2 = 0;
unsigned side1 = 0, side2 = 0;
comm.recv_buffer(proc_id).unpack<stk::mesh::EntityId>(id1);
comm.recv_buffer(proc_id).unpack<unsigned>(side1);
comm.recv_buffer(proc_id).unpack<stk::mesh::EntityId>(id2);
comm.recv_buffer(proc_id).unpack<unsigned>(side2);
stk::mesh::Entity element = bulkData.get_entity(stk::topology::ELEM_RANK, id2);
ThrowRequireWithSierraHelpMsg(bulkData.is_valid(element));
stk::mesh::impl::LocalId localId2 = graph.get_local_element_id(element);
stk::mesh::GraphEdge edge(localId2, side2, -id1, side1);
return edge;
}
void pack_data_for_part_ordinals(stk::CommSparse &comm, const ElemElemGraph& graph, const stk::mesh::BulkData& bulkData)
{
const stk::mesh::impl::ParallelGraphInfo& parallel_info = graph.get_parallel_graph().get_parallel_graph_info();
for(const auto& item : parallel_info)
{
const stk::mesh::GraphEdge &edge = item.first;
const stk::mesh::impl::ParallelInfo &pinfo = item.second;
stk::mesh::Entity local_element = graph.get_entity(edge.elem1());
std::vector<stk::mesh::PartOrdinal> partOrdinals = stk::mesh::impl::get_element_block_part_ordinals(local_element, bulkData);
pack_edge(comm, graph, bulkData, edge, pinfo.get_proc_rank_of_neighbor());
comm.send_buffer(pinfo.get_proc_rank_of_neighbor()).pack<size_t>(partOrdinals.size());
for(stk::mesh::PartOrdinal partOrdinal : partOrdinals)
comm.send_buffer(pinfo.get_proc_rank_of_neighbor()).pack<stk::mesh::PartOrdinal>(partOrdinal);
}
}
void NoGhostGameofLife::unpack_remote_elem_key_info_from_buffer(stk::CommSparse& buffer)
{
for (int proc = 0; proc < m_numProcs; proc++)
{
stk::CommBuffer& buf = buffer.recv_buffer(proc);
while (buf.remaining())
unpack_remote_info_from_this_processor(proc, buf);
}
}
void unpack_and_update_part_ordinals(stk::CommSparse &comm, const stk::mesh::BulkData& bulkData, const ElemElemGraph& graph, ParallelPartInfo ¶llelPartInfo)
{
for(int i=0;i<bulkData.parallel_size();++i)
{
while(comm.recv_buffer(i).remaining())
{
stk::mesh::GraphEdge edge = unpack_edge(comm, bulkData, graph, i);
size_t num_ordinals = 0;
comm.recv_buffer(i).unpack<size_t>(num_ordinals);
std::vector<stk::mesh::PartOrdinal> partOrdinals(num_ordinals);
for(stk::mesh::PartOrdinal &partOrdinal : partOrdinals)
comm.recv_buffer(i).unpack<stk::mesh::PartOrdinal>(partOrdinal);
parallelPartInfo[edge.elem2()] = partOrdinals;
}
}
}
void NoGhostGameofLife::recieve_num_active_neighbors_of_local_elements(stk::CommSparse& buffer)
{
for (int procNum = 0; procNum < m_numProcs; procNum++)
{
stk::CommBuffer& buf = buffer.recv_buffer(procNum);
while(buf.remaining())
update_local_element_with_remote_neighbor_data(buf);
}
}
void NoGhostGameofLife::unpack_local_and_remote_key_info_from_each_processor(stk::CommSparse&
buffer)
{
for (int procRank = 0; procRank < m_numProcs; procRank++)
{
stk::CommBuffer& buf = buffer.recv_buffer(procRank);
while (buf.remaining())
unpack_local_and_remote_keys_from_buffer(buf);
}
}
void NoGhostGameofLife::recieve_local_element_keys_to_check(stk::CommSparse& buffer)
{
for (int procNum = 0; procNum < m_numProcs; procNum++)
{
stk::CommBuffer& buf = buffer.recv_buffer(procNum);
while (buf.remaining())
m_localElementsToVisit.insert(m_bulkData->
get_entity(stk::unpack<stk::mesh::EntityKey>(buf)));
}
}
void unpack_data(stk::CommSparse& comm, int my_proc_id, int num_procs, std::vector<SideSharingData>& sideSharingDataThisProc)
{
for(int i=0;i<num_procs;++i)
{
while(comm.recv_buffer(i).remaining())
{
stk::mesh::impl::IdViaSidePair receivedIdSide;
stk::mesh::EntityId chosenId;
comm.recv_buffer(i).unpack<stk::mesh::EntityId>(receivedIdSide.id);
comm.recv_buffer(i).unpack<int>(receivedIdSide.side);
comm.recv_buffer(i).unpack<stk::mesh::EntityId>(chosenId);
SideSharingData localTemp;
localTemp.elementAndSide = receivedIdSide;
localTemp.owningProc = std::min(my_proc_id, i);;
localTemp.sharingProc = i;
localTemp.chosenSideId = chosenId;
unpack_vector(comm.recv_buffer(i), localTemp.sideNodes);
unpack_vector(comm.recv_buffer(i), localTemp.partOrdinals);
sideSharingDataThisProc.push_back(localTemp);
}
}
}
void pack_data(stk::CommSparse& comm, const std::vector<SideSharingData>& sideSharingDataThisProc, const std::vector<stk::mesh::impl::IdViaSidePair>& idAndSides)
{
for(size_t i=0;i<idAndSides.size();++i)
{
int other_proc = sideSharingDataThisProc[i].sharingProc;
comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(idAndSides[i].id);
comm.send_buffer(other_proc).pack<int>(idAndSides[i].side);
comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(sideSharingDataThisProc[i].chosenSideId);
comm.send_buffer(other_proc).pack<size_t>(sideSharingDataThisProc[i].sideNodes.size());
for(stk::mesh::EntityId nodeId : sideSharingDataThisProc[i].sideNodes) {
comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(nodeId);
}
comm.send_buffer(other_proc).pack<size_t>(sideSharingDataThisProc[i].partOrdinals.size());
for(stk::mesh::PartOrdinal partOrd : sideSharingDataThisProc[i].partOrdinals) {
comm.send_buffer(other_proc).pack<stk::mesh::PartOrdinal>(partOrd);
}
}
}
void unpack_selector_value(stk::CommSparse& comm, int rank, RemoteSelectedValue &remoteSelectedValue)
{
int64_t id;
comm.recv_buffer(rank).unpack<int64_t>(id);
remoteSelectedValue.set_id_as_selected(id);
}
void pack_vector_to_proc(stk::CommSparse& comm, const T& data, int otherProc)
{
comm.send_buffer(otherProc).pack<unsigned>(data.size());
for(size_t i=0; i<data.size(); ++i)
comm.send_buffer(otherProc).pack<typename T::value_type>(data[i]);
}
void pack_selected_value_for_par_info(stk::CommSparse &comm, int procRank, const stk::mesh::BulkData& bulkData, stk::mesh::Entity local_element, stk::mesh::Selector sel)
{
if(sel(bulkData.bucket(local_element)))
comm.send_buffer(procRank).pack<int64_t>(bulkData.identifier(local_element));
}
stk::mesh::EntityId unpack_chosen_side_id(stk::CommSparse &commSparse, int procId)
{
stk::mesh::EntityId chosenSideId;
commSparse.recv_buffer(procId).unpack<stk::mesh::EntityId>(chosenSideId);
return chosenSideId;
}