void unpack_remote_elem_side(stk::CommSparse& commSparse, int procId, stk::mesh::GraphEdge& recvGraphEdge) { stk::mesh::EntityId globalId; commSparse.recv_buffer(procId).unpack<stk::mesh::EntityId>(globalId); recvGraphEdge.elem2 = -globalId; commSparse.recv_buffer(procId).unpack<int>(recvGraphEdge.side2); }
void allocate_and_send(stk::CommSparse& comm, const std::vector<SideSharingData>& sideSharingDataThisProc, const std::vector<stk::mesh::impl::IdViaSidePair>& idAndSides) { pack_data(comm, sideSharingDataThisProc, idAndSides); comm.allocate_buffers(); pack_data(comm, sideSharingDataThisProc, idAndSides); comm.communicate(); }
void pack_graph_edge_and_chosen_side_id_to_proc(stk::CommSparse& commSparse, int otherProc, const stk::mesh::GraphEdge& graphEdge, stk::mesh::EntityId chosenSideId) { commSparse.send_buffer(otherProc).pack<stk::mesh::EntityId>(graphEdge.elem1); commSparse.send_buffer(otherProc).pack<int>(graphEdge.side1); commSparse.send_buffer(otherProc).pack<stk::mesh::EntityId>(graphEdge.elem2); commSparse.send_buffer(otherProc).pack<int>(graphEdge.side2); commSparse.send_buffer(otherProc).pack<stk::mesh::EntityId>(chosenSideId); }
void NoGhostGameofLife::pack_num_active_neighbors_into_buffer(stk::CommSparse& buffer, int numActive, stk::mesh::EntityKey remoteElemKey) { int procNum = m_remoteElementKeyToOwningProcessor[remoteElemKey]; buffer.send_buffer(procNum).pack<stk::mesh::EntityKey>(remoteElemKey); buffer.send_buffer(procNum).pack<int>(numActive); }
void unpack_local_elem_side(stk::CommSparse& commSparse, int procId, const IdMapper& idMapper, stk::mesh::GraphEdge& recvGraphEdge) { stk::mesh::EntityId globalId; commSparse.recv_buffer(procId).unpack<stk::mesh::EntityId>(globalId); recvGraphEdge.elem1 = idMapper.globalToLocal(globalId); commSparse.recv_buffer(procId).unpack<int>(recvGraphEdge.side1); }
void allocate_or_communicate(int iphase, stk::CommSparse& comm) { if (iphase == 0) { comm.allocate_buffers(); } else { comm.communicate(); } }
void send_data_to_other_procs(const std::vector<double>& data, stk::CommSparse& comm) { int myprocId = comm.parallel_rank(); for(int i=0;i<comm.parallel_size();++i) { if(i!=myprocId) { stk::pack_vector_to_proc(comm, data, i); } } }
void pack_edge(stk::CommSparse &comm, const ElemElemGraph& graph, const stk::mesh::BulkData& bulkData, const stk::mesh::GraphEdge& edge, int other_proc) { stk::mesh::EntityId id1 = bulkData.identifier(graph.get_entity(edge.elem1())); unsigned side1 = edge.side1(); stk::mesh::EntityId id2 = -edge.elem2(); unsigned side2 = edge.side2(); comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(id1); comm.send_buffer(other_proc).pack<unsigned>(side1); comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(id2); comm.send_buffer(other_proc).pack<unsigned>(side2); }
void NoGhostGameofLife::fill_buffer_with_local_neighbors_of_remote_keys(stk::CommSparse& buffer) { for (int phase = 0; phase < 2; phase++) { for (stk::mesh::EntityKey remoteElemKey : m_remoteElementKeys) fill_buffer_with_local_neighbors_of_remote_element_key(remoteElemKey, buffer); if (0 == phase) buffer.allocate_buffers(); else buffer.communicate(); } }
void NoGhostGameofLife::send_num_active_neighbors_of_remote_elem_keys(stk::CommSparse& buffer) { for (int phase = 0 ; phase < 2; phase++) { for (stk::mesh::EntityKey remoteElemKey : m_remoteElementKeysToVisit) pack_number_of_local_neighbors_of_remote_element_into_buffer(buffer, remoteElemKey); if (0 == phase) buffer.allocate_buffers(); else if (1 == phase) buffer.communicate(); } }
void NoGhostGameofLife::fill_buffer_with_local_neighbors_of_remote_element_key(stk::mesh::EntityKey remoteKey, stk::CommSparse& buffer) { int procNum = m_remoteElementKeyToOwningProcessor[remoteKey]; size_t numNeighbors = m_remoteElementKeyToLocalNeighborElements[remoteKey].size(); buffer.send_buffer(procNum).pack<stk::mesh::EntityKey>(remoteKey); buffer.send_buffer(procNum).pack<size_t>(numNeighbors); for (stk::mesh::Entity localElem : m_remoteElementKeyToLocalNeighborElements[remoteKey]) buffer.send_buffer(procNum).pack<stk::mesh::EntityKey>(m_bulkData->entity_key(localElem)); }
void NoGhostGameofLife::fill_buffer_with_local_element_keys_and_remote_node_keys(stk::CommSparse& buffer) { for (int phase = 0; phase < 2; phase++) { for (stk::mesh::Entity elem : m_elements) fill_buffer_with_this_elements_info(elem, buffer); if (0 == phase) buffer.allocate_buffers(); else buffer.communicate(); } }
void NoGhostGameofLife::communicate_remote_element_keys_to_check(stk::CommSparse& buffer) { for (int phase = 0; phase < 2; phase++) { for (stk::mesh::EntityKey remoteElemKey : m_remoteElementKeysToVisit) buffer.send_buffer(m_remoteElementKeyToOwningProcessor[remoteElemKey]). pack<stk::mesh::EntityKey>(remoteElemKey); if (0 == phase) buffer.allocate_buffers(); else buffer.communicate(); } }
void NoGhostGameofLife::fill_buffer_with_map_info(stk::mesh::Entity elem, stk::CommSparse& buffer, std::unordered_map<int,std::unordered_set <stk::mesh::EntityKey, std::hash <stk::mesh::EntityKey>>>& map) { for (std::pair< const int,std::unordered_set<stk::mesh::EntityKey, std::hash<stk::mesh::EntityKey>>>& pair : map) { int remoteProc = pair.first; buffer.send_buffer(remoteProc).pack<stk::mesh::EntityKey>(m_bulkData->entity_key(elem)); buffer.send_buffer(remoteProc).pack<size_t>(pair.second.size()); for (stk::mesh::EntityKey nodeKey : pair.second) buffer.send_buffer(remoteProc).pack<stk::mesh::EntityKey>(nodeKey); } }
stk::mesh::GraphEdge unpack_edge(stk::CommSparse& comm, const stk::mesh::BulkData& bulkData, const ElemElemGraph& graph, int proc_id) { stk::mesh::EntityId id1 = 0, id2 = 0; unsigned side1 = 0, side2 = 0; comm.recv_buffer(proc_id).unpack<stk::mesh::EntityId>(id1); comm.recv_buffer(proc_id).unpack<unsigned>(side1); comm.recv_buffer(proc_id).unpack<stk::mesh::EntityId>(id2); comm.recv_buffer(proc_id).unpack<unsigned>(side2); stk::mesh::Entity element = bulkData.get_entity(stk::topology::ELEM_RANK, id2); ThrowRequireWithSierraHelpMsg(bulkData.is_valid(element)); stk::mesh::impl::LocalId localId2 = graph.get_local_element_id(element); stk::mesh::GraphEdge edge(localId2, side2, -id1, side1); return edge; }
void pack_data_for_part_ordinals(stk::CommSparse &comm, const ElemElemGraph& graph, const stk::mesh::BulkData& bulkData) { const stk::mesh::impl::ParallelGraphInfo& parallel_info = graph.get_parallel_graph().get_parallel_graph_info(); for(const auto& item : parallel_info) { const stk::mesh::GraphEdge &edge = item.first; const stk::mesh::impl::ParallelInfo &pinfo = item.second; stk::mesh::Entity local_element = graph.get_entity(edge.elem1()); std::vector<stk::mesh::PartOrdinal> partOrdinals = stk::mesh::impl::get_element_block_part_ordinals(local_element, bulkData); pack_edge(comm, graph, bulkData, edge, pinfo.get_proc_rank_of_neighbor()); comm.send_buffer(pinfo.get_proc_rank_of_neighbor()).pack<size_t>(partOrdinals.size()); for(stk::mesh::PartOrdinal partOrdinal : partOrdinals) comm.send_buffer(pinfo.get_proc_rank_of_neighbor()).pack<stk::mesh::PartOrdinal>(partOrdinal); } }
void NoGhostGameofLife::unpack_remote_elem_key_info_from_buffer(stk::CommSparse& buffer) { for (int proc = 0; proc < m_numProcs; proc++) { stk::CommBuffer& buf = buffer.recv_buffer(proc); while (buf.remaining()) unpack_remote_info_from_this_processor(proc, buf); } }
void unpack_and_update_part_ordinals(stk::CommSparse &comm, const stk::mesh::BulkData& bulkData, const ElemElemGraph& graph, ParallelPartInfo ¶llelPartInfo) { for(int i=0;i<bulkData.parallel_size();++i) { while(comm.recv_buffer(i).remaining()) { stk::mesh::GraphEdge edge = unpack_edge(comm, bulkData, graph, i); size_t num_ordinals = 0; comm.recv_buffer(i).unpack<size_t>(num_ordinals); std::vector<stk::mesh::PartOrdinal> partOrdinals(num_ordinals); for(stk::mesh::PartOrdinal &partOrdinal : partOrdinals) comm.recv_buffer(i).unpack<stk::mesh::PartOrdinal>(partOrdinal); parallelPartInfo[edge.elem2()] = partOrdinals; } } }
void NoGhostGameofLife::recieve_num_active_neighbors_of_local_elements(stk::CommSparse& buffer) { for (int procNum = 0; procNum < m_numProcs; procNum++) { stk::CommBuffer& buf = buffer.recv_buffer(procNum); while(buf.remaining()) update_local_element_with_remote_neighbor_data(buf); } }
void NoGhostGameofLife::unpack_local_and_remote_key_info_from_each_processor(stk::CommSparse& buffer) { for (int procRank = 0; procRank < m_numProcs; procRank++) { stk::CommBuffer& buf = buffer.recv_buffer(procRank); while (buf.remaining()) unpack_local_and_remote_keys_from_buffer(buf); } }
void NoGhostGameofLife::recieve_local_element_keys_to_check(stk::CommSparse& buffer) { for (int procNum = 0; procNum < m_numProcs; procNum++) { stk::CommBuffer& buf = buffer.recv_buffer(procNum); while (buf.remaining()) m_localElementsToVisit.insert(m_bulkData-> get_entity(stk::unpack<stk::mesh::EntityKey>(buf))); } }
void unpack_data(stk::CommSparse& comm, int my_proc_id, int num_procs, std::vector<SideSharingData>& sideSharingDataThisProc) { for(int i=0;i<num_procs;++i) { while(comm.recv_buffer(i).remaining()) { stk::mesh::impl::IdViaSidePair receivedIdSide; stk::mesh::EntityId chosenId; comm.recv_buffer(i).unpack<stk::mesh::EntityId>(receivedIdSide.id); comm.recv_buffer(i).unpack<int>(receivedIdSide.side); comm.recv_buffer(i).unpack<stk::mesh::EntityId>(chosenId); SideSharingData localTemp; localTemp.elementAndSide = receivedIdSide; localTemp.owningProc = std::min(my_proc_id, i);; localTemp.sharingProc = i; localTemp.chosenSideId = chosenId; unpack_vector(comm.recv_buffer(i), localTemp.sideNodes); unpack_vector(comm.recv_buffer(i), localTemp.partOrdinals); sideSharingDataThisProc.push_back(localTemp); } } }
void pack_data(stk::CommSparse& comm, const std::vector<SideSharingData>& sideSharingDataThisProc, const std::vector<stk::mesh::impl::IdViaSidePair>& idAndSides) { for(size_t i=0;i<idAndSides.size();++i) { int other_proc = sideSharingDataThisProc[i].sharingProc; comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(idAndSides[i].id); comm.send_buffer(other_proc).pack<int>(idAndSides[i].side); comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(sideSharingDataThisProc[i].chosenSideId); comm.send_buffer(other_proc).pack<size_t>(sideSharingDataThisProc[i].sideNodes.size()); for(stk::mesh::EntityId nodeId : sideSharingDataThisProc[i].sideNodes) { comm.send_buffer(other_proc).pack<stk::mesh::EntityId>(nodeId); } comm.send_buffer(other_proc).pack<size_t>(sideSharingDataThisProc[i].partOrdinals.size()); for(stk::mesh::PartOrdinal partOrd : sideSharingDataThisProc[i].partOrdinals) { comm.send_buffer(other_proc).pack<stk::mesh::PartOrdinal>(partOrd); } } }
void unpack_selector_value(stk::CommSparse& comm, int rank, RemoteSelectedValue &remoteSelectedValue) { int64_t id; comm.recv_buffer(rank).unpack<int64_t>(id); remoteSelectedValue.set_id_as_selected(id); }
void pack_vector_to_proc(stk::CommSparse& comm, const T& data, int otherProc) { comm.send_buffer(otherProc).pack<unsigned>(data.size()); for(size_t i=0; i<data.size(); ++i) comm.send_buffer(otherProc).pack<typename T::value_type>(data[i]); }
void pack_selected_value_for_par_info(stk::CommSparse &comm, int procRank, const stk::mesh::BulkData& bulkData, stk::mesh::Entity local_element, stk::mesh::Selector sel) { if(sel(bulkData.bucket(local_element))) comm.send_buffer(procRank).pack<int64_t>(bulkData.identifier(local_element)); }
stk::mesh::EntityId unpack_chosen_side_id(stk::CommSparse &commSparse, int procId) { stk::mesh::EntityId chosenSideId; commSparse.recv_buffer(procId).unpack<stk::mesh::EntityId>(chosenSideId); return chosenSideId; }