void Vertices::fillFieldVertexWeights(const stk::balance::BalanceSettings& balanceSettings,
const stk::mesh::BulkData& stkMeshBulkData,
const std::vector<stk::mesh::Selector>& selectors,
const stk::mesh::EntityVector &entitiesToBalance)
{
unsigned numSelectors = selectors.size();
unsigned numEntities = entitiesToBalance.size();
unsigned numCriteria = balanceSettings.getNumCriteria();
mVertexWeights.resize(numSelectors*numEntities*numCriteria);
for (double &weight : mVertexWeights) { weight = 0.0; }
for(size_t i=0;i<entitiesToBalance.size();++i)
{
for (unsigned sel=0 ; sel<numSelectors ; ++sel)
{
stk::mesh::Selector selector = selectors[sel];
if (selector(stkMeshBulkData.bucket(entitiesToBalance[i])))
{
for(size_t weight_index=0;weight_index<numCriteria;weight_index++)
{
unsigned index = stk::balance::internal::get_index(numSelectors, numCriteria, i, sel, weight_index);
mVertexWeights[index] = balanceSettings.getGraphVertexWeight(entitiesToBalance[i], weight_index);;
}
}
}
}
}
inline stk::mesh::Entity get_face_between_element_ids(stk::mesh::ElemElemGraph& graph, stk::mesh::BulkData& bulkData, stk::mesh::EntityId elem1Id, stk::mesh::EntityId elem2Id)
{
stk::mesh::Entity elem1 = bulkData.get_entity(stk::topology::ELEM_RANK, elem1Id);
stk::mesh::Entity elem2 = bulkData.get_entity(stk::topology::ELEM_RANK, elem2Id);
bool isElem1LocallyOwnedAndValid = bulkData.is_valid(elem1) && bulkData.bucket(elem1).owned();
bool isElem2LocallyOwnedAndValid = bulkData.is_valid(elem2) && bulkData.bucket(elem2).owned();
stk::mesh::Entity face_between_elem1_and_elem2;
if(isElem1LocallyOwnedAndValid && isElem2LocallyOwnedAndValid)
{
int side = graph.get_side_from_element1_to_locally_owned_element2(elem1, elem2);
EXPECT_TRUE(side != -1);
face_between_elem1_and_elem2 = stk::mesh::impl::get_side_for_element(bulkData, elem1, side);
}
else if(isElem1LocallyOwnedAndValid)
{
int side = graph.get_side_from_element1_to_remote_element2(elem1, elem2Id);
EXPECT_TRUE(side != -1);
face_between_elem1_and_elem2 = stk::mesh::impl::get_side_for_element(bulkData, elem1, side);
}
else if(isElem2LocallyOwnedAndValid)
{
int side = graph.get_side_from_element1_to_remote_element2(elem2, elem1Id);
EXPECT_TRUE(side != -1);
face_between_elem1_and_elem2 = stk::mesh::impl::get_side_for_element(bulkData, elem2, side);
}
return face_between_elem1_and_elem2;
}
void filter_out_invalid_solid_shell_connections(const stk::mesh::BulkData & mesh,
const stk::mesh::Entity localElement,
const unsigned sideOrdinal,
std::vector<SideData> & connectedElementData)
{
stk::topology localElemTopology = mesh.bucket(localElement).topology();
if (localElemTopology.is_shell())
{
std::vector<SideData> filteredConnectedElements;
for (const SideData & connectedElem: connectedElementData)
{
if(mesh.identifier(localElement) != connectedElem.get_element_identifier())
{
if(connectedElem.get_element_topology().is_shell())
add_shell_element_if_coincident(mesh, sideOrdinal, localElement, connectedElem, filteredConnectedElements);
else
add_solid_element_if_normals_oppose_to_shell(mesh, sideOrdinal, localElement, connectedElem, filteredConnectedElements);
}
}
connectedElementData.swap(filteredConnectedElements);
}
else
{
add_shell_connections_to_this_solid_if_normals_oppose(mesh, localElement, sideOrdinal, connectedElementData);
}
}
void performNodalMeshReduction(
stk::mesh::Part &samplePart,
stk::mesh::BulkData& bulkData)
{
const stk::mesh::MetaData &metaData = stk::mesh::MetaData::get(bulkData);
std::vector<stk::mesh::Entity> sampleNodes;
stk::mesh::get_selected_entities(samplePart, bulkData.buckets(stk::topology::NODE_RANK), sampleNodes);
const stk::mesh::Selector locallyOwned = stk::mesh::MetaData::get(bulkData).locally_owned_part();
std::vector<stk::mesh::Entity> relatedEntities;
typedef std::vector<stk::mesh::Entity>::const_iterator EntityIterator;
for (EntityIterator it(sampleNodes.begin()), it_end(sampleNodes.end()); it != it_end; ++it) {
for (stk::mesh::EntityRank r = stk::topology::NODE_RANK; r < metaData.entity_rank_count(); ++r) {
stk::mesh::Entity const* relations = bulkData.begin(*it, r);
const int num_rels = bulkData.num_connectivity(*it, r);
for (int i = 0; i < num_rels; ++i) {
stk::mesh::Entity relatedEntity = relations[i];
if (bulkData.is_valid(relatedEntity) && locallyOwned(bulkData.bucket(relatedEntity))) {
relatedEntities.push_back(relatedEntity);
}
}
}
}
std::sort(relatedEntities.begin(), relatedEntities.end(), stk::mesh::EntityLess(bulkData));
relatedEntities.erase(
std::unique(relatedEntities.begin(), relatedEntities.end()),
relatedEntities.end());
std::vector<stk::mesh::Entity> sampleClosure;
stk::mesh::find_closure(bulkData, relatedEntities, sampleClosure);
// Keep only the closure, remove the rest, by decreasing entityRanks
{
const stk::mesh::Selector ownedOrShared = metaData.locally_owned_part() | metaData.globally_shared_part();
EntityIterator allKeepersEnd(sampleClosure.end());
const EntityIterator allKeepersBegin(sampleClosure.begin());
for (size_t candidateRankCount = metaData.entity_rank_count(); candidateRankCount > 0; --candidateRankCount) {
const stk::mesh::EntityRank candidateRank = static_cast<stk::mesh::EntityRank>(candidateRankCount - 1);
const EntityIterator keepersBegin = std::lower_bound(allKeepersBegin, allKeepersEnd,
stk::mesh::EntityKey(candidateRank, 0),
stk::mesh::EntityLess(bulkData));
const EntityIterator keepersEnd = allKeepersEnd;
std::vector<stk::mesh::Entity> candidates;
stk::mesh::get_selected_entities(ownedOrShared, bulkData.buckets(candidateRank), candidates);
{
BulkModification modification(bulkData);
std::set_difference(candidates.begin(), candidates.end(),
keepersBegin.base(), keepersEnd.base(),
EntityDestructor(modification),
stk::mesh::EntityLess(bulkData));
}
allKeepersEnd = keepersBegin;
}
}
}
std::string get_non_unique_key_messages(const stk::mesh::BulkData& bulkData, const std::vector<stk::mesh::EntityKeyProc> &badKeyProcs)
{
std::ostringstream os;
for(const stk::mesh::EntityKeyProc& keyProc : badKeyProcs)
{
stk::mesh::Entity entity = bulkData.get_entity(keyProc.first);
os << "[" << bulkData.parallel_rank() << "] Key " << keyProc.first <<
get_topology(bulkData.bucket(entity).topology()) << "is also present (inappropriately) on processor " <<
keyProc.second << "." << std::endl;
}
return os.str();
}
OrdinalAndPermutation
get_ordinal_and_permutation(const stk::mesh::BulkData& mesh, stk::mesh::Entity parent_entity, stk::mesh::EntityRank to_rank, const stk::mesh::EntityVector &nodes_of_sub_rank)
{
std::pair<stk::mesh::ConnectivityOrdinal, stk::mesh::Permutation> ordinalAndPermutation = std::make_pair(stk::mesh::INVALID_CONNECTIVITY_ORDINAL,
stk::mesh::INVALID_PERMUTATION);
unsigned nodes_of_sub_rank_size = nodes_of_sub_rank.size();
const Entity* elemNodes = mesh.begin_nodes(parent_entity);
stk::topology elemTopology = mesh.bucket(parent_entity).topology();
unsigned num_entities_of_sub_topology = elemTopology.num_sub_topology(to_rank);
unsigned max_nodes_possible = 100;
stk::mesh::EntityVector nodes_of_sub_topology;
nodes_of_sub_topology.reserve(max_nodes_possible);
std::pair<bool, unsigned> result;
for (unsigned i=0;i<num_entities_of_sub_topology;++i)
{
stk::topology sub_topology = elemTopology.sub_topology(to_rank, i);
unsigned num_nodes = sub_topology.num_nodes();
if (num_nodes != nodes_of_sub_rank_size)
{
continue;
}
ThrowRequireMsg(num_nodes == nodes_of_sub_rank.size(), "AHA! num_nodes != nodes_of_sub_rank.size()");
ThrowRequireMsg(num_nodes<=max_nodes_possible, "Program error. Exceeded expected array dimensions. Contact sierra-help for support.");
nodes_of_sub_topology.resize(num_nodes);
elemTopology.sub_topology_nodes(elemNodes, to_rank, i, nodes_of_sub_topology.begin());
if (!elemTopology.is_shell() || (to_rank == stk::topology::EDGE_RANK))
{
result = sub_topology.equivalent(nodes_of_sub_rank, nodes_of_sub_topology);
}
else
{
result = sub_topology.equivalent(nodes_of_sub_rank, nodes_of_sub_topology);
if (result.first && result.second >= sub_topology.num_positive_permutations())
{
result.first = false;
}
}
if (result.first == true)
{
ordinalAndPermutation.first = static_cast<stk::mesh::ConnectivityOrdinal>(i);
ordinalAndPermutation.second = static_cast<stk::mesh::Permutation>(result.second);
}
}
return ordinalAndPermutation;
}
inline size_t get_number_sides_in_sideset(const stk::mesh::BulkData& bulk,
int sideset_id,
stk::mesh::Selector selector,
stk::topology stk_element_topology,
const stk::mesh::BucketVector& buckets)
{
if (bulk.has_sideset_data())
{
selector &= ( bulk.mesh_meta_data().locally_owned_part() | bulk.mesh_meta_data().globally_shared_part());
size_t num_sides = 0;
const stk::mesh::SideSet& sset = bulk.get_sideset_data(sideset_id);
for(const stk::mesh::SideSetEntry& elem_and_side : sset)
{
stk::mesh::Entity element = elem_and_side.element;
stk::mesh::Entity side = stk::mesh::get_side_entity_for_elem_side_pair(bulk, element, elem_and_side.side);
if(bulk.is_valid(side))
{
if(selector(bulk.bucket(side)))
{
if(stk_element_topology == stk::topology::INVALID_TOPOLOGY ||
stk_element_topology == bulk.bucket(element).topology())
{
++num_sides;
}
}
}
}
return num_sides;
}
else
{
selector &= bulk.mesh_meta_data().locally_owned_part();
return count_selected_entities(selector, buckets);
}
}
bool bucket_part_memberships_match(const stk::mesh::BulkData& bulk, stk::EnvData& env_data)
{
int numGlobalDiffs = bucket_counts_match(bulk, env_data);
if(numGlobalDiffs > 0)
{
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::CommSparse comm(env_data.m_worldComm);
stk::CommBuffer &buff = stk::diff::get_comm_buffer_for_destination_proc(comm);
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
stk::mesh::EntityVector entities;
stk::mesh::get_entities(bulk, rank, entities);
for(int iphase = 0; iphase < 2; ++iphase)
{
for(size_t i=0;i<entities.size();++i)
{
const stk::mesh::PartVector& parts = bulk.bucket(entities[i]).supersets();
std::string part_names_for_entity = create_string_from_parts(parts);
std::string string_to_send;
if(irank != 1 && irank != 2)
{
string_to_send = std::to_string(bulk.identifier(entities[i])) + " " + part_names_for_entity;
}
else
{
string_to_send = part_names_for_entity;
}
stk::diff::pack_string(buff, string_to_send);
}
stk::diff::allocate_or_communicate(iphase, comm);
}
}
}
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::CommSparse comm(env_data.m_worldComm);
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
const stk::mesh::BucketVector& buckets = bulk.buckets(rank);
for(int iphase = 0; iphase < 2; ++iphase)
{
pack_buckets_parts(buckets, stk::diff::get_comm_buffer_for_destination_proc(comm));
stk::diff::allocate_or_communicate(iphase, comm);
}
}
numGlobalDiffs += get_global_bucket_part_membership_differences(env_data.m_worldComm, 0);
return numGlobalDiffs == 0;
}
void Vertices::fillVertexWeights(const stk::mesh::BulkData& bulkData,
const stk::balance::BalanceSettings& balanceSettings,
const stk::mesh::EntityVector &entities,
const std::vector<stk::mesh::Selector> &selectors)
{
if(balanceSettings.fieldSpecifiedVertexWeights())
{
fillFieldVertexWeights(balanceSettings, bulkData, selectors, entities);
}
else
{
mVertexWeights.resize(entities.size(), 0.0);
for(size_t i=0;i<entities.size();++i)
mVertexWeights[i] = balanceSettings.getGraphVertexWeight(bulkData.bucket(entities[i]).topology());
}
}
void addNodesToPart(
const Teuchos::ArrayView<const stk::mesh::EntityId> &nodeIds,
stk::mesh::Part &samplePart,
stk::mesh::BulkData& bulkData)
{
const stk::mesh::PartVector samplePartVec(1, &samplePart);
const stk::mesh::Selector locallyOwned = stk::mesh::MetaData::get(bulkData).locally_owned_part();
BulkModification mod(bulkData);
typedef Teuchos::ArrayView<const stk::mesh::EntityId>::const_iterator Iter;
for (Iter it = nodeIds.begin(), it_end = nodeIds.end(); it != it_end; ++it) {
stk::mesh::Entity node = bulkData.get_entity(stk::topology::NODE_RANK, *it);
if (bulkData.is_valid(node) && locallyOwned(bulkData.bucket(node))) {
bulkData.change_entity_parts(node, samplePartVec);
}
}
}
std::string get_message_for_split_coincident_elements(const stk::mesh::BulkData& bulkData, const std::map<stk::mesh::EntityId, std::pair<stk::mesh::EntityId, int> > & splitCoincidentElements)
{
std::ostringstream out;
for(const auto& item : splitCoincidentElements) {
stk::mesh::Entity element = bulkData.get_entity(stk::topology::ELEM_RANK,item.first);
const stk::mesh::PartVector& elementParts = bulkData.bucket(element).supersets();
std::string blockNames;
blockNames = "{";
for (const stk::mesh::Part* part : elementParts) {
if (stk::mesh::impl::is_element_block(*part)) {
blockNames += " " + part->name();
}
}
blockNames += " }";
out << "[" << bulkData.parallel_rank() << "] Element " << item.first << " (" << bulkData.bucket(element).topology() << ") in blocks " << blockNames << " is coincident with element " << item.second.first << " on processor " << item.second.second << std::endl;
}
return out.str();
}
void add_shell_connections_to_this_solid_if_normals_oppose(const stk::mesh::BulkData& mesh,
const stk::mesh::Entity localElement,
const unsigned sideOrdinal,
std::vector<SideData>& connectedElementData)
{
std::vector<SideData> filteredConnectedElements;
stk::topology localElemTopology = mesh.bucket(localElement).topology();
stk::topology localSideTopology = localElemTopology.side_topology(sideOrdinal);
bool foundAnySingleElementThatIsEquivalentToLocalElement = false;
const stk::mesh::Entity* localElemNodes = mesh.begin_nodes(localElement);
stk::mesh::EntityVector localElemSideNodes;
localElemSideNodes.resize(localSideTopology.num_nodes());
localElemTopology.side_nodes(localElemNodes, sideOrdinal, localElemSideNodes.begin());
for (const SideData & connectedElem: connectedElementData)
{
std::pair<bool,unsigned> result = localSideTopology.equivalent(localElemSideNodes, connectedElem.get_side_nodes());
const bool isEquivalentNodes = result.first;
foundAnySingleElementThatIsEquivalentToLocalElement = foundAnySingleElementThatIsEquivalentToLocalElement || isEquivalentNodes;
if (connectedElem.get_element_topology().is_shell() && isEquivalentNodes)
{
stk::mesh::OrdinalAndPermutation localElemOrdAndPerm = stk::mesh::get_ordinal_and_permutation(mesh,
localElement,
mesh.mesh_meta_data().side_rank(),
connectedElem.get_side_nodes());
bool localNegativeRelativeFacePolarity = !localSideTopology.is_positive_polarity(localElemOrdAndPerm.second);
if (localNegativeRelativeFacePolarity)
{
filteredConnectedElements.push_back(connectedElem);
}
}
}
if (!filteredConnectedElements.empty())
connectedElementData.swap(filteredConnectedElements);
if (!foundAnySingleElementThatIsEquivalentToLocalElement)
connectedElementData.clear();
}
void fill_sharing_data(stk::mesh::BulkData& bulkData, stk::mesh::ElemElemGraph &graph, const stk::mesh::EntityVector& sidesThatNeedFixing, std::vector<SideSharingData>& sideSharingDataThisProc, std::vector<stk::mesh::impl::IdViaSidePair>& idAndSides)
{
// Element 1, side 5: face 15
// Element 2, side 3: face 23
// Are these faces the same? Yes: delete face 23, then connect face 15 to element 2 with negative permutation
const stk::mesh::PartOrdinal sharedOrd = bulkData.mesh_meta_data().globally_shared_part().mesh_meta_data_ordinal();
for(size_t i=0;i<sidesThatNeedFixing.size();++i)
{
stk::mesh::impl::ElementViaSidePair elementAndSide = get_element_and_side_ordinal(bulkData, sidesThatNeedFixing[i]);
stk::mesh::impl::LocalId localElemId = graph.get_local_element_id(elementAndSide.element);
for(const stk::mesh::GraphEdge& edge : graph.get_edges_for_element(localElemId))
{
if(edge.side1() == elementAndSide.side && edge.elem2() < 0)
{
const stk::mesh::impl::ParallelInfo &pInfo = graph.get_parallel_info_for_graph_edge(edge);
const stk::mesh::Entity* nodes = bulkData.begin_nodes(sidesThatNeedFixing[i]);
unsigned numNodes = bulkData.num_nodes(sidesThatNeedFixing[i]);
SideSharingData localTemp({bulkData.identifier(elementAndSide.element), elementAndSide.side},
sidesThatNeedFixing[i],
pInfo.get_proc_rank_of_neighbor(),
std::min(bulkData.parallel_rank(),pInfo.get_proc_rank_of_neighbor()),
bulkData.identifier(sidesThatNeedFixing[i]));
localTemp.sideNodes.resize(numNodes);
for(unsigned j=0; j<numNodes; ++j) {
localTemp.sideNodes[j] = bulkData.identifier(nodes[j]);
}
fill_part_ordinals_besides_owned_and_shared(bulkData.bucket(sidesThatNeedFixing[i]), sharedOrd, localTemp.partOrdinals);
sideSharingDataThisProc.push_back(localTemp);
stk::mesh::EntityId localId = -edge.elem2();
idAndSides.push_back({localId, edge.side2()});
}
}
}
}
stk::mesh::impl::ElementViaSidePair find_element_side_ord_for_side(const stk::mesh::BulkData& bulkData, unsigned num_elements, const stk::mesh::Entity* elements, stk::mesh::Entity side)
{
const stk::mesh::MetaData& meta = bulkData.mesh_meta_data();
for(unsigned i=0;i<num_elements;++i)
{
if(bulkData.bucket(elements[i]).owned())
{
const stk::mesh::Entity* sides = bulkData.begin(elements[i], meta.side_rank());
unsigned num_sides = bulkData.num_connectivity(elements[i], meta.side_rank());
for(unsigned j=0;j<num_sides;++j)
{
if(sides[j]==side)
{
const stk::mesh::ConnectivityOrdinal* ordinals = bulkData.begin_ordinals(elements[i], meta.side_rank());
return stk::mesh::impl::ElementViaSidePair{elements[i], static_cast<int>(ordinals[j])};
}
}
}
}
return stk::mesh::impl::ElementViaSidePair{stk::mesh::Entity(),0};
}
void fill_element_and_side_ids(Ioss::GroupingEntity & io,
stk::mesh::Part * const part,
const stk::mesh::BulkData & bulk_data,
stk::topology stk_element_topology,
const stk::mesh::Selector *subset_selector,
stk::mesh::EntityVector &sides,
std::vector<INT>& elem_side_ids)
{
if (bulk_data.has_sideset_data())
{
const stk::mesh::SideSet& sset = bulk_data.get_sideset_data(part->id());
size_t num_sides = sset.size();
elem_side_ids.reserve(num_sides*2);
stk::mesh::Selector selector = *part & ( bulk_data.mesh_meta_data().locally_owned_part() | bulk_data.mesh_meta_data().globally_shared_part() );
if(subset_selector)
selector &= *subset_selector;
for(size_t i=0;i<sset.size();++i)
{
stk::mesh::Entity element = sset[i].element;
stk::mesh::EntityId elemId = bulk_data.identifier(element);
int zero_based_side_ord = sset[i].side;
stk::mesh::Entity side = stk::mesh::get_side_entity_for_elem_id_side_pair_of_rank(bulk_data, elemId, zero_based_side_ord, bulk_data.mesh_meta_data().side_rank());
if(bulk_data.is_valid(side))
{
if(selector(bulk_data.bucket(side)))
{
if(bulk_data.bucket(element).topology() == stk_element_topology)
{
elem_side_ids.push_back(elemId);
elem_side_ids.push_back(zero_based_side_ord+1);
sides.push_back(side);
}
}
}
}
}
else
{
const stk::mesh::MetaData & meta_data = stk::mesh::MetaData::get(*part);
stk::mesh::EntityRank type = part_primary_entity_rank(*part);
size_t num_sides = get_entities(*part, type, bulk_data, sides, false, subset_selector);
elem_side_ids.reserve(num_sides * 2);
stk::mesh::EntityRank elem_rank = stk::topology::ELEMENT_RANK;
for(size_t i = 0; i < num_sides; ++i)
{
std::vector<stk::mesh::Entity> side;
side.push_back(sides[i]);
std::vector<stk::mesh::Entity> side_elements;
std::vector<stk::mesh::Entity> side_nodes(bulk_data.begin_nodes(sides[i]), bulk_data.end_nodes(sides[i]));
get_entities_through_relations(bulk_data, side_nodes, elem_rank, side_elements);
const size_t num_side_elem = side_elements.size();
std::sort(side_elements.begin(), side_elements.end(), stk::mesh::EntityLess(bulk_data));
stk::mesh::Entity suitable_elem = stk::mesh::Entity();
stk::mesh::ConnectivityOrdinal suitable_ordinal = stk::mesh::INVALID_CONNECTIVITY_ORDINAL;
for(size_t j = 0; j < num_side_elem; ++j)
{
const stk::mesh::Entity elem = side_elements[j];
const stk::mesh::Bucket &elemBucket = bulk_data.bucket(elem);
const bool isSelectingEverything = subset_selector == NULL;
const bool isElementBeingOutput = (isSelectingEverything || (*subset_selector)(elemBucket))
&& elemBucket.member(meta_data.locally_owned_part());
if(isElementBeingOutput)
{
const stk::mesh::Entity * elem_sides = bulk_data.begin(elem, type);
stk::mesh::ConnectivityOrdinal const * side_ordinal = bulk_data.begin_ordinals(elem, type);
const size_t num_elem_sides = bulk_data.num_connectivity(elem, type);
for(size_t k = 0; k < num_elem_sides; ++k)
{
if(elem_sides[k] == side[0])
{
suitable_elem = elem;
suitable_ordinal = side_ordinal[k];
break;
}
}
}
}
if(!bulk_data.is_valid(suitable_elem))
{
std::ostringstream oss;
oss << "ERROR, no suitable element found";
throw std::runtime_error(oss.str());
}
elem_side_ids.push_back(bulk_data.identifier(suitable_elem));
elem_side_ids.push_back(suitable_ordinal + 1); // Ioss is 1-based, mesh is 0-based.
}
}
}
void pack_selected_value_for_par_info(stk::CommSparse &comm, int procRank, const stk::mesh::BulkData& bulkData, stk::mesh::Entity local_element, stk::mesh::Selector sel)
{
if(sel(bulkData.bucket(local_element)))
comm.send_buffer(procRank).pack<int64_t>(bulkData.identifier(local_element));
}