inline void createBoundingBoxesForElementsInElementBlocks(const stk::mesh::BulkData &bulk, FlaotBoxVector& domainBoxes)
{
stk::mesh::EntityVector elements;
stk::mesh::get_selected_entities(bulk.mesh_meta_data().locally_owned_part(), bulk.buckets(stk::topology::ELEM_RANK), elements);
size_t numberBoundingBoxes = elements.size();
domainBoxes.resize(numberBoundingBoxes);
stk::mesh::FieldBase const * coords = bulk.mesh_meta_data().coordinate_field();
std::vector<double> boxCoordinates(6);
for(size_t i=0;i<elements.size();++i)
{
unsigned num_nodes = bulk.num_nodes(elements[i]);
std::vector<double> coordinates(3*num_nodes,0);
const stk::mesh::Entity* nodes = bulk.begin_nodes(elements[i]);
for(unsigned j=0;j<num_nodes;++j)
{
double* data = static_cast<double*>(stk::mesh::field_data(*coords, nodes[j]));
coordinates[3*j] = data[0];
coordinates[3*j+1] = data[1];
coordinates[3*j+2] = data[2];
}
findBoundingBoxCoordinates(coordinates, boxCoordinates);
unsigned id = bulk.identifier(elements[i]);
Ident domainBoxId(id, bulk.parallel_rank());
domainBoxes[i] = std::make_pair(FloatBox(boxCoordinates[0], boxCoordinates[1], boxCoordinates[2],
boxCoordinates[3], boxCoordinates[4], boxCoordinates[5]),
domainBoxId);
}
}
void setup_node_coords(const stk::mesh::BulkData& bulk, const CoordFieldType& coords, const stk::mesh::Selector& selector)
{
const stk::mesh::BucketVector& nodeBuckets = bulk.buckets(stk::topology::NODE_RANK);
bucketCapacity = nodeBuckets[0]->capacity();
unsigned nodeAllocSize = nodeBuckets.size()*bucketCapacity;
nodeCoords = DeviceViewMatrixType("DNodeCoords", nodeAllocSize, bulk.mesh_meta_data().spatial_dimension());
constNodeCoords = nodeCoords;
hostNodeCoords = Kokkos::create_mirror_view(nodeCoords);
for(unsigned bktIndex = 0; bktIndex < nodeBuckets.size(); ++bktIndex)
{
const stk::mesh::Bucket& bucket = *nodeBuckets[bktIndex];
for(unsigned nodeIndex = 0; nodeIndex < bucket.size(); ++nodeIndex)
{
stk::mesh::Entity node = bucket[nodeIndex];
double *node_coords = stk::mesh::field_data(coords, node);
unsigned nodeCoordIndex = host_get_index(node);
for(unsigned k=0; k<bulk.mesh_meta_data().spatial_dimension(); ++k)
{
hostNodeCoords(nodeCoordIndex, k) = node_coords[k];
}
}
}
Kokkos::deep_copy(nodeCoords, hostNodeCoords);
}
void copy_mesh_subset(stk::mesh::BulkData &inputBulk, stk::mesh::Selector selectedElems, stk::mesh::BulkData &outputBulk)
{
ThrowRequireMsg(&inputBulk != &outputBulk, "Can't copy to same mesh.");
stk::mesh::MetaData &inputMeta = inputBulk.mesh_meta_data();
stk::mesh::MetaData &outputMeta = outputBulk.mesh_meta_data();
outputMeta.initialize(inputMeta.spatial_dimension(), inputMeta.entity_rank_names());
copy_parts(inputMeta, outputMeta);
copy_fields(inputMeta, outputMeta);
}
bool bucket_part_memberships_match(const stk::mesh::BulkData& bulk, stk::EnvData& env_data)
{
int numGlobalDiffs = bucket_counts_match(bulk, env_data);
if(numGlobalDiffs > 0)
{
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::CommSparse comm(env_data.m_worldComm);
stk::CommBuffer &buff = stk::diff::get_comm_buffer_for_destination_proc(comm);
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
stk::mesh::EntityVector entities;
stk::mesh::get_entities(bulk, rank, entities);
for(int iphase = 0; iphase < 2; ++iphase)
{
for(size_t i=0;i<entities.size();++i)
{
const stk::mesh::PartVector& parts = bulk.bucket(entities[i]).supersets();
std::string part_names_for_entity = create_string_from_parts(parts);
std::string string_to_send;
if(irank != 1 && irank != 2)
{
string_to_send = std::to_string(bulk.identifier(entities[i])) + " " + part_names_for_entity;
}
else
{
string_to_send = part_names_for_entity;
}
stk::diff::pack_string(buff, string_to_send);
}
stk::diff::allocate_or_communicate(iphase, comm);
}
}
}
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::CommSparse comm(env_data.m_worldComm);
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
const stk::mesh::BucketVector& buckets = bulk.buckets(rank);
for(int iphase = 0; iphase < 2; ++iphase)
{
pack_buckets_parts(buckets, stk::diff::get_comm_buffer_for_destination_proc(comm));
stk::diff::allocate_or_communicate(iphase, comm);
}
}
numGlobalDiffs += get_global_bucket_part_membership_differences(env_data.m_worldComm, 0);
return numGlobalDiffs == 0;
}
std::map<stk::mesh::EntityId, std::pair<stk::mesh::EntityId, int> > get_split_coincident_elements(stk::mesh::BulkData& bulkData)
{
stk::mesh::Selector sel = bulkData.mesh_meta_data().locally_owned_part();
ElemGraphForDiagnostics graph(bulkData, sel);
const stk::mesh::impl::SparseGraph& coingraph = graph.get_coincident_graph();
std::map<stk::mesh::EntityId, std::pair<stk::mesh::EntityId, int> > badElements;
for(const stk::mesh::impl::SparseGraph::value_type& extractedEdgesForElem : coingraph)
{
const std::vector<stk::mesh::GraphEdge>& coincidentEdgesForElem = extractedEdgesForElem.second;
for(const stk::mesh::GraphEdge& edge : coincidentEdgesForElem)
{
if(edge.elem2 < 0)
{
stk::mesh::Entity entity = graph.get_entity(edge.elem1);
stk::mesh::EntityId id = bulkData.identifier(entity);
stk::mesh::impl::ParallelGraphInfo& par_info = graph.get_parallel_info();
stk::mesh::impl::ParallelGraphInfo::iterator iter = par_info.find(edge);
ThrowRequireMsg(iter!=par_info.end(), "Program error. Contact [email protected] for support.");
badElements[id] = std::make_pair(-edge.elem2, iter->second.m_other_proc);
}
}
}
return badElements;
}
inline
void setupKeyholeMesh2D_case2(stk::mesh::BulkData& bulk)
{
//
// proc 0 proc 1
// |
// | block_2 block_3
// |
// block_1 | 12---11
// | | 4 |
// 4----3 | 3----6 6----10
// | 1 | | | 2 |
// 1----2 | 2----5 5----9
// | | 3 |
// | 7----8
// |
//
//nodes 5 and 6 are ghosts (aura) on proc 0,
//and should be members of block_2 and block_3 on proc 0
//if edges are added, the edge between nodes 5 and 6 should
//be a member of block_2 not block_3.
//
stk::mesh::MetaData& meta = bulk.mesh_meta_data();
stk::mesh::Part& block_1 = meta.declare_part_with_topology("block_1", stk::topology::QUAD_4_2D);
stk::mesh::Part& block_2 = meta.declare_part_with_topology("block_2", stk::topology::QUAD_4_2D);
stk::mesh::Part& block_3 = meta.declare_part_with_topology("block_3", stk::topology::QUAD_4_2D);
meta.commit();
bulk.modification_begin();
stk::mesh::EntityIdVector elem1_nodes {1, 2, 3, 4};
stk::mesh::EntityIdVector elem2_nodes {2, 5, 6, 3};
stk::mesh::EntityIdVector elem3_nodes {7, 8, 9, 5};
stk::mesh::EntityIdVector elem4_nodes {6, 10, 11, 12};
stk::mesh::EntityId elemId = 1;
if (bulk.parallel_rank() == 0) {
stk::mesh::declare_element(bulk, block_1, elemId, elem1_nodes);
stk::mesh::Entity node2 = bulk.get_entity(stk::topology::NODE_RANK, 2);
stk::mesh::Entity node3 = bulk.get_entity(stk::topology::NODE_RANK, 3);
bulk.add_node_sharing(node2, 1);
bulk.add_node_sharing(node3, 1);
}
else if (bulk.parallel_rank() == 1) {
elemId = 2;
stk::mesh::declare_element(bulk, block_2, elemId, elem2_nodes);
elemId = 3;
stk::mesh::declare_element(bulk, block_3, elemId, elem3_nodes);
elemId = 4;
stk::mesh::declare_element(bulk, block_3, elemId, elem4_nodes);
stk::mesh::Entity node2 = bulk.get_entity(stk::topology::NODE_RANK, 2);
stk::mesh::Entity node3 = bulk.get_entity(stk::topology::NODE_RANK, 3);
bulk.add_node_sharing(node2, 0);
bulk.add_node_sharing(node3, 0);
}
bulk.modification_end();
}
MomentumBuoyancyBoussinesqSrcElemKernel<AlgTraits>::MomentumBuoyancyBoussinesqSrcElemKernel(
const stk::mesh::BulkData& bulkData,
const SolutionOptions& solnOpts,
ElemDataRequests& dataPreReqs)
: Kernel(),
rhoRef_(solnOpts.referenceDensity_),
ipNodeMap_(sierra::nalu::MasterElementRepo::get_volume_master_element(AlgTraits::topo_)->ipNodeMap())
{
const stk::mesh::MetaData& metaData = bulkData.mesh_meta_data();
ScalarFieldType *temperature = metaData.get_field<ScalarFieldType>(stk::topology::NODE_RANK, "temperature");
temperatureNp1_ = &(temperature->field_of_state(stk::mesh::StateNP1));
coordinates_ = metaData.get_field<VectorFieldType>(stk::topology::NODE_RANK, solnOpts.get_coordinates_name());
const std::vector<double>& solnOptsGravity = solnOpts.get_gravity_vector(AlgTraits::nDim_);
for (int i = 0; i < AlgTraits::nDim_; i++)
gravity_(i) = solnOptsGravity[i];
tRef_ = solnOpts.referenceTemperature_;
rhoRef_ = solnOpts.referenceDensity_;
beta_ = solnOpts.thermalExpansionCoeff_;
MasterElement* meSCV = sierra::nalu::MasterElementRepo::get_volume_master_element(AlgTraits::topo_);
get_scv_shape_fn_data<AlgTraits>([&](double* ptr){meSCV->shape_fcn(ptr);}, v_shape_function_);
// add master elements
dataPreReqs.add_cvfem_volume_me(meSCV);
// fields and data
dataPreReqs.add_coordinates_field(*coordinates_, AlgTraits::nDim_, CURRENT_COORDINATES);
dataPreReqs.add_gathered_nodal_field(*temperatureNp1_, 1);
dataPreReqs.add_master_element_call(SCV_VOLUME, CURRENT_COORDINATES);
}
inline void createBoundingBoxesForSidesInSidesets(const stk::mesh::BulkData& bulk, std::vector<FloatBox>& domainBoxes)
{
stk::mesh::ExodusTranslator exoTranslator(bulk);
size_t numberBoundingBoxes = 0;
std::vector<int64_t> sidesetIds;
exoTranslator.fill_side_set_ids(sidesetIds);
for (size_t i=0;i<sidesetIds.size();i++)
{
numberBoundingBoxes += exoTranslator.get_local_num_entities_for_id(sidesetIds[i], bulk.mesh_meta_data().side_rank());
}
domainBoxes.resize(numberBoundingBoxes);
stk::mesh::FieldBase const * coords = bulk.mesh_meta_data().coordinate_field();
size_t boxCounter = 0;
std::vector<double> boxCoordinates(6);
for (size_t ssetCounter=0;ssetCounter<sidesetIds.size();ssetCounter++)
{
const stk::mesh::Part* sideset = exoTranslator.get_exodus_part_of_rank(sidesetIds[ssetCounter], bulk.mesh_meta_data().side_rank());
stk::mesh::EntityVector sides;
stk::mesh::Selector sel = bulk.mesh_meta_data().locally_owned_part() & *sideset;
stk::mesh::get_selected_entities(sel, bulk.buckets(bulk.mesh_meta_data().side_rank()), sides);
for(size_t j=0;j<sides.size();++j)
{
unsigned num_nodes_per_side = bulk.num_nodes(sides[j]);
const stk::mesh::Entity* nodes = bulk.begin_nodes(sides[j]);
std::vector<double> coordinates(3*num_nodes_per_side,0);
for(unsigned k=0;k<num_nodes_per_side;++k)
{
double *data = static_cast<double*>(stk::mesh::field_data(*coords, nodes[k]));
coordinates[3*k] = data[0];
coordinates[3*k+1] = data[1];
coordinates[3*k+2] = data[2];
}
findBoundingBoxCoordinates(coordinates, boxCoordinates);
domainBoxes[boxCounter].set_box(boxCoordinates[0], boxCoordinates[1], boxCoordinates[2],
boxCoordinates[3], boxCoordinates[4], boxCoordinates[5]);
boxCounter++;
}
}
ThrowRequireMsg(boxCounter == numberBoundingBoxes, "Program error. Please contact sierra-help for support");
}
MomentumNSOKeElemKernel<AlgTraits>::MomentumNSOKeElemKernel(
const stk::mesh::BulkData& bulkData,
const SolutionOptions& solnOpts,
VectorFieldType* ,
GenericFieldType* Gju,
const double fourthFac,
ElemDataRequests& dataPreReqs)
: Kernel(),
Gju_(Gju),
lrscv_(sierra::nalu::MasterElementRepo::get_surface_master_element(AlgTraits::topo_)->adjacentNodes()),
fourthFac_(fourthFac),
shiftedGradOp_(solnOpts.get_shifted_grad_op("velocity"))
{
const stk::mesh::MetaData& metaData = bulkData.mesh_meta_data();
velocityNp1_ = metaData.get_field<VectorFieldType>(
stk::topology::NODE_RANK, "velocity");
densityNp1_ = metaData.get_field<ScalarFieldType>(
stk::topology::NODE_RANK, "density");
pressure_ = metaData.get_field<ScalarFieldType>(
stk::topology::NODE_RANK, "pressure");
if (solnOpts.does_mesh_move())
velocityRTM_ = metaData.get_field<VectorFieldType>(
stk::topology::NODE_RANK, "velocity_rtm");
else
velocityRTM_ = metaData.get_field<VectorFieldType>(
stk::topology::NODE_RANK, "velocity");
pressure_ = metaData.get_field<ScalarFieldType>(
stk::topology::NODE_RANK, "pressure");
coordinates_ = metaData.get_field<VectorFieldType>(
stk::topology::NODE_RANK, solnOpts.get_coordinates_name());
Gjp_ = metaData.get_field<VectorFieldType>(stk::topology::NODE_RANK, "dpdx");
MasterElement *meSCS = sierra::nalu::MasterElementRepo::get_surface_master_element(AlgTraits::topo_);
get_scs_shape_fn_data<AlgTraits>([&](double* ptr){meSCS->shape_fcn(ptr);}, v_shape_function_);
// add master elements
dataPreReqs.add_cvfem_surface_me(meSCS);
// fields
dataPreReqs.add_gathered_nodal_field(*Gju_, AlgTraits::nDim_, AlgTraits::nDim_);
dataPreReqs.add_coordinates_field(*coordinates_, AlgTraits::nDim_, CURRENT_COORDINATES);
dataPreReqs.add_gathered_nodal_field(*velocityNp1_, AlgTraits::nDim_);
dataPreReqs.add_gathered_nodal_field(*velocityRTM_, AlgTraits::nDim_);
dataPreReqs.add_gathered_nodal_field(*Gjp_, AlgTraits::nDim_);
dataPreReqs.add_gathered_nodal_field(*densityNp1_,1);
dataPreReqs.add_gathered_nodal_field(*pressure_,1);
// master element data
dataPreReqs.add_master_element_call(SCS_AREAV, CURRENT_COORDINATES);
if ( shiftedGradOp_ )
dataPreReqs.add_master_element_call(SCS_SHIFTED_GRAD_OP, CURRENT_COORDINATES);
else
dataPreReqs.add_master_element_call(SCS_GRAD_OP, CURRENT_COORDINATES);
dataPreReqs.add_master_element_call(SCS_GIJ, CURRENT_COORDINATES);
}
void send_part_names_to_diffing_tool_except(const stk::mesh::BulkData& bulk, stk::ParallelMachine communicator, stk::mesh::Part* skipPart)
{
stk::CommSparse comm(communicator);
for(int iphase = 0; iphase < 2; ++iphase)
{
pack_part_names_except(get_comm_buffer_for_destination_proc(comm), bulk.mesh_meta_data().get_parts(), skipPart);
allocate_or_communicate(iphase, comm);
}
}
void Vertices::fillCoordinates(const stk::mesh::BulkData& bulkData,
const std::string& coords_field_name,
const stk::mesh::EntityVector &entities)
{
mVertexCoordinates.resize(entities.size()*mSpatialDim, 0);
const stk::mesh::FieldBase * coord = bulkData.mesh_meta_data().get_field(stk::topology::NODE_RANK, coords_field_name);
for(size_t i=0;i<entities.size();++i)
stk::balance::internal::fillEntityCentroid(bulkData, coord, entities[i], &mVertexCoordinates[mSpatialDim*i]);
}
void put_mesh_into_part(stk::mesh::BulkData& bulkData, stk::mesh::Part& part)
{
stk::mesh::EntityVector entitiesToMakeActive;
std::vector<stk::mesh::PartVector> add_parts;
std::vector<stk::mesh::PartVector> rm_parts;
for(stk::topology::rank_t rank=stk::topology::BEGIN_RANK; rank < bulkData.mesh_meta_data().entity_rank_count(); rank++)
{
const stk::mesh::BucketVector &buckets = bulkData.get_buckets(rank, bulkData.mesh_meta_data().locally_owned_part());
for(const stk::mesh::Bucket *bucket : buckets)
{
for(stk::mesh::Entity entity : *bucket)
{
entitiesToMakeActive.push_back(entity);
add_parts.push_back(stk::mesh::PartVector(1, &part));
rm_parts.push_back(stk::mesh::PartVector());
}
}
}
bulkData.batch_change_entity_parts(entitiesToMakeActive, add_parts, rm_parts);
}
stk::parallel::DistributedIndex::KeyTypeVector get_all_local_keys(const stk::mesh::BulkData & bulkData)
{
stk::parallel::DistributedIndex::KeyTypeVector localKeys;
for(stk::mesh::EntityRank rank = stk::topology::NODE_RANK;rank < bulkData.mesh_meta_data().entity_rank_count();++rank)
{
stk::mesh::EntityVector entities;
stk::mesh::get_selected_entities(get_owned_or_shared_selector(bulkData), bulkData.buckets(rank), entities);
for(stk::mesh::Entity entity: entities)
localKeys.push_back(bulkData.entity_key(entity));
}
return localKeys;
}
void setupKeyholeMesh2D_case1(stk::mesh::BulkData& bulk)
{
//
// proc 0 proc 1
// |
// | block_2 block_3
// |
// block_1 | 10---9 9----12
// | | 3 | | 4 |
// 4----3 | 3----8 8----11
// | 1 | |
// 1----2 | 2----7
// | | 2 |
// | 5----6
// |
//
//shared nodes 2 and 3 should be members of block_1 and block_2 on both procs
//nodes 8 and 9 are ghosts on proc 0, and should be members of block_2 and block_3
//
//if edges are added, the edge between nodes 2 and 3 should be a member of block_1 not block_2.
//
//also, the edge between nodes 8 and 9 should be a member of block_2 and block_3 on both procs.
stk::mesh::MetaData& meta = bulk.mesh_meta_data();
stk::mesh::Part& block_1 = meta.declare_part_with_topology("block_1", stk::topology::QUAD_4_2D);
stk::mesh::Part& block_2 = meta.declare_part_with_topology("block_2", stk::topology::QUAD_4_2D);
stk::mesh::Part& block_3 = meta.declare_part_with_topology("block_3", stk::topology::QUAD_4_2D);
meta.commit();
bulk.modification_begin();
const int nodesPerElem = 4;
stk::mesh::EntityId elem1_nodes[nodesPerElem] = {1, 2, 3, 4};
stk::mesh::EntityId elem2_nodes[nodesPerElem] = {5, 6, 7, 2};
stk::mesh::EntityId elem3_nodes[nodesPerElem] = {3, 8, 9, 10};
stk::mesh::EntityId elem4_nodes[nodesPerElem] = {8, 11, 12, 9};
stk::mesh::EntityId elemId = 1;
if (bulk.parallel_rank() == 0) {
stk::mesh::declare_element(bulk, block_1, elemId, elem1_nodes);
}
else if (bulk.parallel_rank() == 1) {
elemId = 2;
stk::mesh::declare_element(bulk, block_2, elemId, elem2_nodes);
elemId = 3;
stk::mesh::declare_element(bulk, block_2, elemId, elem3_nodes);
elemId = 4;
stk::mesh::declare_element(bulk, block_3, elemId, elem4_nodes);
}
bulk.modification_end();
}
inline size_t get_number_sides_in_sideset(const stk::mesh::BulkData& bulk,
int sideset_id,
stk::mesh::Selector selector,
stk::topology stk_element_topology,
const stk::mesh::BucketVector& buckets)
{
if (bulk.has_sideset_data())
{
selector &= ( bulk.mesh_meta_data().locally_owned_part() | bulk.mesh_meta_data().globally_shared_part());
size_t num_sides = 0;
const stk::mesh::SideSet& sset = bulk.get_sideset_data(sideset_id);
for(const stk::mesh::SideSetEntry& elem_and_side : sset)
{
stk::mesh::Entity element = elem_and_side.element;
stk::mesh::Entity side = stk::mesh::get_side_entity_for_elem_side_pair(bulk, element, elem_and_side.side);
if(bulk.is_valid(side))
{
if(selector(bulk.bucket(side)))
{
if(stk_element_topology == stk::topology::INVALID_TOPOLOGY ||
stk_element_topology == bulk.bucket(element).topology())
{
++num_sides;
}
}
}
}
return num_sides;
}
else
{
selector &= bulk.mesh_meta_data().locally_owned_part();
return count_selected_entities(selector, buckets);
}
}
GeometricVertices::GeometricVertices(const stk::balance::BalanceSettings& balanceSettings,
const stk::mesh::BulkData& bulkData,
const stk::mesh::EntityVector& entities_to_balance,
const std::vector<stk::mesh::Selector> criteria)
: mEntitiesToBalance(entities_to_balance)
, mSelectors(criteria)
{
set_num_field_criteria( balanceSettings.getNumCriteria() );
set_spatial_dim( bulkData.mesh_meta_data().spatial_dimension() );
fillVertexIds(bulkData, mEntitiesToBalance);
fillCoordinates(bulkData, balanceSettings.getCoordinateFieldName(), mEntitiesToBalance);
fillVertexWeights(bulkData, balanceSettings, mEntitiesToBalance, mSelectors);
}
HelperObjectsABLWallFrictionVelocity(stk::mesh::BulkData& bulk, stk::mesh::Part* part, const double &gravity, const double &z0, const double &Tref)
: yamlNode(unit_test_utils::get_default_inputs()),
realmDefaultNode(unit_test_utils::get_realm_default_node()),
naluObj(new unit_test_utils::NaluTest(yamlNode)),
realm(naluObj->create_realm(realmDefaultNode, "multi_physics")),
ABLWallFrictionAlgorithm(nullptr),
gravity_(gravity),
z0_(z0),
Tref_(Tref)
{
realm.metaData_ = &bulk.mesh_meta_data();
realm.bulkData_ = &bulk;
ABLWallFrictionAlgorithm = new ComputeABLWallFrictionVelocityAlgorithm(realm, part, false, gravity_, z0_, Tref_);
}
void add_shell_element_if_coincident(const stk::mesh::BulkData& mesh,
const unsigned sideOrdinal,
const stk::mesh::Entity localElement,
const SideData& connectedElem,
std::vector<SideData>& filteredConnectedElements)
{
const stk::mesh::EntityVector &sideNodesOfReceivedElement = connectedElem.get_side_nodes();
stk::mesh::OrdinalAndPermutation localElemOrdAndPerm =
stk::mesh::get_ordinal_and_permutation(mesh, localElement, mesh.mesh_meta_data().side_rank(), sideNodesOfReceivedElement);
// for shell element, want the nodes of the solid to be in opposite order. So getting non-matching side ordinals
// means the normals oppose
bool does_local_shell_side_normal_oppose_other_element_side_normal = (localElemOrdAndPerm.first == sideOrdinal);
if (does_local_shell_side_normal_oppose_other_element_side_normal)
filteredConnectedElements.push_back(connectedElem);
}
int bucket_counts_match(const stk::mesh::BulkData& bulk, stk::EnvData& env_data)
{
stk::CommSparse comm(env_data.m_worldComm);
int destinationProc = getDestinationProc(env_data.m_worldComm);
stk::CommBuffer& buf = comm.send_buffer(destinationProc);
for(int iphase = 0; iphase < 2; ++iphase)
{
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
const stk::mesh::BucketVector& buckets = bulk.buckets(rank);
buf.pack<size_t>(buckets.size());
}
allocate_or_communicate(iphase, comm);
}
int num_diffs = 0;
return get_global_bucket_count_differences(env_data.m_worldComm, num_diffs);
}
ScratchViews::ScratchViews(const TeamHandleType& team,
const stk::mesh::BulkData& bulkData,
stk::topology topo,
ElemDataRequests& dataNeeded)
: elemNodes(nullptr), scs_areav(), dndx(), dndx_shifted(), deriv(), det_j(), scv_volume(), gijUpper(), gijLower()
{
/* master elements are allowed to be null if they are not required */
MasterElement *meSCS = dataNeeded.get_cvfem_surface_me();
MasterElement *meSCV = dataNeeded.get_cvfem_volume_me();
int nDim = bulkData.mesh_meta_data().spatial_dimension();
int nodesPerElem = topo.num_nodes();
int numScsIp = meSCS != nullptr ? meSCS->numIntPoints_ : 0;
int numScvIp = meSCV != nullptr ? meSCV->numIntPoints_ : 0;
create_needed_field_views(team, dataNeeded, bulkData, nodesPerElem);
create_needed_master_element_views(team, dataNeeded, nDim, nodesPerElem, numScsIp, numScvIp);
}
void ScratchViews::create_needed_field_views(const TeamHandleType& team,
const ElemDataRequests& dataNeeded,
const stk::mesh::BulkData& bulkData,
int nodesPerElem)
{
const stk::mesh::MetaData& meta = bulkData.mesh_meta_data();
unsigned numFields = meta.get_fields().size();
fieldViews.resize(numFields, nullptr);
const FieldSet& neededFields = dataNeeded.get_fields();
for(const FieldInfo& fieldInfo : neededFields) {
stk::mesh::EntityRank fieldEntityRank = fieldInfo.field->entity_rank();
ThrowAssertMsg(fieldEntityRank == stk::topology::NODE_RANK || fieldEntityRank == stk::topology::ELEM_RANK, "Currently only node and element fields are supported.");
unsigned scalarsDim1 = fieldInfo.scalarsDim1;
unsigned scalarsDim2 = fieldInfo.scalarsDim2;
if (fieldEntityRank==stk::topology::ELEM_RANK) {
if (scalarsDim2 == 0) {
fieldViews[fieldInfo.field->mesh_meta_data_ordinal()] = new ViewT<SharedMemView<double*>>(get_shmem_view_1D(team, scalarsDim1));
}
else {
fieldViews[fieldInfo.field->mesh_meta_data_ordinal()] = new ViewT<SharedMemView<double**>>(get_shmem_view_2D(team, scalarsDim1, scalarsDim2));
}
}
else if (fieldEntityRank==stk::topology::NODE_RANK) {
if (scalarsDim2 == 0) {
if (scalarsDim1 == 1) {
fieldViews[fieldInfo.field->mesh_meta_data_ordinal()] = new ViewT<SharedMemView<double*>>(get_shmem_view_1D(team, nodesPerElem));
}
else {
fieldViews[fieldInfo.field->mesh_meta_data_ordinal()] = new ViewT<SharedMemView<double**>>(get_shmem_view_2D(team, nodesPerElem, scalarsDim1));
}
}
else {
fieldViews[fieldInfo.field->mesh_meta_data_ordinal()] = new ViewT<SharedMemView<double***>>(get_shmem_view_3D(team, nodesPerElem, scalarsDim1, scalarsDim2));
}
}
else {
ThrowRequireMsg(false,"Only elem-rank and node-rank fields supported for scratch-views currently.");
}
}
}
void print_entity_proc_map( stk::diag::Writer & writer ,
const stk::mesh::BulkData & mesh )
{
const stk::mesh::MetaData & meta = mesh.mesh_meta_data();
const std::vector<stk::mesh::Entity*> & comm = mesh.entity_comm();
const std::vector<stk::mesh::Ghosting*> & ghost = mesh.ghostings();
size_t counter = 0 ;
for ( size_t ig = 0 ; ig < ghost.size() ; ++ig ) {
const stk::mesh::Ghosting & g = * ghost[ig] ;
writer << "P" << mesh.parallel_rank()
<< " " << g.name() << " Communication:" << std::endl ;
for ( std::vector<stk::mesh::Entity*>::const_iterator
i = comm.begin() ; i != comm.end() ; ++i ) {
const stk::mesh::Entity & entity = **i ;
std::vector<unsigned> procs ;
stk::mesh::comm_procs( g , entity , procs );
if ( ! procs.empty() ) {
writer << "[" << counter << "] "
<< meta.entity_rank_name( entity.entity_rank() )
<< "[" << entity.identifier() << " " ;
if ( entity.owner_rank() != mesh.parallel_rank() ) {
writer << "not_" ;
}
writer << "owned ] {" ;
for ( size_t j = 0 ; j < procs.size() ; ++j ) {
writer << " " << procs[j] ;
}
writer << " }" << std::endl ;
}
}
}
}
MomentumWallFunctionElemKernel<BcAlgTraits>::MomentumWallFunctionElemKernel(
const stk::mesh::BulkData& bulkData,
const SolutionOptions& solnOpts,
ElemDataRequests& dataPreReqs)
: Kernel(),
elog_(solnOpts.get_turb_model_constant(TM_elog)),
kappa_(solnOpts.get_turb_model_constant(TM_kappa)),
yplusCrit_(solnOpts.get_turb_model_constant(TM_yplus_crit)),
ipNodeMap_(sierra::nalu::MasterElementRepo::get_surface_master_element(BcAlgTraits::topo_)->ipNodeMap())
{
const stk::mesh::MetaData& metaData = bulkData.mesh_meta_data();
VectorFieldType *velocity = metaData.get_field<VectorFieldType>(stk::topology::NODE_RANK, "velocity");
velocityNp1_ = &(velocity->field_of_state(stk::mesh::StateNP1));
bcVelocity_ = metaData.get_field<VectorFieldType>(
stk::topology::NODE_RANK, "wall_velocity_bc");
density_ = metaData.get_field<ScalarFieldType>(stk::topology::NODE_RANK, "density");
viscosity_ = metaData.get_field<ScalarFieldType>(stk::topology::NODE_RANK, "viscosity");
exposedAreaVec_ = metaData.get_field<GenericFieldType>(metaData.side_rank(), "exposed_area_vector");
wallFrictionVelocityBip_ = metaData.get_field<GenericFieldType>(metaData.side_rank(), "wall_friction_velocity_bip");
wallNormalDistanceBip_ = metaData.get_field<GenericFieldType>(metaData.side_rank(), "wall_normal_distance_bip");
VectorFieldType *coordinates = metaData.get_field<VectorFieldType>(
stk::topology::NODE_RANK, solnOpts.get_coordinates_name());
MasterElement *meFC = sierra::nalu::MasterElementRepo::get_surface_master_element(BcAlgTraits::topo_);
// compute and save shape function
get_face_shape_fn_data<BcAlgTraits>([&](double* ptr){meFC->shape_fcn(ptr);}, vf_shape_function_);
// add master elements
dataPreReqs.add_cvfem_face_me(meFC);
// fields and data; mdot not gathered as element data
dataPreReqs.add_coordinates_field(*coordinates, BcAlgTraits::nDim_, CURRENT_COORDINATES);
dataPreReqs.add_gathered_nodal_field(*velocityNp1_, BcAlgTraits::nDim_);
dataPreReqs.add_gathered_nodal_field(*bcVelocity_, BcAlgTraits::nDim_);
dataPreReqs.add_gathered_nodal_field(*density_, 1);
dataPreReqs.add_gathered_nodal_field(*viscosity_, 1);
dataPreReqs.add_face_field(*exposedAreaVec_, BcAlgTraits::numFaceIp_, BcAlgTraits::nDim_);
dataPreReqs.add_face_field(*wallFrictionVelocityBip_, BcAlgTraits::numFaceIp_);
dataPreReqs.add_face_field(*wallNormalDistanceBip_, BcAlgTraits::numFaceIp_);
}
void fill_sharing_data(stk::mesh::BulkData& bulkData, stk::mesh::ElemElemGraph &graph, const stk::mesh::EntityVector& sidesThatNeedFixing, std::vector<SideSharingData>& sideSharingDataThisProc, std::vector<stk::mesh::impl::IdViaSidePair>& idAndSides)
{
// Element 1, side 5: face 15
// Element 2, side 3: face 23
// Are these faces the same? Yes: delete face 23, then connect face 15 to element 2 with negative permutation
const stk::mesh::PartOrdinal sharedOrd = bulkData.mesh_meta_data().globally_shared_part().mesh_meta_data_ordinal();
for(size_t i=0;i<sidesThatNeedFixing.size();++i)
{
stk::mesh::impl::ElementViaSidePair elementAndSide = get_element_and_side_ordinal(bulkData, sidesThatNeedFixing[i]);
stk::mesh::impl::LocalId localElemId = graph.get_local_element_id(elementAndSide.element);
for(const stk::mesh::GraphEdge& edge : graph.get_edges_for_element(localElemId))
{
if(edge.side1() == elementAndSide.side && edge.elem2() < 0)
{
const stk::mesh::impl::ParallelInfo &pInfo = graph.get_parallel_info_for_graph_edge(edge);
const stk::mesh::Entity* nodes = bulkData.begin_nodes(sidesThatNeedFixing[i]);
unsigned numNodes = bulkData.num_nodes(sidesThatNeedFixing[i]);
SideSharingData localTemp({bulkData.identifier(elementAndSide.element), elementAndSide.side},
sidesThatNeedFixing[i],
pInfo.get_proc_rank_of_neighbor(),
std::min(bulkData.parallel_rank(),pInfo.get_proc_rank_of_neighbor()),
bulkData.identifier(sidesThatNeedFixing[i]));
localTemp.sideNodes.resize(numNodes);
for(unsigned j=0; j<numNodes; ++j) {
localTemp.sideNodes[j] = bulkData.identifier(nodes[j]);
}
fill_part_ordinals_besides_owned_and_shared(bulkData.bucket(sidesThatNeedFixing[i]), sharedOrd, localTemp.partOrdinals);
sideSharingDataThisProc.push_back(localTemp);
stk::mesh::EntityId localId = -edge.elem2();
idAndSides.push_back({localId, edge.side2()});
}
}
}
}
ScalarAdvDiffElemKernel<AlgTraits>::ScalarAdvDiffElemKernel(
const stk::mesh::BulkData& bulkData,
const SolutionOptions& solnOpts,
ScalarFieldType* scalarQ,
ScalarFieldType* diffFluxCoeff,
ElemDataRequests& dataPreReqs)
: Kernel(),
scalarQ_(scalarQ),
diffFluxCoeff_(diffFluxCoeff),
lrscv_(sierra::nalu::MasterElementRepo::get_surface_master_element(AlgTraits::topo_)->adjacentNodes()),
shiftedGradOp_(solnOpts.get_shifted_grad_op(scalarQ->name()))
{
// Save of required fields
const stk::mesh::MetaData& metaData = bulkData.mesh_meta_data();
coordinates_ = metaData.get_field<VectorFieldType>(
stk::topology::NODE_RANK, solnOpts.get_coordinates_name());
massFlowRate_ = metaData.get_field<GenericFieldType>(
stk::topology::ELEMENT_RANK, "mass_flow_rate_scs");
MasterElement *meSCS = sierra::nalu::MasterElementRepo::get_surface_master_element(AlgTraits::topo_);
get_scs_shape_fn_data<AlgTraits>([&](double* ptr){meSCS->shape_fcn(ptr);}, v_shape_function_);
const bool skewSymmetric = solnOpts.get_skew_symmetric(scalarQ->name());
get_scs_shape_fn_data<AlgTraits>([&](double* ptr){skewSymmetric ? meSCS->shifted_shape_fcn(ptr) : meSCS->shape_fcn(ptr);},
v_adv_shape_function_);
dataPreReqs.add_cvfem_surface_me(meSCS);
// fields and data
dataPreReqs.add_coordinates_field(*coordinates_, AlgTraits::nDim_, CURRENT_COORDINATES);
dataPreReqs.add_gathered_nodal_field(*scalarQ_, 1);
dataPreReqs.add_gathered_nodal_field(*diffFluxCoeff_, 1);
dataPreReqs.add_element_field(*massFlowRate_, AlgTraits::numScsIp_);
dataPreReqs.add_master_element_call(SCS_AREAV, CURRENT_COORDINATES);
if ( shiftedGradOp_ )
dataPreReqs.add_master_element_call(SCS_SHIFTED_GRAD_OP, CURRENT_COORDINATES);
else
dataPreReqs.add_master_element_call(SCS_GRAD_OP, CURRENT_COORDINATES);
}
stk::mesh::impl::ElementViaSidePair find_element_side_ord_for_side(const stk::mesh::BulkData& bulkData, unsigned num_elements, const stk::mesh::Entity* elements, stk::mesh::Entity side)
{
const stk::mesh::MetaData& meta = bulkData.mesh_meta_data();
for(unsigned i=0;i<num_elements;++i)
{
if(bulkData.bucket(elements[i]).owned())
{
const stk::mesh::Entity* sides = bulkData.begin(elements[i], meta.side_rank());
unsigned num_sides = bulkData.num_connectivity(elements[i], meta.side_rank());
for(unsigned j=0;j<num_sides;++j)
{
if(sides[j]==side)
{
const stk::mesh::ConnectivityOrdinal* ordinals = bulkData.begin_ordinals(elements[i], meta.side_rank());
return stk::mesh::impl::ElementViaSidePair{elements[i], static_cast<int>(ordinals[j])};
}
}
}
}
return stk::mesh::impl::ElementViaSidePair{stk::mesh::Entity(),0};
}
HelperObjectsABLWallFunction(stk::mesh::BulkData& bulk, int numDof, stk::mesh::Part* part, const double &z0, const double &Tref, const double &gravity)
: yamlNode(unit_test_utils::get_default_inputs()),
realmDefaultNode(unit_test_utils::get_realm_default_node()),
naluObj(new unit_test_utils::NaluTest(yamlNode)),
realm(naluObj->create_realm(realmDefaultNode, "multi_physics")),
eqSystems(realm),
eqSystem(eqSystems),
linsys(new unit_test_utils::TestLinearSystem(realm, numDof, &eqSystem)),
elemABLWallFunctionSolverAlg(nullptr),
edgeABLWallFunctionSolverAlg(nullptr),
computeGeomBoundAlg(nullptr),
z0_(z0),
Tref_(Tref),
gravity_(gravity)
{
realm.metaData_ = &bulk.mesh_meta_data();
realm.bulkData_ = &bulk;
eqSystem.linsys_ = linsys;
elemABLWallFunctionSolverAlg = new AssembleMomentumElemABLWallFunctionSolverAlgorithm(realm, part, &eqSystem, false, gravity_, z0_, Tref_);
edgeABLWallFunctionSolverAlg = new AssembleMomentumEdgeABLWallFunctionSolverAlgorithm(realm, part, &eqSystem, gravity_, z0_, Tref_);
computeGeomBoundAlg = new ComputeGeometryBoundaryAlgorithm(realm, part);
}
stk::mesh::EntityVector fill_shared_entities_that_need_fixing(const stk::mesh::BulkData& bulkData)
{
stk::mesh::EntityVector sides;
stk::mesh::get_selected_entities(bulkData.mesh_meta_data().locally_owned_part(), bulkData.buckets(bulkData.mesh_meta_data().side_rank()), sides);
stk::mesh::EntityVector sidesThatNeedFixing;
for(stk::mesh::Entity side : sides)
if(bulkData.state(side) == stk::mesh::Created)
{
unsigned num_nodes = bulkData.num_nodes(side);
const stk::mesh::Entity* nodes = bulkData.begin_nodes(side);
std::vector<stk::mesh::EntityKey> nodeKeys(num_nodes);
for(unsigned int i=0;i<num_nodes;++i)
nodeKeys[i] = bulkData.entity_key(nodes[i]);
std::vector<int> shared_procs;
bulkData.shared_procs_intersection(nodeKeys, shared_procs);
if(!shared_procs.empty())
sidesThatNeedFixing.push_back(side);
}
return sidesThatNeedFixing;
}
void find_ghosted_nodes_that_need_to_be_shared(const stk::mesh::BulkData & bulk, stk::mesh::EntityVector& ghosted_nodes_that_are_now_shared)
{
stk::mesh::EntityRank endRank = static_cast<stk::mesh::EntityRank>(bulk.mesh_meta_data().entity_rank_count());
if (endRank >= stk::topology::END_RANK)
{
endRank = stk::topology::END_RANK;
}
for (stk::mesh::EntityRank rank=stk::topology::EDGE_RANK; rank<endRank; ++rank)
{
const stk::mesh::BucketVector& entity_buckets = bulk.buckets(rank);
for(size_t i=0; i<entity_buckets.size(); ++i)
{
const stk::mesh::Bucket& bucket = *entity_buckets[i];
if ( bucket.owned() )
{
for(size_t n=0; n<bucket.size(); ++n)
{
const stk::mesh::Entity * nodes = bulk.begin_nodes(bucket[n]);
unsigned num_nodes = bulk.num_nodes(bucket[n]);
for (unsigned j=0;j<num_nodes;++j)
{
if (bulk.in_receive_ghost(bulk.entity_key(nodes[j])))
{
ghosted_nodes_that_are_now_shared.push_back(nodes[j]);
}
}
}
}
}
}
std::sort(ghosted_nodes_that_are_now_shared.begin(), ghosted_nodes_that_are_now_shared.end());
stk::mesh::EntityVector::iterator iter = std::unique(ghosted_nodes_that_are_now_shared.begin(), ghosted_nodes_that_are_now_shared.end());
ghosted_nodes_that_are_now_shared.erase(iter, ghosted_nodes_that_are_now_shared.end());
}
void fill_element_and_side_ids(Ioss::GroupingEntity & io,
stk::mesh::Part * const part,
const stk::mesh::BulkData & bulk_data,
stk::topology stk_element_topology,
const stk::mesh::Selector *subset_selector,
stk::mesh::EntityVector &sides,
std::vector<INT>& elem_side_ids)
{
if (bulk_data.has_sideset_data())
{
const stk::mesh::SideSet& sset = bulk_data.get_sideset_data(part->id());
size_t num_sides = sset.size();
elem_side_ids.reserve(num_sides*2);
stk::mesh::Selector selector = *part & ( bulk_data.mesh_meta_data().locally_owned_part() | bulk_data.mesh_meta_data().globally_shared_part() );
if(subset_selector)
selector &= *subset_selector;
for(size_t i=0;i<sset.size();++i)
{
stk::mesh::Entity element = sset[i].element;
stk::mesh::EntityId elemId = bulk_data.identifier(element);
int zero_based_side_ord = sset[i].side;
stk::mesh::Entity side = stk::mesh::get_side_entity_for_elem_id_side_pair_of_rank(bulk_data, elemId, zero_based_side_ord, bulk_data.mesh_meta_data().side_rank());
if(bulk_data.is_valid(side))
{
if(selector(bulk_data.bucket(side)))
{
if(bulk_data.bucket(element).topology() == stk_element_topology)
{
elem_side_ids.push_back(elemId);
elem_side_ids.push_back(zero_based_side_ord+1);
sides.push_back(side);
}
}
}
}
}
else
{
const stk::mesh::MetaData & meta_data = stk::mesh::MetaData::get(*part);
stk::mesh::EntityRank type = part_primary_entity_rank(*part);
size_t num_sides = get_entities(*part, type, bulk_data, sides, false, subset_selector);
elem_side_ids.reserve(num_sides * 2);
stk::mesh::EntityRank elem_rank = stk::topology::ELEMENT_RANK;
for(size_t i = 0; i < num_sides; ++i)
{
std::vector<stk::mesh::Entity> side;
side.push_back(sides[i]);
std::vector<stk::mesh::Entity> side_elements;
std::vector<stk::mesh::Entity> side_nodes(bulk_data.begin_nodes(sides[i]), bulk_data.end_nodes(sides[i]));
get_entities_through_relations(bulk_data, side_nodes, elem_rank, side_elements);
const size_t num_side_elem = side_elements.size();
std::sort(side_elements.begin(), side_elements.end(), stk::mesh::EntityLess(bulk_data));
stk::mesh::Entity suitable_elem = stk::mesh::Entity();
stk::mesh::ConnectivityOrdinal suitable_ordinal = stk::mesh::INVALID_CONNECTIVITY_ORDINAL;
for(size_t j = 0; j < num_side_elem; ++j)
{
const stk::mesh::Entity elem = side_elements[j];
const stk::mesh::Bucket &elemBucket = bulk_data.bucket(elem);
const bool isSelectingEverything = subset_selector == NULL;
const bool isElementBeingOutput = (isSelectingEverything || (*subset_selector)(elemBucket))
&& elemBucket.member(meta_data.locally_owned_part());
if(isElementBeingOutput)
{
const stk::mesh::Entity * elem_sides = bulk_data.begin(elem, type);
stk::mesh::ConnectivityOrdinal const * side_ordinal = bulk_data.begin_ordinals(elem, type);
const size_t num_elem_sides = bulk_data.num_connectivity(elem, type);
for(size_t k = 0; k < num_elem_sides; ++k)
{
if(elem_sides[k] == side[0])
{
suitable_elem = elem;
suitable_ordinal = side_ordinal[k];
break;
}
}
}
}
if(!bulk_data.is_valid(suitable_elem))
{
std::ostringstream oss;
oss << "ERROR, no suitable element found";
throw std::runtime_error(oss.str());
}
elem_side_ids.push_back(bulk_data.identifier(suitable_elem));
elem_side_ids.push_back(suitable_ordinal + 1); // Ioss is 1-based, mesh is 0-based.
}
}
}