void performNodalMeshReduction(
stk::mesh::Part &samplePart,
stk::mesh::BulkData& bulkData)
{
const stk::mesh::EntityRank nodeEntityRank(0);
const stk::mesh::MetaData &metaData = stk::mesh::MetaData::get(bulkData);
std::vector<stk::mesh::Entity *> sampleNodes;
stk::mesh::get_selected_entities(samplePart, bulkData.buckets(nodeEntityRank), sampleNodes);
const stk::mesh::Selector locallyOwned = stk::mesh::MetaData::get(bulkData).locally_owned_part();
std::vector<stk::mesh::Entity *> relatedEntities;
typedef boost::indirect_iterator<std::vector<stk::mesh::Entity *>::const_iterator> EntityIterator;
for (EntityIterator it(sampleNodes.begin()), it_end(sampleNodes.end()); it != it_end; ++it) {
const stk::mesh::PairIterRelation relations = it->relations();
typedef stk::mesh::PairIterRelation::first_type RelationIterator;
for (RelationIterator rel_it = relations.first, rel_it_end = relations.second; rel_it != rel_it_end; ++rel_it) {
const Teuchos::Ptr<stk::mesh::Entity> relatedEntity(rel_it->entity());
if (Teuchos::nonnull(relatedEntity) && locallyOwned(*relatedEntity)) {
relatedEntities.push_back(relatedEntity.get());
}
}
}
std::sort(relatedEntities.begin(), relatedEntities.end(), stk::mesh::EntityLess());
relatedEntities.erase(
std::unique(relatedEntities.begin(), relatedEntities.end(), stk::mesh::EntityEqual()),
relatedEntities.end());
std::vector<stk::mesh::Entity *> sampleClosure;
stk::mesh::find_closure(bulkData, relatedEntities, sampleClosure);
// Keep only the closure, remove the rest, by decreasing entityRanks
{
const stk::mesh::Selector ownedOrShared = metaData.locally_owned_part() | metaData.globally_shared_part();
typedef boost::indirect_iterator<std::vector<stk::mesh::Entity *>::const_iterator> EntityIterator;
EntityIterator allKeepersEnd(sampleClosure.end());
const EntityIterator allKeepersBegin(sampleClosure.begin());
for (stk::mesh::EntityRank candidateRankCount = metaData.entity_rank_count(); candidateRankCount > 0; --candidateRankCount) {
const stk::mesh::EntityRank candidateRank = candidateRankCount - 1;
const EntityIterator keepersBegin = std::lower_bound(allKeepersBegin, allKeepersEnd,
stk::mesh::EntityKey(candidateRank, 0),
stk::mesh::EntityLess());
const EntityIterator keepersEnd = allKeepersEnd;
std::vector<stk::mesh::Entity *> candidates;
stk::mesh::get_selected_entities(ownedOrShared, bulkData.buckets(candidateRank), candidates);
{
BulkModification modification(bulkData);
std::set_difference(candidates.begin(), candidates.end(),
keepersBegin.base(), keepersEnd.base(),
EntityDestructor(modification),
stk::mesh::EntityLess());
}
allKeepersEnd = keepersBegin;
}
}
}
void performNodalMeshReduction(
stk::mesh::Part &samplePart,
stk::mesh::BulkData& bulkData)
{
const stk::mesh::MetaData &metaData = stk::mesh::MetaData::get(bulkData);
std::vector<stk::mesh::Entity> sampleNodes;
stk::mesh::get_selected_entities(samplePart, bulkData.buckets(stk::topology::NODE_RANK), sampleNodes);
const stk::mesh::Selector locallyOwned = stk::mesh::MetaData::get(bulkData).locally_owned_part();
std::vector<stk::mesh::Entity> relatedEntities;
typedef std::vector<stk::mesh::Entity>::const_iterator EntityIterator;
for (EntityIterator it(sampleNodes.begin()), it_end(sampleNodes.end()); it != it_end; ++it) {
for (stk::mesh::EntityRank r = stk::topology::NODE_RANK; r < metaData.entity_rank_count(); ++r) {
stk::mesh::Entity const* relations = bulkData.begin(*it, r);
const int num_rels = bulkData.num_connectivity(*it, r);
for (int i = 0; i < num_rels; ++i) {
stk::mesh::Entity relatedEntity = relations[i];
if (bulkData.is_valid(relatedEntity) && locallyOwned(bulkData.bucket(relatedEntity))) {
relatedEntities.push_back(relatedEntity);
}
}
}
}
std::sort(relatedEntities.begin(), relatedEntities.end(), stk::mesh::EntityLess(bulkData));
relatedEntities.erase(
std::unique(relatedEntities.begin(), relatedEntities.end()),
relatedEntities.end());
std::vector<stk::mesh::Entity> sampleClosure;
stk::mesh::find_closure(bulkData, relatedEntities, sampleClosure);
// Keep only the closure, remove the rest, by decreasing entityRanks
{
const stk::mesh::Selector ownedOrShared = metaData.locally_owned_part() | metaData.globally_shared_part();
EntityIterator allKeepersEnd(sampleClosure.end());
const EntityIterator allKeepersBegin(sampleClosure.begin());
for (size_t candidateRankCount = metaData.entity_rank_count(); candidateRankCount > 0; --candidateRankCount) {
const stk::mesh::EntityRank candidateRank = static_cast<stk::mesh::EntityRank>(candidateRankCount - 1);
const EntityIterator keepersBegin = std::lower_bound(allKeepersBegin, allKeepersEnd,
stk::mesh::EntityKey(candidateRank, 0),
stk::mesh::EntityLess(bulkData));
const EntityIterator keepersEnd = allKeepersEnd;
std::vector<stk::mesh::Entity> candidates;
stk::mesh::get_selected_entities(ownedOrShared, bulkData.buckets(candidateRank), candidates);
{
BulkModification modification(bulkData);
std::set_difference(candidates.begin(), candidates.end(),
keepersBegin.base(), keepersEnd.base(),
EntityDestructor(modification),
stk::mesh::EntityLess(bulkData));
}
allKeepersEnd = keepersBegin;
}
}
}
void stk::app::use_case_14_initialize_nodal_data(stk::mesh::BulkData & mesh ,
const CartesianField & model_coordinates ,
const CartesianField & coordinates ,
const CartesianField & velocity,
double dt)
{
const std::vector<stk::mesh::Bucket*> & buckets = mesh.buckets( stk::mesh::fem::FEMMetaData::NODE_RANK);
for ( std::vector<stk::mesh::Bucket*>::const_iterator k = buckets.begin() ; k != buckets.end() ; ++k ) {
stk::mesh::Bucket & bucket = **k ;
const unsigned length = bucket.size();
double * X = stk::mesh::field_data( model_coordinates , bucket.begin() );
double * coord = stk::mesh::field_data( coordinates , bucket.begin() );
double * v = stk::mesh::field_data( velocity , bucket.begin() );
for ( unsigned i = 0 ; i < length ; ++i ) {
v[0] = X[0];
v[1] = X[1];
v[2] = X[2];
coord[0] = X[0] + dt * v[0];
coord[1] = X[1] + dt * v[1];
coord[2] = X[2] + dt * v[2];
X += 3;
v += 3;
coord += 3;
}
}
}
inline void createBoundingBoxesForElementsInElementBlocks(const stk::mesh::BulkData &bulk, FlaotBoxVector& domainBoxes)
{
stk::mesh::EntityVector elements;
stk::mesh::get_selected_entities(bulk.mesh_meta_data().locally_owned_part(), bulk.buckets(stk::topology::ELEM_RANK), elements);
size_t numberBoundingBoxes = elements.size();
domainBoxes.resize(numberBoundingBoxes);
stk::mesh::FieldBase const * coords = bulk.mesh_meta_data().coordinate_field();
std::vector<double> boxCoordinates(6);
for(size_t i=0;i<elements.size();++i)
{
unsigned num_nodes = bulk.num_nodes(elements[i]);
std::vector<double> coordinates(3*num_nodes,0);
const stk::mesh::Entity* nodes = bulk.begin_nodes(elements[i]);
for(unsigned j=0;j<num_nodes;++j)
{
double* data = static_cast<double*>(stk::mesh::field_data(*coords, nodes[j]));
coordinates[3*j] = data[0];
coordinates[3*j+1] = data[1];
coordinates[3*j+2] = data[2];
}
findBoundingBoxCoordinates(coordinates, boxCoordinates);
unsigned id = bulk.identifier(elements[i]);
Ident domainBoxId(id, bulk.parallel_rank());
domainBoxes[i] = std::make_pair(FloatBox(boxCoordinates[0], boxCoordinates[1], boxCoordinates[2],
boxCoordinates[3], boxCoordinates[4], boxCoordinates[5]),
domainBoxId);
}
}
void use_case_5_initialize_data(
stk::mesh::BulkData & mesh ,
const VectorFieldType & node_coord ,
const VectorFieldType & node_displ ,
const VectorFieldType & node_rotat )
{
const std::vector<stk::mesh::Bucket*> & buckets = mesh.buckets( stk::mesh::fem::FEMMetaData::NODE_RANK );
for ( std::vector<stk::mesh::Bucket*>::const_iterator
k = buckets.begin() ; k != buckets.end() ; ++k ) {
stk::mesh::Bucket & bucket = **k ;
const unsigned length = bucket.size();
const unsigned length_3 = length * 3 ;
double * const coord = stk::mesh::field_data( node_coord , bucket.begin() );
double * const displ = stk::mesh::field_data( node_displ , bucket.begin() );
double * const rotat = stk::mesh::field_data( node_rotat , bucket.begin() );
for ( unsigned i = 0 ; i < length_3 ; ++i ) {
displ[i] = 0.1 * coord[i] ;
}
if ( rotat ) {
for ( unsigned i = 0 ; i < length_3 ; ++i ) {
rotat[i] = 0.01 * coord[i] ;
}
}
}
}
void setup_node_coords(const stk::mesh::BulkData& bulk, const CoordFieldType& coords, const stk::mesh::Selector& selector)
{
const stk::mesh::BucketVector& nodeBuckets = bulk.buckets(stk::topology::NODE_RANK);
bucketCapacity = nodeBuckets[0]->capacity();
unsigned nodeAllocSize = nodeBuckets.size()*bucketCapacity;
nodeCoords = DeviceViewMatrixType("DNodeCoords", nodeAllocSize, bulk.mesh_meta_data().spatial_dimension());
constNodeCoords = nodeCoords;
hostNodeCoords = Kokkos::create_mirror_view(nodeCoords);
for(unsigned bktIndex = 0; bktIndex < nodeBuckets.size(); ++bktIndex)
{
const stk::mesh::Bucket& bucket = *nodeBuckets[bktIndex];
for(unsigned nodeIndex = 0; nodeIndex < bucket.size(); ++nodeIndex)
{
stk::mesh::Entity node = bucket[nodeIndex];
double *node_coords = stk::mesh::field_data(coords, node);
unsigned nodeCoordIndex = host_get_index(node);
for(unsigned k=0; k<bulk.mesh_meta_data().spatial_dimension(); ++k)
{
hostNodeCoords(nodeCoordIndex, k) = node_coords[k];
}
}
}
Kokkos::deep_copy(nodeCoords, hostNodeCoords);
}
void setup_mesh_indices(const stk::mesh::BulkData& bulk, const stk::mesh::Selector& selector)
{
const stk::mesh::BucketVector& nodeBuckets = bulk.buckets(stk::topology::NODE_RANK);
meshIndices = DeviceViewMeshIndicesType("DMeshIndices", bulk.get_size_of_entity_index_space());
constMeshIndices = meshIndices;
hostMeshIndices = Kokkos::create_mirror_view(meshIndices);
for(unsigned bktIndex = 0; bktIndex < nodeBuckets.size(); ++bktIndex)
{
const stk::mesh::Bucket& bucket = *nodeBuckets[bktIndex];
for(unsigned nodeIndex = 0; nodeIndex < bucket.size(); ++nodeIndex)
{
stk::mesh::Entity node = bucket[nodeIndex];
stk::mesh::FastMeshIndex meshIndex(bucket.bucket_id(), nodeIndex);
hostMeshIndices(node.local_offset()) = meshIndex;
}
}
const stk::mesh::BucketVector& elemBuckets = bulk.get_buckets(stk::topology::ELEM_RANK, selector);
for(unsigned bktIndex = 0; bktIndex < elemBuckets.size(); ++bktIndex)
{
const stk::mesh::Bucket& bucket = *elemBuckets[bktIndex];
for(unsigned elemIndex = 0; elemIndex < bucket.size(); ++elemIndex)
{
stk::mesh::Entity elem = bucket[elemIndex];
stk::mesh::FastMeshIndex meshIndex(bucket.bucket_id(), elemIndex);
hostMeshIndices(elem.local_offset()) = meshIndex;
}
}
Kokkos::deep_copy(meshIndices, hostMeshIndices);
}
stk::parallel::DistributedIndex::KeyTypeVector get_all_local_keys(const stk::mesh::BulkData & bulkData)
{
stk::parallel::DistributedIndex::KeyTypeVector localKeys;
for(stk::mesh::EntityRank rank = stk::topology::NODE_RANK;rank < bulkData.mesh_meta_data().entity_rank_count();++rank)
{
stk::mesh::EntityVector entities;
stk::mesh::get_selected_entities(get_owned_or_shared_selector(bulkData), bulkData.buckets(rank), entities);
for(stk::mesh::Entity entity: entities)
localKeys.push_back(bulkData.entity_key(entity));
}
return localKeys;
}
bool bucket_part_memberships_match(const stk::mesh::BulkData& bulk, stk::EnvData& env_data)
{
int numGlobalDiffs = bucket_counts_match(bulk, env_data);
if(numGlobalDiffs > 0)
{
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::CommSparse comm(env_data.m_worldComm);
stk::CommBuffer &buff = stk::diff::get_comm_buffer_for_destination_proc(comm);
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
stk::mesh::EntityVector entities;
stk::mesh::get_entities(bulk, rank, entities);
for(int iphase = 0; iphase < 2; ++iphase)
{
for(size_t i=0;i<entities.size();++i)
{
const stk::mesh::PartVector& parts = bulk.bucket(entities[i]).supersets();
std::string part_names_for_entity = create_string_from_parts(parts);
std::string string_to_send;
if(irank != 1 && irank != 2)
{
string_to_send = std::to_string(bulk.identifier(entities[i])) + " " + part_names_for_entity;
}
else
{
string_to_send = part_names_for_entity;
}
stk::diff::pack_string(buff, string_to_send);
}
stk::diff::allocate_or_communicate(iphase, comm);
}
}
}
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::CommSparse comm(env_data.m_worldComm);
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
const stk::mesh::BucketVector& buckets = bulk.buckets(rank);
for(int iphase = 0; iphase < 2; ++iphase)
{
pack_buckets_parts(buckets, stk::diff::get_comm_buffer_for_destination_proc(comm));
stk::diff::allocate_or_communicate(iphase, comm);
}
}
numGlobalDiffs += get_global_bucket_part_membership_differences(env_data.m_worldComm, 0);
return numGlobalDiffs == 0;
}
inline void createBoundingBoxesForSidesInSidesets(const stk::mesh::BulkData& bulk, std::vector<FloatBox>& domainBoxes)
{
stk::mesh::ExodusTranslator exoTranslator(bulk);
size_t numberBoundingBoxes = 0;
std::vector<int64_t> sidesetIds;
exoTranslator.fill_side_set_ids(sidesetIds);
for (size_t i=0;i<sidesetIds.size();i++)
{
numberBoundingBoxes += exoTranslator.get_local_num_entities_for_id(sidesetIds[i], bulk.mesh_meta_data().side_rank());
}
domainBoxes.resize(numberBoundingBoxes);
stk::mesh::FieldBase const * coords = bulk.mesh_meta_data().coordinate_field();
size_t boxCounter = 0;
std::vector<double> boxCoordinates(6);
for (size_t ssetCounter=0;ssetCounter<sidesetIds.size();ssetCounter++)
{
const stk::mesh::Part* sideset = exoTranslator.get_exodus_part_of_rank(sidesetIds[ssetCounter], bulk.mesh_meta_data().side_rank());
stk::mesh::EntityVector sides;
stk::mesh::Selector sel = bulk.mesh_meta_data().locally_owned_part() & *sideset;
stk::mesh::get_selected_entities(sel, bulk.buckets(bulk.mesh_meta_data().side_rank()), sides);
for(size_t j=0;j<sides.size();++j)
{
unsigned num_nodes_per_side = bulk.num_nodes(sides[j]);
const stk::mesh::Entity* nodes = bulk.begin_nodes(sides[j]);
std::vector<double> coordinates(3*num_nodes_per_side,0);
for(unsigned k=0;k<num_nodes_per_side;++k)
{
double *data = static_cast<double*>(stk::mesh::field_data(*coords, nodes[k]));
coordinates[3*k] = data[0];
coordinates[3*k+1] = data[1];
coordinates[3*k+2] = data[2];
}
findBoundingBoxCoordinates(coordinates, boxCoordinates);
domainBoxes[boxCounter].set_box(boxCoordinates[0], boxCoordinates[1], boxCoordinates[2],
boxCoordinates[3], boxCoordinates[4], boxCoordinates[5]);
boxCounter++;
}
}
ThrowRequireMsg(boxCounter == numberBoundingBoxes, "Program error. Please contact sierra-help for support");
}
int bucket_counts_match(const stk::mesh::BulkData& bulk, stk::EnvData& env_data)
{
stk::CommSparse comm(env_data.m_worldComm);
int destinationProc = getDestinationProc(env_data.m_worldComm);
stk::CommBuffer& buf = comm.send_buffer(destinationProc);
for(int iphase = 0; iphase < 2; ++iphase)
{
for(size_t irank = 0; irank < bulk.mesh_meta_data().entity_rank_count(); ++irank)
{
stk::mesh::EntityRank rank = static_cast<stk::mesh::EntityRank>(irank);
const stk::mesh::BucketVector& buckets = bulk.buckets(rank);
buf.pack<size_t>(buckets.size());
}
allocate_or_communicate(iphase, comm);
}
int num_diffs = 0;
return get_global_bucket_count_differences(env_data.m_worldComm, num_diffs);
}
void testGoldValues(stk::mesh::MetaData &stkMeshMetaData, stk::mesh::BulkData &stkMeshBulkData, double alpha, double beta, double gamma)
{
double goldX = initial_value1[0] * (alpha + beta + gamma);
double goldY = initial_value1[1] * (alpha + beta + gamma);
double goldZ = initial_value1[2] * (alpha + beta + gamma);
stk::mesh::Field<double, stk::mesh::Cartesian3d> &force_field = *stkMeshMetaData.get_field<stk::mesh::Field<double, stk::mesh::Cartesian3d> >(stk::topology::NODE_RANK, "force");
double tol = 1.0e-4;
const stk::mesh::BucketVector& buckets = stkMeshBulkData.buckets(stk::topology::NODE_RANK);
for(size_t bucketIndex = 0; bucketIndex < buckets.size(); ++bucketIndex)
{
const stk::mesh::Bucket &bucket = *buckets[bucketIndex];
double *force = stk::mesh::field_data(force_field, bucket);
for(size_t nodeIndex = 0; nodeIndex < 3 * bucket.size(); nodeIndex += 3)
{
EXPECT_NEAR(goldX, force[nodeIndex+0], tol);
EXPECT_NEAR(goldY, force[nodeIndex+1], tol);
EXPECT_NEAR(goldZ, force[nodeIndex+2], tol);
}
}
}
inline
size_t get_entities(stk::mesh::Part &part,
stk::mesh::EntityRank type,
const stk::mesh::BulkData &bulk,
stk::mesh::EntityVector &entities,
bool include_shared,
const stk::mesh::Selector *subset_selector)
{
stk::mesh::MetaData & meta = stk::mesh::MetaData::get(part);
stk::mesh::Selector own_share = meta.locally_owned_part();
if(include_shared)
own_share |= meta.globally_shared_part();
stk::mesh::Selector selector = part & own_share;
if(subset_selector)
selector &= *subset_selector;
get_selected_entities(selector, bulk.buckets(type), entities);
return entities.size();
}
stk::mesh::EntityVector fill_shared_entities_that_need_fixing(const stk::mesh::BulkData& bulkData)
{
stk::mesh::EntityVector sides;
stk::mesh::get_selected_entities(bulkData.mesh_meta_data().locally_owned_part(), bulkData.buckets(bulkData.mesh_meta_data().side_rank()), sides);
stk::mesh::EntityVector sidesThatNeedFixing;
for(stk::mesh::Entity side : sides)
if(bulkData.state(side) == stk::mesh::Created)
{
unsigned num_nodes = bulkData.num_nodes(side);
const stk::mesh::Entity* nodes = bulkData.begin_nodes(side);
std::vector<stk::mesh::EntityKey> nodeKeys(num_nodes);
for(unsigned int i=0;i<num_nodes;++i)
nodeKeys[i] = bulkData.entity_key(nodes[i]);
std::vector<int> shared_procs;
bulkData.shared_procs_intersection(nodeKeys, shared_procs);
if(!shared_procs.empty())
sidesThatNeedFixing.push_back(side);
}
return sidesThatNeedFixing;
}
void find_ghosted_nodes_that_need_to_be_shared(const stk::mesh::BulkData & bulk, stk::mesh::EntityVector& ghosted_nodes_that_are_now_shared)
{
stk::mesh::EntityRank endRank = static_cast<stk::mesh::EntityRank>(bulk.mesh_meta_data().entity_rank_count());
if (endRank >= stk::topology::END_RANK)
{
endRank = stk::topology::END_RANK;
}
for (stk::mesh::EntityRank rank=stk::topology::EDGE_RANK; rank<endRank; ++rank)
{
const stk::mesh::BucketVector& entity_buckets = bulk.buckets(rank);
for(size_t i=0; i<entity_buckets.size(); ++i)
{
const stk::mesh::Bucket& bucket = *entity_buckets[i];
if ( bucket.owned() )
{
for(size_t n=0; n<bucket.size(); ++n)
{
const stk::mesh::Entity * nodes = bulk.begin_nodes(bucket[n]);
unsigned num_nodes = bulk.num_nodes(bucket[n]);
for (unsigned j=0;j<num_nodes;++j)
{
if (bulk.in_receive_ghost(bulk.entity_key(nodes[j])))
{
ghosted_nodes_that_are_now_shared.push_back(nodes[j]);
}
}
}
}
}
}
std::sort(ghosted_nodes_that_are_now_shared.begin(), ghosted_nodes_that_are_now_shared.end());
stk::mesh::EntityVector::iterator iter = std::unique(ghosted_nodes_that_are_now_shared.begin(), ghosted_nodes_that_are_now_shared.end());
ghosted_nodes_that_are_now_shared.erase(iter, ghosted_nodes_that_are_now_shared.end());
}
void assemble_elem_matrices_and_vectors(stk::mesh::BulkData& mesh,
stk::mesh::FieldBase& field,
stk::linsys::DofMapper& dof_mapper,
fei::Matrix& matrix,
fei::Vector& rhs)
{
stk::mesh::fem::FEMMetaData &fem = stk::mesh::fem::FEMMetaData::get(mesh);
const stk::mesh::EntityRank element_rank = fem.element_rank();
const std::vector<stk::mesh::Bucket*>& mesh_buckets = mesh.buckets(element_rank);
std::vector<stk::mesh::Bucket*> part_buckets;
stk::mesh::Selector select_owned(stk::mesh::MetaData::get(mesh).locally_owned_part());
stk::mesh::get_buckets(select_owned, mesh_buckets, part_buckets);
int field_id = dof_mapper.get_field_id(field);
stk::mesh::Entity& first_entity = *(part_buckets[0]->begin());
stk::mesh::PairIterRelation rel = first_entity.relations(stk::mesh::fem::FEMMetaData::NODE_RANK);
int num_nodes_per_elem = rel.second - rel.first;
fei::SharedPtr<fei::MatrixGraph> matgraph = matrix.getMatrixGraph();
int pattern_id = matgraph->definePattern(num_nodes_per_elem, stk::mesh::fem::FEMMetaData::NODE_RANK, field_id);
std::vector<int> node_ids(num_nodes_per_elem);
const int field_size = dof_mapper.get_fei_VectorSpace()->getFieldSize(field_id);
const int matsize = num_nodes_per_elem*field_size*num_nodes_per_elem*field_size;
const int vecsize = num_nodes_per_elem*field_size;
std::vector<double> elem_matrix_1d(matsize, 0);
std::vector<double*> elem_matrix_2d(vecsize);
std::vector<double> elem_vector(vecsize, 0);
for(size_t i=0; i<elem_matrix_2d.size(); ++i) {
elem_matrix_2d[i] = &elem_matrix_1d[i*vecsize];
}
//fill our dummy elem-matrix:
//This dummy matrix will be the same for every element. A real application
//would form a different elem-matrix for each element.
for(size_t i=0; i<elem_matrix_2d.size(); ++i) {
double* row = elem_matrix_2d[i];
if (i>=1) row[i-1] = -1;
row[i] = 2;
if (i<elem_matrix_2d.size()-1) row[i+1] = -1;
elem_vector[i] = 1;
}
std::vector<int> eqn_indices(vecsize);
for(size_t i=0; i<part_buckets.size(); ++i) {
stk::mesh::Bucket::iterator
b_iter = part_buckets[i]->begin(),
b_end = part_buckets[i]->end();
for(; b_iter != b_end; ++b_iter) {
stk::mesh::Entity& elem = *b_iter;
rel = elem.relations(stk::mesh::fem::FEMMetaData::NODE_RANK);
for(int j=0; rel.first != rel.second; ++rel.first, ++j) {
node_ids[j] = rel.first->entity()->identifier();
}
matgraph->getPatternIndices(pattern_id, &node_ids[0], eqn_indices);
matrix.sumIn(vecsize, &eqn_indices[0], vecsize, &eqn_indices[0],
&elem_matrix_2d[0]);
rhs.sumIn(vecsize, &eqn_indices[0], &elem_vector[0]);
}
}
}
void get_owned_nodes( std::vector<stk::mesh::Entity*> & owned_nodes ) const
{
owned_nodes.clear();
const stk::mesh::Selector select_owned( fmeta.locally_owned_part() );
stk::mesh::get_selected_entities( select_owned, bulk.buckets(fmeta.node_rank()), owned_nodes);
}
/**
* Check for nonpositive Jacobian
*/
bool GeometryVerifier::isGeometryBad(stk::mesh::BulkData& bulk, bool printTable) //, stk::mesh::Part& mesh_part )
{
const stk::mesh::fem::FEMMetaData& meta = stk::mesh::fem::FEMMetaData::get(bulk);
const unsigned p_rank = bulk.parallel_rank();
unsigned foundBad=0;
jac_data_map jac_data;
stk::mesh::Field<double, stk::mesh::Cartesian> *coord_field =
meta.get_field<stk::mesh::Field<double, stk::mesh::Cartesian> >("coordinates");
mesh::Selector select_owned( meta.locally_owned_part() );
const std::vector<mesh::Bucket*> & buckets = bulk.buckets( meta.element_rank() );
//for ( std::vector<mesh::Bucket *>::const_iterator ik = buckets.begin() ; ik != buckets.end() ; ++ik )
const stk::mesh::PartVector & all_parts = meta.get_parts();
for ( stk::mesh::PartVector::const_iterator ip = all_parts.begin(); ip != all_parts.end(); ++ip )
{
stk::mesh::Part * part = *ip;
if ( stk::mesh::is_auto_declared_part(*part) )
continue;
const CellTopologyData * const part_cell_topo_data = stk::percept::PerceptMesh::get_cell_topology(*part);
//std::cout << "P[" << p_rank << "] part = " << part->name() << " part_cell_topo_data= " << part_cell_topo_data << " topo-name= "
// << (part_cell_topo_data ? part_cell_topo_data->name : "null") << std::endl;
if (part_cell_topo_data)
jac_data[part_cell_topo_data->name] = jacData();
}
for (unsigned ipass = 0; ipass < 1; ipass++)
{
for ( std::vector<mesh::Bucket *>::const_iterator ik = buckets.begin() ; ik != buckets.end() ; ++ik )
{
if ( select_owned( **ik ) ) {
const mesh::Bucket & bucket = **ik ;
// Number of elems in this bucket of elems and elem field data
const unsigned number_elems = bucket.size();
double * elem_node_data = field_data( *coord_field , bucket.begin() );
//double * elem_centroid_data = field_data( elem_centroid_field , bucket.begin() );
//double * const coord = field_data( m_coordinates_field , *node );
// FIXME
if (0) { elem_node_data[0]++;}
#if 1
const CellTopologyData * const bucket_cell_topo_data = stk::percept::PerceptMesh::get_cell_topology(bucket);
int bucket_shardsId = ShardsInterfaceTable::s_singleton.lookupShardsId(bucket_cell_topo_data->name);
#endif
//if (0) { std::cout << bucket_cell_topo_data->name; }
if (0) { std::cout << "bucket_shardsId= " << bucket_shardsId << " name= " << bucket_cell_topo_data->name << std::endl; }
if (0) { std::cout << "number_elems= " << number_elems << std::endl;}
CellTopology cell_topo(bucket_cell_topo_data);
double volEqui = getEquiVol(cell_topo);
unsigned numCells = number_elems;
unsigned numNodes = cell_topo.getNodeCount();
unsigned spaceDim = cell_topo.getDimension();
//unsigned spatialDimMeta = stk::mesh::fem::FEMMetaData::get(bulk).spatial_dimension();
// Rank-3 array with dimensions (C,N,D) for the node coordinates of 3 traingle cells
FieldContainer<double> cellNodes(numCells, numNodes, spaceDim);
PerceptMesh::fillCellNodes(bucket, coord_field, cellNodes, spaceDim);
FieldContainer<double> volume(numCells);
// get min/max edge length
FieldContainer<double> elem_min_edge_length(number_elems);
FieldContainer<double> elem_max_edge_length(number_elems);
PerceptMesh::findMinMaxEdgeLength(bucket, *coord_field, elem_min_edge_length, elem_max_edge_length);
/// note: we're using cubature here instead of explicitly specifying some reference points
/// the idea is that we'll get a good estimate of the Jacobian's sign by testing it at all the
/// cubature points
DefaultCubatureFactory<double> cubFactory; // create cubature factory
unsigned cubDegree = 2; // set cubature degree, e.g. 2
Teuchos::RCP<Cubature<double> > myCub;
bool hasGoodTopo = true;
try {
myCub = cubFactory.create(cell_topo, cubDegree); // create default cubature
}
catch(...)
{
if (!p_rank)
std::cout << "WARNING: mesh contains elements that Intrepid doesn't support for quadrature, cell_topo= " << cell_topo.getName() << std::endl;
//continue;
hasGoodTopo = false;
}
FieldContainer<double> jacobian_det(numCells, 1);
unsigned numCubPoints = 1;
FieldContainer<double> jacobian(numCells, numCubPoints, spaceDim, spaceDim);
if (hasGoodTopo)
{
numCubPoints = myCub->getNumPoints(); // retrieve number of cubature points
FieldContainer<double> cub_points(numCubPoints, spaceDim);
FieldContainer<double> cub_weights(numCubPoints);
// Rank-4 array (C,P,D,D) for the Jacobian and its inverse and Rank-2 array (C,P) for its determinant
//FieldContainer<double> jacobian(numCells, numCubPoints, spaceDim, spaceDim);
jacobian.resize(numCells, numCubPoints, spaceDim, spaceDim);
FieldContainer<double> jacobian_inv(numCells, numCubPoints, spaceDim, spaceDim);
//FieldContainer<double> jacobian_det(numCells, numCubPoints);
jacobian_det.resize(numCells, numCubPoints);
myCub->getCubature(cub_points, cub_weights); // retrieve cubature points and weights
// Methods to compute cell Jacobians, their inverses and their determinants
CellTools<double>::setJacobian(jacobian, cub_points, cellNodes, cell_topo); // compute cell Jacobians
CellTools<double>::setJacobianInv(jacobian_inv, jacobian); // compute inverses of cell Jacobians
CellTools<double>::setJacobianDet(jacobian_det, jacobian); // compute determinants of cell Jacobians
FieldContainer<double> weightedMeasure(numCells, numCubPoints);
FieldContainer<double> onesLeft(numCells, numCubPoints);
onesLeft.initialize(1.0);
// compute weighted measure
FunctionSpaceTools::computeCellMeasure<double>(weightedMeasure, jacobian_det, cub_weights);
// integrate to get volume
FunctionSpaceTools::integrate<double>(volume, onesLeft, weightedMeasure, COMP_BLAS);
}
jacData& jdata = jac_data[cell_topo.getName()];
jdata.numEle += numCells;
for (unsigned iCell = 0; iCell < numCells; iCell++)
{
mesh::Entity & elem = bucket[iCell];
double min_edge_length = elem_min_edge_length[iCell];
double max_edge_length = elem_max_edge_length[iCell];
double max_edge_lengthNotZero = (fabs(max_edge_length) < 1.e-20? 1.e-20 : max_edge_length);
double cellVolActual = volume(iCell);
double cellVol = cellVolActual/volEqui; // scaled so that equilateral cell has vol=1.0
for (unsigned iCubPt = 0; iCubPt < numCubPoints; iCubPt++)
{
double jacDet = jacobian_det(iCell, iCubPt);
if (hasGoodTopo && jacDet < m_badJacobian)
{
++foundBad;
}
double cellVolNotZero = fabs(cellVol) < 1.e-20? 1.e-20 : cellVol;
double quality_measure_1 = (cellVolNotZero < 0? -1.0 : 1.0) * min_edge_length / pow(fabs(cellVolNotZero), 1./(double(spaceDim)));
if (0 && iCubPt==0)
{
std::cout << "quality_measure_1= " << quality_measure_1 << " cellVolNotZero= " << cellVolNotZero << " cellVolActual= "
<< cellVolActual << " volEqui= " << volEqui << " min_edge_length= " << min_edge_length
<< " max_edge_length= " << max_edge_length << std::endl;
}
double quality_measure_2 = min_edge_length / max_edge_lengthNotZero;
if (ipass == 0)
{
jdata.jac.registerValue(elem.identifier(), jacDet);
jdata.QM_1.registerValue(elem.identifier(), quality_measure_1);
jdata.QM_2.registerValue(elem.identifier(), quality_measure_2);
}
}
}
if (m_dump)
{
for (unsigned iCell = 0; iCell < numCells; iCell++)
{
for (unsigned iCubPt = 0; iCubPt < numCubPoints; iCubPt++)
{
stk::PrintTable table;
std::ostringstream msg; msg << "Jacobian"<<" iCell= "<<iCell<<" iCubPt= "<<iCubPt << " Det= " << jacobian_det(iCell, iCubPt);
table.setTitle(msg.str());
for (unsigned id = 0; id < spaceDim; id++)
{
for (unsigned jd = 0; jd < spaceDim; jd++)
{
table << jacobian(iCell, iCubPt, id, jd);
}
table << stk::end_row;
}
std::cout << "P["<< bulk.parallel_rank() << "] " << cell_topo.getName() << "\n" << table;
}
}
}
}
} // buckets
// setup the histogram ranges and counts
} // ipass
for (jac_data_map::iterator itMap = jac_data.begin(); itMap != jac_data.end(); itMap++)
{
itMap->second.jac.finish(bulk);
itMap->second.QM_1.finish(bulk);
itMap->second.QM_2.finish(bulk);
}
// all_reduce( mesh.parallel() , ReduceMax<1>( & error_flag ) );
stk::PrintTable table;
if (0)
{
const unsigned rank = bulk.parallel_rank();
std::string title = "Jacobian and Quality Table P["+toString(rank)+"]\n";
table.setTitle(title.c_str());
}
table.setTitle("Jacobian and Quality Table\n");
table << "|" << "Element Type" << "|"
<< "Min JacDet" << "|" << "Id" << "|"
<< "Max JacDet" << "|" << "Id" << "|"
<< "Ave JacDet" << "|"
<< "Sum JacDet" << "|"
<< "Min QM1" << "|" << "Id" << "|"
<< "Max QM1" << "|" << "Id" << "|"
<< "Ave QM1" << "|"
<< "Min QM2" << "|" << "Id" << "|"
<< "Max QM2" << "|" << "Id" << "|"
<< "Ave QM2" << "|"
<< stk::end_header;
for (jac_data_map::iterator itMap = jac_data.begin(); itMap != jac_data.end(); itMap++)
{
if (1)
{
std::cout << "P[" << p_rank << "] nele = " << itMap->second.numEle << std::endl;
}
table << "|" << itMap->first << "|"
<< itMap->second.jac.min << "|"
<< itMap->second.jac.min_i << "|"
<< itMap->second.jac.max << "|"
<< itMap->second.jac.max_i << "|"
<< itMap->second.jac.ave << "|"
<< itMap->second.jac.sum << "|"
<< itMap->second.QM_1.min << "|"
<< itMap->second.QM_1.min_i << "|"
<< itMap->second.QM_1.max << "|"
<< itMap->second.QM_1.max_i << "|"
<< itMap->second.QM_1.ave << "|"
<< itMap->second.QM_2.min << "|"
<< itMap->second.QM_2.min_i << "|"
<< itMap->second.QM_2.max << "|"
<< itMap->second.QM_2.max_i << "|"
<< itMap->second.QM_2.ave << "|"
<< stk::end_row;
}
if (!p_rank && printTable)
//if (printTable)
{
std::cout << "P[" << p_rank << "] Explanation: JacDet=det(element jacobian), QM1=min(element edge length)/(elemement vol)^(1/dim), QM2=min(element edge length)/max(element edge length)\n"
<< " NOTE: QM1 is normalized to 1 for ideally shaped elements, < 1 or > 1 values signify badly shaped elements\n"
<< " NOTE: QM2 is small for badly shaped elements, normalized to 1 for ideally shaped elements\n"
<< std::endl;
std::cout << table;
}
return (foundBad > 0);
}
/**
* Algorithm:
* 1. for each element, loop over its edges, for each edge's node, loop over elements attached to node
* 1a. for each neighboring element, check if current edge is invalid
*/
bool TopologyVerifier::isTopologyBad(stk::mesh::BulkData& bulk) //, stk::mesh::Part& mesh_part )
{
const stk::mesh::fem::FEMMetaData& meta = stk::mesh::fem::FEMMetaData::get(bulk);
stk::mesh::Field<double, stk::mesh::Cartesian> *coord_field =
meta.get_field<stk::mesh::Field<double, stk::mesh::Cartesian> >("coordinates");
//mesh::Selector select_owned( meta_data.locally_owned_part() );
const std::vector<mesh::Bucket*> & buckets = bulk.buckets(meta.element_rank() );
for ( std::vector<mesh::Bucket *>::const_iterator ik = buckets.begin() ; ik != buckets.end() ; ++ik )
{
// if ( select_owned( **ik ) ) {...
const mesh::Bucket & bucket = **ik ;
// Number of elems in this bucket of elems and elem field data
const unsigned number_elems = bucket.size();
double * elem_node_data = field_data( *coord_field , bucket.begin() );
//double * elem_centroid_data = field_data( elem_centroid_field , bucket.begin() );
// FIXME
if (0) { elem_node_data[0]++;}
#if 1
const CellTopologyData * const bucket_cell_topo = stk::percept::PerceptMesh::get_cell_topology(bucket);
int bucket_shardsId = ShardsInterfaceTable::s_singleton.lookupShardsId(bucket_cell_topo->name);
#endif
//if (0) { std::cout << bucket_cell_topo->name; }
if (0) { std::cout << "bucket_shardsId= " << bucket_shardsId << " name= " << bucket_cell_topo->name << std::endl; }
if (0) { std::cout << "number_elems= " << number_elems << std::endl;}
for ( unsigned i = 0 ; i < number_elems ; ++i)
{
mesh::Entity & elem = bucket[i] ;
bool isDuplicateNode = isTopologyBad(elem);
if (isDuplicateNode)
{
std::cout << "duplicate node found: elem = " << elem << std::endl;
return true;
}
if (0) std::cout << "elemOfBucket= " << elem << std::endl;
const mesh::PairIterRelation elem_nodes = elem.relations( stk::mesh::fem::FEMMetaData::NODE_RANK );
//const CellTopologyData * const cell_topo = stk::percept::PerceptMesh::get_cell_topology(elem);
const CellTopologyData * const cell_topo = stk::percept::PerceptMesh::get_cell_topology(elem);
int shardsId = ShardsInterfaceTable::s_singleton.lookupShardsId(cell_topo->name);
if (0) { std::cout << "shardsId= " << shardsId << " name= " << cell_topo->name << std::endl; }
for (unsigned iedgeOrd = 0; iedgeOrd < cell_topo->edge_count; iedgeOrd++)
{
//const CellTopologyData_Subcell& edge =
unsigned in0 = cell_topo->edge[iedgeOrd].node[0];
unsigned in1 = cell_topo->edge[iedgeOrd].node[1];
MyEdge<unsigned> potential_bad_edge(elem_nodes[in0].entity()->identifier(), elem_nodes[in1].entity()->identifier());
//if (potential_bad_edge.getId0() == 3 && potential_bad_edge.getId1() == 6)
if (0) std::cout << "potential_bad_edge: " << potential_bad_edge.getId0() << " " << potential_bad_edge.getId1() << std::endl;
}
for (unsigned iedgeOrd = 0; iedgeOrd < cell_topo->edge_count; iedgeOrd++)
{
//const CellTopologyData_Subcell& edge =
unsigned in0 = cell_topo->edge[iedgeOrd].node[0];
unsigned in1 = cell_topo->edge[iedgeOrd].node[1];
MyEdge<unsigned> potential_bad_edge(elem_nodes[in0].entity()->identifier(), elem_nodes[in1].entity()->identifier());
//if (potential_bad_edge.getId0() == 3 && potential_bad_edge.getId1() == 6)
if (0) std::cout << "potential_bad_edge: " << potential_bad_edge.getId0() << " " << potential_bad_edge.getId1() << std::endl;
if (0 && MyEdge<unsigned>(3,6) == potential_bad_edge)
{
std::cout << "bad edge" << std::endl;
}
for (unsigned inodeOnPotBadEdge = 0; inodeOnPotBadEdge < 2; inodeOnPotBadEdge++)
{
unsigned inodeOnPotBadEdgeInElem = cell_topo->edge[iedgeOrd].node[inodeOnPotBadEdge];
const mesh::PairIterRelation node_elems = elem_nodes[inodeOnPotBadEdgeInElem].entity()->relations( meta.element_rank() );
unsigned num_elems_on_node = node_elems.size();
for (unsigned iele = 0; iele < num_elems_on_node; iele++)
{
mesh::Entity & elemOnNode = *node_elems[iele].entity();
const mesh::PairIterRelation elemOnNode_nodes = elemOnNode.relations( stk::mesh::fem::FEMMetaData::NODE_RANK );
const CellTopologyData * const local_cell_topo = stk::percept::PerceptMesh::get_cell_topology(elemOnNode);
int local_shardsId = ShardsInterfaceTable::s_singleton.lookupShardsId(local_cell_topo->name);
//if (1) { std::cout << "shardsId= " << shardsId << " name= " << cell_topo->name << std::endl; }
if (0) std::cout << "elemOnNode= " << elemOnNode << std::endl;
unsigned num_invalid_edges = m_invalid_edge_set[local_shardsId].size();
if (0) { std::cout << num_invalid_edges; }
for (invalid_edge_set_type::iterator inv_edge = m_invalid_edge_set[local_shardsId].begin();
inv_edge != m_invalid_edge_set[local_shardsId].end(); inv_edge++)
{
MyEdge<unsigned> globalIdInvEdge( elemOnNode_nodes[ (*inv_edge).getId0()].entity()->identifier(),
elemOnNode_nodes[ (*inv_edge).getId1()].entity()->identifier() );
if (0) std::cout << "globalIdInvEdge: " << globalIdInvEdge.getId0() << " " << globalIdInvEdge.getId1() << std::endl;
if(potential_bad_edge == globalIdInvEdge)
{
return true;
}
}
}
}
}
}
}
return false;
}
