void BulkData::generate_new_entities(const std::vector<size_t>& requests,
std::vector<Entity *>& requested_entities)
{
Trace_("stk_classic::mesh::BulkData::generate_new_entities");
typedef stk_classic::parallel::DistributedIndex::KeyType KeyType;
std::vector< std::vector<KeyType> >
requested_key_types;
m_entities_index.generate_new_keys(requests, requested_key_types);
//generating 'owned' entities
Part * const owns = & m_mesh_meta_data.locally_owned_part();
std::vector<Part*> rem ;
std::vector<Part*> add;
add.push_back( owns );
requested_entities.clear();
unsigned cnt=0;
for (std::vector< std::vector<KeyType> >::const_iterator itr = requested_key_types.begin(); itr != requested_key_types.end(); ++itr) {
const std::vector<KeyType>& key_types = *itr;
for (std::vector<KeyType>::const_iterator
kitr = key_types.begin(); kitr != key_types.end(); ++kitr) {
++cnt;
}
}
requested_entities.reserve(cnt);
for (std::vector< std::vector<KeyType> >::const_iterator itr = requested_key_types.begin(); itr != requested_key_types.end(); ++itr) {
const std::vector<KeyType>& key_types = *itr;
for (std::vector<KeyType>::const_iterator
kitr = key_types.begin(); kitr != key_types.end(); ++kitr) {
EntityKey key(&(*kitr));
std::pair<Entity *, bool> result = m_entity_repo.internal_create_entity(key);
//if an entity is declare with the declare_entity function in
//the same modification cycle as the generate_new_entities
//function, and it happens to generate a key that was declare
//previously in the same cycle it is an error
ThrowErrorMsgIf( ! result.second,
"Generated " << print_entity_key(m_mesh_meta_data, key) <<
" which was already used in this modification cycle.");
// A new application-created entity
Entity* new_entity = result.first;
m_entity_repo.set_entity_owner_rank( *new_entity, m_parallel_rank);
m_entity_repo.set_entity_sync_count( *new_entity, m_sync_count);
//add entity to 'owned' part
change_entity_parts( *new_entity , add , rem );
requested_entities.push_back(new_entity);
}
}
}
Entity & BulkData::declare_entity( EntityRank ent_rank , EntityId ent_id ,
const PartVector & parts )
{
require_ok_to_modify();
require_good_rank_and_id(ent_rank, ent_id);
EntityKey key( ent_rank , ent_id );
TraceIfWatching("stk_classic::mesh::BulkData::declare_entity", LOG_ENTITY, key);
DiagIfWatching(LOG_ENTITY, key, "declaring entity with parts " << parts);
std::pair< Entity * , bool > result = m_entity_repo.internal_create_entity( key );
Entity* declared_entity = result.first;
if ( result.second ) {
// A new application-created entity
m_entity_repo.set_entity_owner_rank( *declared_entity, m_parallel_rank);
m_entity_repo.set_entity_sync_count( *declared_entity, m_sync_count);
DiagIfWatching(LOG_ENTITY, key, "new entity: " << *declared_entity);
}
else {
// An existing entity, the owner must match.
require_entity_owner( *declared_entity , m_parallel_rank );
DiagIfWatching(LOG_ENTITY, key, "existing entity: " << *declared_entity);
}
//------------------------------
Part * const owns = & m_mesh_meta_data.locally_owned_part();
std::vector<Part*> rem ;
std::vector<Part*> add( parts );
add.push_back( owns );
change_entity_parts( *declared_entity , add , rem );
// m_transaction_log.insert_entity ( *(result.first) );
return *declared_entity ;
}
void
createNewElements(percept::PerceptMesh& eMesh, NodeRegistry& nodeRegistry,
stk_classic::mesh::Entity& element, NewSubEntityNodesType& new_sub_entity_nodes, vector<stk_classic::mesh::Entity *>::iterator& element_pool,
stk_classic::mesh::FieldBase *proc_rank_field=0)
{
const CellTopologyData * const cell_topo_data = stk_classic::percept::PerceptMesh::get_cell_topology(element);
typedef boost::tuple<stk_classic::mesh::EntityId, stk_classic::mesh::EntityId, stk_classic::mesh::EntityId> tri_tuple_type;
static vector<tri_tuple_type> elems(6);
CellTopology cell_topo(cell_topo_data);
const stk_classic::mesh::PairIterRelation elem_nodes = element.relations(stk_classic::mesh::fem::FEMMetaData::NODE_RANK);
//stk_classic::mesh::Part & active = mesh->ActivePart();
//stk_classic::mesh::Part & quad4 = mesh->QuadPart();
std::vector<stk_classic::mesh::Part*> add_parts;
std::vector<stk_classic::mesh::Part*> remove_parts;
//add_parts.push_back( &active );
//FIXME
//add_parts.push_back( const_cast<mesh::Part*>( eMesh.getPart(m_toTopoPartName) ));
add_parts = m_toParts;
/**
\node[above] at (p4.side 1){2};
\node[left] at (p4.side 2){3};
\node[below] at (p4.side 3){0};
\node[right] at (p4.side 4){1};
*/
double tmp_x[3];
for (int iedge = 0; iedge < 4; iedge++)
{
double * mp = midPoint(EDGE_COORD(iedge,0), EDGE_COORD(iedge,1), eMesh.get_spatial_dim(), tmp_x);
if (!EDGE_N(iedge))
{
std::cout << "P[" << eMesh.get_rank() << " nid ## = 0 << " << std::endl;
}
eMesh.createOrGetNode(EDGE_N(iedge), mp);
}
elems[0] = tri_tuple_type(VERT_N(0), EDGE_N(0), EDGE_N(3));
elems[1] = tri_tuple_type(VERT_N(1), EDGE_N(1), EDGE_N(0));
elems[2] = tri_tuple_type(EDGE_N(0), EDGE_N(1), EDGE_N(3));
elems[3] = tri_tuple_type(VERT_N(2), EDGE_N(2), EDGE_N(1));
elems[4] = tri_tuple_type(VERT_N(3), EDGE_N(3), EDGE_N(2));
elems[5] = tri_tuple_type(EDGE_N(2), EDGE_N(3), EDGE_N(1));
// write a diagram of the refinement pattern as a vtk file, or a latex/tikz/pgf file
#define WRITE_DIAGRAM 0
#if WRITE_DIAGRAM
#endif
for (unsigned ielem=0; ielem < elems.size(); ielem++)
{
//stk_classic::mesh::Entity& newElement = eMesh.get_bulk_data()->declare_entity(Element, *element_id_pool, eMesh.getPart(interface_table::shards_Triangle_3) );
//stk_classic::mesh::Entity& newElement = eMesh.get_bulk_data()->declare_entity(Element, *element_id_pool, eMesh.getPart(interface_table::shards_Triangle_3) );
stk_classic::mesh::Entity& newElement = *(*element_pool);
if (proc_rank_field)
{
double *fdata = stk_classic::mesh::field_data( *static_cast<const ScalarFieldType *>(proc_rank_field) , newElement );
//fdata[0] = double(m_eMesh.get_rank());
fdata[0] = double(newElement.owner_rank());
}
//eMesh.get_bulk_data()->change_entity_parts( newElement, add_parts, remove_parts );
change_entity_parts(eMesh, element, newElement);
{
if (!elems[ielem].get<0>())
{
std::cout << "P[" << eMesh.get_rank() << " nid = 0 << " << std::endl;
exit(1);
}
}
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<0>()), 0);
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<1>()), 1);
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<2>()), 2);
set_parent_child_relations(eMesh, element, newElement, ielem);
element_pool++;
}
}
void
createNewElements(percept::PerceptMesh& eMesh, NodeRegistry& nodeRegistry,
stk::mesh::Entity& element, NewSubEntityNodesType& new_sub_entity_nodes, vector<stk::mesh::Entity *>::iterator& element_pool,
stk::mesh::FieldBase *proc_rank_field=0)
{
const CellTopologyData * const cell_topo_data = stk::percept::PerceptMesh::get_cell_topology(element);
typedef boost::tuple<stk::mesh::EntityId, stk::mesh::EntityId> line_tuple_type;
static vector<line_tuple_type> elems(2);
CellTopology cell_topo(cell_topo_data);
const stk::mesh::PairIterRelation elem_nodes = element.relations(stk::mesh::fem::FEMMetaData::NODE_RANK);
std::vector<stk::mesh::Part*> add_parts;
std::vector<stk::mesh::Part*> remove_parts;
add_parts = m_toParts;
unsigned num_nodes_on_edge = new_sub_entity_nodes[m_eMesh.edge_rank()][0].size();
if (!num_nodes_on_edge)
return;
double coord_x[3];
for (int iedge = 0; iedge < 1; iedge++)
{
//double * mp = midPoint(EDGE_COORD(iedge,0), EDGE_COORD(iedge,1), eMesh.get_spatial_dim(), coord_x);
//double * mp = midPoint(FACE_COORD(iedge,0), FACE_COORD(iedge,1), eMesh.get_spatial_dim(), coord_x);
double * mp = midPoint(VERT_COORD(0), VERT_COORD(1), eMesh.get_spatial_dim(), coord_x);
if (!EDGE_N(iedge))
{
std::cout << "P[" << eMesh.get_rank() << " nid ## = 0 " << std::endl;
}
eMesh.createOrGetNode(EDGE_N(iedge), mp);
}
// FIXME
nodeRegistry.makeCentroidCoords(*const_cast<stk::mesh::Entity *>(&element), m_primaryEntityRank, 0u);
nodeRegistry.addToExistingParts(*const_cast<stk::mesh::Entity *>(&element), m_primaryEntityRank, 0u);
nodeRegistry.interpolateFields(*const_cast<stk::mesh::Entity *>(&element), m_primaryEntityRank, 0u);
Elem::CellTopology elem_celltopo = Elem::getCellTopology< FromTopology >();
const Elem::RefinementTopology* ref_topo_p = Elem::getRefinementTopology(elem_celltopo);
const Elem::RefinementTopology& ref_topo = *ref_topo_p;
#ifndef NDEBUG
unsigned num_child = ref_topo.num_child();
VERIFY_OP(num_child, == , 2, "createNewElements num_child problem");
bool homogeneous_child = ref_topo.homogeneous_child();
VERIFY_OP(homogeneous_child, ==, true, "createNewElements homogeneous_child");
#endif
// new_sub_entity_nodes[i][j]
//const UInt * const * child_nodes() const {
//const UInt * child_node_0 = ref_topo.child_node(0);
typedef Elem::StdMeshObjTopologies::RefTopoX RefTopoX;
RefTopoX& l2 = Elem::StdMeshObjTopologies::RefinementTopologyExtra< FromTopology > ::refinement_topology;
#define CENTROID_N NN(m_primaryEntityRank,0)
for (unsigned iChild = 0; iChild < 2; iChild++)
{
unsigned EN[2];
for (unsigned jNode = 0; jNode < 2; jNode++)
{
unsigned childNodeIdx = ref_topo.child_node(iChild)[jNode];
#ifndef NDEBUG
unsigned childNodeIdxCheck = l2[childNodeIdx].ordinal_of_node;
VERIFY_OP(childNodeIdx, ==, childNodeIdxCheck, "childNodeIdxCheck");
#endif
unsigned inode=0;
if (l2[childNodeIdx].rank_of_subcell == 0)
inode = VERT_N(l2[childNodeIdx].ordinal_of_subcell);
else if (l2[childNodeIdx].rank_of_subcell == 1)
inode = EDGE_N(l2[childNodeIdx].ordinal_of_subcell);
// else if (l2[childNodeIdx].rank_of_subcell == 2)
// inode = CENTROID_N;
EN[jNode] = inode;
}
elems[iChild] = line_tuple_type(EN[0], EN[1]);
}
#undef CENTROID_N
for (unsigned ielem=0; ielem < elems.size(); ielem++)
{
stk::mesh::Entity& newElement = *(*element_pool);
#if 0
if (proc_rank_field && proc_rank_field->rank() == m_eMesh.edge_rank()) //&& m_eMesh.get_spatial_dim()==1)
{
double *fdata = stk::mesh::field_data( *static_cast<const ScalarFieldType *>(proc_rank_field) , newElement );
//fdata[0] = double(m_eMesh.get_rank());
fdata[0] = double(newElement.owner_rank());
}
#endif
stk::mesh::FieldBase * proc_rank_field_edge = m_eMesh.get_field("proc_rank_edge");
if (proc_rank_field_edge)
{
double *fdata = stk::mesh::field_data( *static_cast<const ScalarFieldType *>(proc_rank_field_edge) , newElement );
fdata[0] = double(newElement.owner_rank());
//fdata[0] = 1234.56;
if (0)
std::cout << "P[" << m_eMesh.get_rank() << "] tmp set proc_rank_field_edge to value = " << newElement.owner_rank()
<< " for side element = " << newElement.identifier()
<< std::endl;
}
//eMesh.get_bulk_data()->change_entity_parts( newElement, add_parts, remove_parts );
change_entity_parts(eMesh, element, newElement);
{
if (!elems[ielem].get<0>())
{
std::cout << "P[" << eMesh.get_rank() << " nid = 0 " << std::endl;
exit(1);
}
}
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<0>()), 0);
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<1>()), 1);
set_parent_child_relations(eMesh, element, newElement, ielem);
element_pool++;
}
}
void
createNewElements(percept::PerceptMesh& eMesh, NodeRegistry& nodeRegistry,
stk_classic::mesh::Entity& element, NewSubEntityNodesType& new_sub_entity_nodes, vector<stk_classic::mesh::Entity *>::iterator& element_pool,
stk_classic::mesh::FieldBase *proc_rank_field=0)
{
const CellTopologyData * const cell_topo_data = stk_classic::percept::PerceptMesh::get_cell_topology(element);
typedef boost::tuple<stk_classic::mesh::EntityId, stk_classic::mesh::EntityId, stk_classic::mesh::EntityId> tri_tuple_type;
static vector<tri_tuple_type> elems(4);
CellTopology cell_topo(cell_topo_data);
const stk_classic::mesh::PairIterRelation elem_nodes = element.relations(stk_classic::mesh::fem::FEMMetaData::NODE_RANK);
//stk_classic::mesh::Part & active = mesh->ActivePart();
//stk_classic::mesh::Part & quad4 = mesh->QuadPart();
std::vector<stk_classic::mesh::Part*> add_parts;
std::vector<stk_classic::mesh::Part*> remove_parts;
add_parts = m_toParts;
//std::cout << "P["<< m_eMesh.get_rank() << "] add_parts = " << add_parts << std::endl;
stk_classic::mesh::EntityRank my_rank = m_primaryEntityRank;
nodeRegistry.makeCentroidCoords(*const_cast<stk_classic::mesh::Entity *>(&element), my_rank, 0u);
nodeRegistry.addToExistingParts(*const_cast<stk_classic::mesh::Entity *>(&element), my_rank, 0u);
nodeRegistry.interpolateFields(*const_cast<stk_classic::mesh::Entity *>(&element), my_rank, 0u);
#define CENTROID_N NN(m_primaryEntityRank, 0)
elems[0] = tri_tuple_type(VERT_N(0), VERT_N(1), CENTROID_N);
elems[1] = tri_tuple_type(VERT_N(1), VERT_N(2), CENTROID_N);
elems[2] = tri_tuple_type(VERT_N(2), VERT_N(3), CENTROID_N);
elems[3] = tri_tuple_type(VERT_N(3), VERT_N(0), CENTROID_N);
#undef CENTROID_N
// write a diagram of the refinement pattern as a vtk file, or a latex/tikz/pgf file
#define WRITE_DIAGRAM 0
#if WRITE_DIAGRAM
/**
\node[above] at (p4.side 1){2};
\node[left] at (p4.side 2){3};
\node[below] at (p4.side 3){0};
\node[right] at (p4.side 4){1};
*/
#endif
for (unsigned ielem=0; ielem < elems.size(); ielem++)
{
stk_classic::mesh::Entity& newElement = *(*element_pool);
//std::cout << "P["<< m_eMesh.get_rank() << "] urp tmp 3 " << proc_rank_field << std::endl;
if (proc_rank_field && element.entity_rank() == m_eMesh.element_rank())
{
double *fdata = stk_classic::mesh::field_data( *static_cast<const ScalarFieldType *>(proc_rank_field) , newElement );
fdata[0] = double(newElement.owner_rank());
}
//std::cout << "P["<< m_eMesh.get_rank() << "] urp tmp 4 " << std::endl;
change_entity_parts(eMesh, element, newElement);
//std::cout << "P["<< m_eMesh.get_rank() << "] urp tmp 5 " << std::endl;
{
if (!elems[ielem].get<0>())
{
std::cout << "P[" << eMesh.get_rank() << " nid = 0 << " << std::endl;
exit(1);
}
}
//std::cout << "P["<< m_eMesh.get_rank() << "] urp tmp 6 " << std::endl;
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<0>()), 0);
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<1>()), 1);
eMesh.get_bulk_data()->declare_relation(newElement, eMesh.createOrGetNode(elems[ielem].get<2>()), 2);
//std::cout << "P["<< m_eMesh.get_rank() << "] urp tmp 7 " << std::endl;
set_parent_child_relations(eMesh, element, newElement, ielem);
element_pool++;
}
}
void BulkData::change_entity_owner( const std::vector<EntityProc> & arg_change )
{
static const char method[] = "stk::mesh::BulkData::change_entity_owner" ;
const MetaData & meta = m_mesh_meta_data ;
const unsigned p_rank = m_parallel_rank ;
const unsigned p_size = m_parallel_size ;
ParallelMachine p_comm = m_parallel_machine ;
//------------------------------
// Verify the input changes, generate a clean local change list, and
// generate the remote change list so that all processes know about
// pending changes.
std::vector<EntityProc> local_change( arg_change );
// Parallel synchronous clean up and verify the requested changes:
clean_and_verify_parallel_change( method , *this , local_change );
//----------------------------------------
// Parallel synchronous determination of changing
// shared and ghosted.
std::vector<EntityProc> ghosted_change ;
std::vector<EntityProc> shared_change ;
generate_parallel_change( *this , local_change ,
shared_change , ghosted_change );
//------------------------------
// Have enough information to delete all effected ghosts.
// If the closure of a ghost contains a changing entity
// then that ghost must be deleted.
// Request that all ghost entities in the closure of the ghost be deleted.
typedef std::set<EntityProc,EntityLess> EntityProcSet;
typedef std::set<Entity*,EntityLess> EntitySet;
// Closure of the owner change for impacted ghost entities.
EntityProcSet send_closure ;
for ( std::vector<EntityProc>::iterator
i = local_change.begin() ; i != local_change.end() ; ++i ) {
insert_closure_send( *i , send_closure );
}
{
EntitySet work ;
for ( std::vector<EntityProc>::const_iterator
i = ghosted_change.begin() ; i != ghosted_change.end() ; ++i ) {
insert_transitive_ghost( i->first , m_parallel_rank , work );
}
for ( std::vector<EntityProc>::const_iterator
i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
insert_transitive_ghost( i->first , m_parallel_rank , work );
}
for ( EntityProcSet::iterator
i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
insert_transitive_ghost( i->first , m_parallel_rank , work );
}
// The ghosted change list will become invalid
ghosted_change.clear();
std::vector<EntityProc> empty ;
std::vector<Entity*> effected_ghosts( work.begin() , work.end() );
// Skip 'm_ghosting[0]' which is the shared subset.
for ( std::vector<Ghosting*>::iterator
ig = m_ghosting.begin() + 1 ; ig != m_ghosting.end() ; ++ig ) {
// parallel synchronous:
internal_change_ghosting( **ig , empty , effected_ghosts );
}
}
//------------------------------
// Consistently change the owner on all processes.
// 1) The local_change list is giving away ownership.
// 2) The shared_change may or may not be receiving ownership
{
PartVector owned( 1 );
owned[0] = & meta.locally_owned_part();
for ( std::vector<EntityProc>::iterator
i = local_change.begin() ; i != local_change.end() ; ++i ) {
// Giving ownership, change the parts first and then
// the owner rank to pass the ownership test.
change_entity_parts( * i->first , PartVector() , owned );
m_entity_repo.set_entity_owner_rank( *(i->first), i->second);
}
for ( std::vector<EntityProc>::iterator
i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
m_entity_repo.set_entity_owner_rank( *(i->first), i->second);
if ( p_rank == i->second ) { // I receive ownership
change_entity_parts( * i->first , owned , PartVector() );
}
}
}
//------------------------------
// Send entities, along with their closure, to the new owner processes
{
std::ostringstream error_msg ;
int error_count = 0 ;
CommAll comm( p_comm );
for ( std::set<EntityProc,EntityLess>::iterator
i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
CommBuffer & buffer = comm.send_buffer( i->second );
Entity & entity = * i->first ;
pack_entity_info( buffer , entity );
pack_field_values( buffer , entity );
}
comm.allocate_buffers( p_size / 4 );
for ( std::set<EntityProc,EntityLess>::iterator
i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
CommBuffer & buffer = comm.send_buffer( i->second );
Entity & entity = * i->first ;
pack_entity_info( buffer , entity );
pack_field_values( buffer , entity );
}
comm.communicate();
for ( unsigned p = 0 ; p < p_size ; ++p ) {
CommBuffer & buf = comm.recv_buffer(p);
while ( buf.remaining() ) {
PartVector parts ;
std::vector<Relation> relations ;
EntityKey key ;
unsigned owner = ~0u ;
unpack_entity_info( buf, *this, key, owner, parts, relations );
// Received entity information will be correct,
// modulo the owned and shared parts
remove( parts , meta.globally_shared_part() );
if ( owner == p_rank ) {
// Must have the locally_owned_part
insert( parts , meta.locally_owned_part() );
}
else {
// Must not have the locally_owned_part
remove( parts , meta.locally_owned_part() );
}
std::pair<Entity*,bool> result =
m_entity_repo.internal_create_entity( key );
m_entity_repo.log_created_parallel_copy( *(result.first) );
// The entity was copied and not created.
m_entity_repo.set_entity_owner_rank( *(result.first), owner);
internal_change_entity_parts( *result.first , parts , PartVector() );
declare_relation( *result.first , relations );
if ( ! unpack_field_values( buf , * result.first , error_msg ) ) {
++error_count ;
}
}
}
all_reduce( p_comm , ReduceSum<1>( & error_count ) );
if ( error_count ) { throw std::runtime_error( error_msg.str() ); }
// Any entity that I sent and is not in an owned closure is deleted.
// The owned closure will be effected by received entities, so can
// only clean up after the newly owned entities have been received.
// Destroy backwards so as not to invalidate closures in the process.
{
Entity * entity = NULL ;
for ( std::set<EntityProc,EntityLess>::iterator
i = send_closure.end() ; i != send_closure.begin() ; ) {
Entity * e = (--i)->first ;
// The same entity may be sent to more than one process.
// Only evaluate it once.
if ( entity != e ) {
entity = e ;
if ( ! member_of_owned_closure( *e , p_rank ) ) {
if ( ! destroy_entity( e ) ) {
throw std::logic_error(std::string("BulkData::destroy_entity FAILED"));
}
}
}
}
}
send_closure.clear(); // Has been invalidated
}
}