std::vector<stk::mesh::EntityKeyProc> get_non_unique_key_procs(const stk::mesh::BulkData& bulkData)
{
stk::parallel::DistributedIndex distributedIndex( bulkData.parallel(), stk::mesh::impl::convert_entity_keys_to_spans(bulkData.mesh_meta_data()));
add_keys_to_distributed_index(bulkData, distributedIndex);
stk::parallel::DistributedIndex::KeyTypeVector localKeys = get_all_local_keys(bulkData);
return get_non_unique_keys(bulkData, distributedIndex, localKeys);
}
void populate_part_ordinals_for_remote_edges(const stk::mesh::BulkData& bulkData, const ElemElemGraph& graph, ParallelPartInfo ¶llelPartInfo)
{
parallelPartInfo.clear();
stk::CommSparse comm(bulkData.parallel());
pack_data_for_part_ordinals(comm, graph, bulkData);
comm.allocate_buffers();
pack_data_for_part_ordinals(comm, graph, bulkData);
comm.communicate();
unpack_and_update_part_ordinals(comm, bulkData, graph, parallelPartInfo);
}
void populate_selected_value_for_remote_elements(const stk::mesh::BulkData& bulkData,
ElemElemGraph& graph,
stk::mesh::Selector selector,
ParallelSelectedInfo &selInfo)
{
selInfo.clear();
stk::CommSparse comm(bulkData.parallel());
pack_and_communicate_selector(bulkData, comm, graph, selector);
unpack_and_update_selector_value(comm, bulkData, graph, selInfo);
}
void finish(stk::mesh::BulkData& mesh)
{
//double parallel_max = max;
//std::cout << "rank= " << mesh.parallel_rank() << " max before= " << max << std::endl;
double min_local = min;
double max_local = max;
// double ave_local = ave;
// double sum_local = sum;
// double numAve_local = numAve;
// unsigned min_i_local = min_i;
// unsigned max_i_local = max_i;
all_reduce( mesh.parallel() , ReduceMax<1>( & max ) );
all_reduce( mesh.parallel() , ReduceMin<1>( & min ) );
all_reduce( mesh.parallel() , ReduceSum<1>( & numAve ) );
all_reduce( mesh.parallel() , ReduceSum<1>( & ave ) );
all_reduce( mesh.parallel() , ReduceSum<1>( & sum ) );
// if this proc doesn't have the max then reset the local max_i to 0, do ReduceMax, thereby picking up
// the value from the proc that does own the actual max_i
if (std::fabs(max-max_local) > 1.e-10)
{
max_i = std::numeric_limits<unsigned>::min();
}
if (std::fabs(min-min_local) > 1.e-10)
{
min_i = std::numeric_limits<unsigned>::max();
}
//std::cout << "P[" << mesh.parallel_rank() << "] max_i before= " << max_i << " max= " << max << " max_local= " << max_local << std::endl;
//std::cout << "P[" << mesh.parallel_rank() << "] min_i before= " << min_i << " min= " << min << " min_local= " << min_local << std::endl;
all_reduce( mesh.parallel() , ReduceMax<1>( & max_i ) );
all_reduce( mesh.parallel() , ReduceMin<1>( & min_i ) );
//std::cout << "P[" << mesh.parallel_rank() << "] max_i after = " << max_i << " max= " << max << " max_local= " << max_local << std::endl;
//std::cout << "rank= " << mesh.parallel_rank() << " max after= " << max << std::endl;
ave /= std::max(numAve,1.e-20);
}
void use_case_5_generate_mesh(
const std::string& mesh_options ,
stk::mesh::BulkData & mesh ,
const VectorFieldType & node_coord ,
stk::mesh::Part & hex_block ,
stk::mesh::Part & quad_shell_block )
{
mesh.modification_begin();
const unsigned parallel_size = mesh.parallel_size();
const unsigned parallel_rank = mesh.parallel_rank();
double t = 0 ;
size_t num_hex = 0 ;
size_t num_shell = 0 ;
size_t num_nodes = 0 ;
size_t num_block = 0 ;
int error_flag = 0 ;
try {
Iogn::GeneratedMesh gmesh( mesh_options, parallel_size, parallel_rank );
num_nodes = gmesh.node_count_proc();
num_block = gmesh.block_count();
t = stk::wall_time();
std::vector<int> node_map( num_nodes , 0 );
gmesh.node_map( node_map );
{
for ( size_t i = 1 ; i <= num_block ; ++i ) {
const size_t num_elem = gmesh.element_count_proc(i);
const std::pair<std::string,int> top_info = gmesh.topology_type(i);
std::vector<int> elem_map( num_elem , 0 );
std::vector<int> elem_conn( num_elem * top_info.second );
gmesh.element_map( i, elem_map );
gmesh.connectivity( i , elem_conn );
if ( top_info.second == 8 ) {
for ( size_t j = 0 ; j < num_elem ; ++j ) {
const int * const local_node_id = & elem_conn[ j * 8 ] ;
const stk::mesh::EntityId node_id[8] = {
local_node_id[0] ,
local_node_id[1] ,
local_node_id[2] ,
local_node_id[3] ,
local_node_id[4] ,
local_node_id[5] ,
local_node_id[6] ,
local_node_id[7]
};
const stk::mesh::EntityId elem_id = elem_map[ j ];
stk::mesh::fem::declare_element( mesh , hex_block , elem_id , node_id );
++num_hex ;
}
}
else if ( top_info.second == 4 ) {
for ( size_t j = 0 ; j < num_elem ; ++j ) {
const int * const local_node_id = & elem_conn[ j * 4 ] ;
const stk::mesh::EntityId node_id[4] = {
local_node_id[0] ,
local_node_id[1] ,
local_node_id[2] ,
local_node_id[3]
};
const stk::mesh::EntityId elem_id = elem_map[ j ];
stk::mesh::fem::declare_element( mesh , quad_shell_block , elem_id , node_id );
++num_shell ;
}
}
}
}
std::vector<double> node_coordinates( 3 * node_map.size() );
gmesh.coordinates( node_coordinates );
if ( 3 * node_map.size() != node_coordinates.size() ) {
std::ostringstream msg ;
msg << " P" << mesh.parallel_rank()
<< ": ERROR, node_map.size() = "
<< node_map.size()
<< " , node_coordinates.size() / 3 = "
<< ( node_coordinates.size() / 3 );
throw std::runtime_error( msg.str() );
}
for ( unsigned i = 0 ; i < node_map.size() ; ++i ) {
const unsigned i3 = i * 3 ;
stk::mesh::Entity * const node = mesh.get_entity( stk::mesh::fem::FEMMetaData::NODE_RANK , node_map[i] );
if ( NULL == node ) {
std::ostringstream msg ;
msg << " P:" << mesh.parallel_rank()
<< " ERROR, Node not found: "
<< node_map[i] << " = node_map[" << i << "]" ;
throw std::runtime_error( msg.str() );
}
double * const data = field_data( node_coord , *node );
data[0] = node_coordinates[ i3 + 0 ];
data[1] = node_coordinates[ i3 + 1 ];
data[2] = node_coordinates[ i3 + 2 ];
}
}
catch ( const std::exception & X ) {
std::cout << " P:" << mesh.parallel_rank() << ": " << X.what()
<< std::endl ;
std::cout.flush();
error_flag = 1 ;
}
catch( ... ) {
std::cout << " P:" << mesh.parallel_rank()
<< " Caught unknown exception"
<< std::endl ;
std::cout.flush();
error_flag = 1 ;
}
stk::all_reduce( mesh.parallel() , stk::ReduceMax<1>( & error_flag ) );
if ( error_flag ) {
std::string msg( "Failed mesh generation" );
throw std::runtime_error( msg );
}
mesh.modification_end();
double dt = stk::wall_dtime( t );
stk::all_reduce( mesh.parallel() , stk::ReduceMax<1>( & dt ) );
std::cout << " P" << mesh.parallel_rank()
<< ": Meshed Hex = " << num_hex
<< " , Shell = " << num_shell
<< " , Node = " << num_nodes
<< " in " << dt << " sec"
<< std::endl ;
std::cout.flush();
}
void communicate_field_data( const stk::mesh::BulkData & mesh ,
const std::vector< const stk::mesh::FieldBase * > & fields )
{
if ( fields.empty() ) { return; }
const unsigned parallel_size = mesh.parallel_size();
const unsigned parallel_rank = mesh.parallel_rank();
// Sizing for send and receive
const unsigned zero = 0 ;
std::vector<unsigned> send_size( parallel_size , zero );
std::vector<unsigned> recv_size( parallel_size , zero );
std::vector<unsigned> procs ;
for ( std::vector<stk::mesh::Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
stk::mesh::Entity & e = **i ;
unsigned size = 0 ;
for ( std::vector<const stk::mesh::FieldBase *>::const_iterator
fi = fields.begin() ; fi != fields.end() ; ++fi ) {
const stk::mesh::FieldBase & f = **fi ;
size += stk::mesh::field_data_size( f , e );
}
if ( size ) {
if ( e.owner_rank() == parallel_rank ) {
// owner sends
stk::mesh::comm_procs( e , procs );
for ( std::vector<unsigned>::iterator
ip = procs.begin() ; ip != procs.end() ; ++ip ) {
send_size[ *ip ] += size ;
}
}
else {
// non-owner receives
recv_size[ e.owner_rank() ] += size ;
}
}
}
// Allocate send and receive buffers:
stk::CommAll sparse ;
{
const unsigned * const s_size = & send_size[0] ;
const unsigned * const r_size = & recv_size[0] ;
sparse.allocate_buffers( mesh.parallel(), parallel_size / 4 , s_size, r_size);
}
// Send packing:
for ( std::vector<stk::mesh::Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
stk::mesh::Entity & e = **i ;
if ( e.owner_rank() == parallel_rank ) {
stk::mesh::comm_procs( e , procs );
for ( std::vector<const stk::mesh::FieldBase *>::const_iterator
fi = fields.begin() ; fi != fields.end() ; ++fi ) {
const stk::mesh::FieldBase & f = **fi ;
const unsigned size = stk::mesh::field_data_size( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , e ));
for ( std::vector<unsigned>::iterator
ip = procs.begin() ; ip != procs.end() ; ++ip ) {
stk::CommBuffer & b = sparse.send_buffer( *ip );
b.pack<unsigned char>( ptr , size );
}
}
}
}
}
// Communicate:
sparse.communicate();
// Unpack for recv:
for ( std::vector<stk::mesh::Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
stk::mesh::Entity & e = **i ;
if ( e.owner_rank() != parallel_rank ) {
for ( std::vector<const stk::mesh::FieldBase *>::const_iterator
fi = fields.begin() ; fi != fields.end() ; ++fi ) {
const stk::mesh::FieldBase & f = **fi ;
const unsigned size = stk::mesh::field_data_size( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , e ));
stk::CommBuffer & b = sparse.recv_buffer( e.owner_rank() );
b.unpack<unsigned char>( ptr , size );
}
}
}
}
}
void fixup_ghosted_to_shared_nodes(stk::mesh::BulkData & bulk)
{
stk::mesh::EntityVector ghosted_nodes_that_are_now_shared;
find_ghosted_nodes_that_need_to_be_shared(bulk, ghosted_nodes_that_are_now_shared);
stk::CommSparse comm(bulk.parallel());
for (int phase=0;phase<2;++phase)
{
for (size_t i = 0; i < ghosted_nodes_that_are_now_shared.size(); ++i)
{
stk::mesh::Entity node = ghosted_nodes_that_are_now_shared[i];
int proc = bulk.parallel_owner_rank(node);
comm.send_buffer(proc).pack<stk::mesh::EntityKey>(bulk.entity_key(node));
}
if (phase == 0 )
{
comm.allocate_buffers();
}
else
{
comm.communicate();
}
}
stk::mesh::EntityVector sharedNodes;
for (int process=0;process<bulk.parallel_size();++process)
{
while(comm.recv_buffer(process).remaining())
{
stk::mesh::EntityKey key;
comm.recv_buffer(process).unpack<stk::mesh::EntityKey>(key);
stk::mesh::Entity entity = bulk.get_entity(key);
if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) )
{
bulk.add_node_sharing(entity, process);
sharedNodes.push_back(entity);
}
}
}
/////////////////////////
stk::CommSparse commSecondStage(bulk.parallel());
for (int phase=0;phase<2;++phase)
{
for (size_t i=0;i<sharedNodes.size();++i)
{
std::vector<int> procs;
stk::mesh::EntityKey key = bulk.entity_key(sharedNodes[i]);
bulk.comm_shared_procs(key, procs);
for (size_t j=0;j<procs.size();++j)
{
if ( procs[j] != bulk.parallel_rank() )
{
commSecondStage.send_buffer(procs[j]).pack<int>(bulk.parallel_rank()).pack<stk::mesh::EntityKey>(key);
for (size_t k=0;k<procs.size();++k)
{
commSecondStage.send_buffer(procs[j]).pack<int>(procs[k]).pack<stk::mesh::EntityKey>(key);
}
}
}
}
if (phase == 0 )
{
commSecondStage.allocate_buffers();
}
else
{
commSecondStage.communicate();
}
}
for (int proc_that_sent_message=0;proc_that_sent_message<bulk.parallel_size();++proc_that_sent_message)
{
if ( proc_that_sent_message == bulk.parallel_rank() ) continue;
while(commSecondStage.recv_buffer(proc_that_sent_message).remaining())
{
stk::mesh::EntityKey key;
int sharingProc;
commSecondStage.recv_buffer(proc_that_sent_message).unpack<int>(sharingProc).unpack<stk::mesh::EntityKey>(key);
if ( sharingProc != bulk.parallel_rank() )
{
stk::mesh::Entity entity = bulk.get_entity(key);
if ( bulk.state(entity) != stk::mesh::Deleted && bulk.is_valid(entity) && !bulk.in_shared(key, sharingProc) )
{
bulk.add_node_sharing(entity, sharingProc);
}
}
}
}
}