void print_entity_proc_map( stk::diag::Writer & writer , const stk::mesh::BulkData & mesh ) { const stk::mesh::MetaData & meta = mesh.mesh_meta_data(); const std::vector<stk::mesh::Entity*> & comm = mesh.entity_comm(); const std::vector<stk::mesh::Ghosting*> & ghost = mesh.ghostings(); size_t counter = 0 ; for ( size_t ig = 0 ; ig < ghost.size() ; ++ig ) { const stk::mesh::Ghosting & g = * ghost[ig] ; writer << "P" << mesh.parallel_rank() << " " << g.name() << " Communication:" << std::endl ; for ( std::vector<stk::mesh::Entity*>::const_iterator i = comm.begin() ; i != comm.end() ; ++i ) { const stk::mesh::Entity & entity = **i ; std::vector<unsigned> procs ; stk::mesh::comm_procs( g , entity , procs ); if ( ! procs.empty() ) { writer << "[" << counter << "] " << meta.entity_rank_name( entity.entity_rank() ) << "[" << entity.identifier() << " " ; if ( entity.owner_rank() != mesh.parallel_rank() ) { writer << "not_" ; } writer << "owned ] {" ; for ( size_t j = 0 ; j < procs.size() ; ++j ) { writer << " " << procs[j] ; } writer << " }" << std::endl ; } } } }
void communicate_field_data( const stk::mesh::BulkData & mesh , const std::vector< const stk::mesh::FieldBase * > & fields ) { if ( fields.empty() ) { return; } const unsigned parallel_size = mesh.parallel_size(); const unsigned parallel_rank = mesh.parallel_rank(); // Sizing for send and receive const unsigned zero = 0 ; std::vector<unsigned> send_size( parallel_size , zero ); std::vector<unsigned> recv_size( parallel_size , zero ); std::vector<unsigned> procs ; for ( std::vector<stk::mesh::Entity*>::const_iterator i = mesh.entity_comm().begin() ; i != mesh.entity_comm().end() ; ++i ) { stk::mesh::Entity & e = **i ; unsigned size = 0 ; for ( std::vector<const stk::mesh::FieldBase *>::const_iterator fi = fields.begin() ; fi != fields.end() ; ++fi ) { const stk::mesh::FieldBase & f = **fi ; size += stk::mesh::field_data_size( f , e ); } if ( size ) { if ( e.owner_rank() == parallel_rank ) { // owner sends stk::mesh::comm_procs( e , procs ); for ( std::vector<unsigned>::iterator ip = procs.begin() ; ip != procs.end() ; ++ip ) { send_size[ *ip ] += size ; } } else { // non-owner receives recv_size[ e.owner_rank() ] += size ; } } } // Allocate send and receive buffers: stk::CommAll sparse ; { const unsigned * const s_size = & send_size[0] ; const unsigned * const r_size = & recv_size[0] ; sparse.allocate_buffers( mesh.parallel(), parallel_size / 4 , s_size, r_size); } // Send packing: for ( std::vector<stk::mesh::Entity*>::const_iterator i = mesh.entity_comm().begin() ; i != mesh.entity_comm().end() ; ++i ) { stk::mesh::Entity & e = **i ; if ( e.owner_rank() == parallel_rank ) { stk::mesh::comm_procs( e , procs ); for ( std::vector<const stk::mesh::FieldBase *>::const_iterator fi = fields.begin() ; fi != fields.end() ; ++fi ) { const stk::mesh::FieldBase & f = **fi ; const unsigned size = stk::mesh::field_data_size( f , e ); if ( size ) { unsigned char * ptr = reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , e )); for ( std::vector<unsigned>::iterator ip = procs.begin() ; ip != procs.end() ; ++ip ) { stk::CommBuffer & b = sparse.send_buffer( *ip ); b.pack<unsigned char>( ptr , size ); } } } } } // Communicate: sparse.communicate(); // Unpack for recv: for ( std::vector<stk::mesh::Entity*>::const_iterator i = mesh.entity_comm().begin() ; i != mesh.entity_comm().end() ; ++i ) { stk::mesh::Entity & e = **i ; if ( e.owner_rank() != parallel_rank ) { for ( std::vector<const stk::mesh::FieldBase *>::const_iterator fi = fields.begin() ; fi != fields.end() ; ++fi ) { const stk::mesh::FieldBase & f = **fi ; const unsigned size = stk::mesh::field_data_size( f , e ); if ( size ) { unsigned char * ptr = reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , e )); stk::CommBuffer & b = sparse.recv_buffer( e.owner_rank() ); b.unpack<unsigned char>( ptr , size ); } } } } }