void comm_procs( const Ghosting & ghost , const Entity & entity , std::vector<unsigned> & procs ) { procs.clear(); for ( PairIterEntityComm ec = entity.comm(); ! ec.empty() ; ++ec ) { if ( ec->ghost_id == ghost.ordinal() ) { procs.push_back( ec->proc ); } } }
bool in_ghost( const Ghosting & ghost , const Entity & entity , unsigned p ) { // Ghost communication from owner. EntityCommInfo tmp( ghost.ordinal() , p ); std::vector<EntityCommInfo>::const_iterator i = std::lower_bound( entity.comm().begin() , entity.comm().end() , tmp ); return i != entity.comm().end() && tmp == *i ; }
void communicate_field_data( const Ghosting & ghosts , const std::vector< const FieldBase *> & fields ) { if ( fields.empty() ) { return; } const BulkData & mesh = BulkData::get(ghosts); const unsigned parallel_size = mesh.parallel_size(); const unsigned parallel_rank = mesh.parallel_rank(); const std::vector<const FieldBase *>::const_iterator fe = fields.end(); const std::vector<const FieldBase *>::const_iterator fb = fields.begin(); std::vector<const FieldBase *>::const_iterator fi ; // Sizing for send and receive const unsigned zero = 0 ; std::vector<unsigned> send_size( parallel_size , zero ); std::vector<unsigned> recv_size( parallel_size , zero ); for ( std::vector<Entity*>::const_iterator i = mesh.entity_comm().begin() ; i != mesh.entity_comm().end() ; ++i ) { Entity & e = **i ; const bool owned = e.owner_rank() == parallel_rank ; unsigned e_size = 0 ; for ( fi = fb ; fi != fe ; ++fi ) { const FieldBase & f = **fi ; e_size += field_data_size( f , e ); } for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) { if ( ghosts.ordinal() == ec->ghost_id ) { if ( owned ) { send_size[ ec->proc ] += e_size ; } else { recv_size[ ec->proc ] += e_size ; } } } } // Allocate send and receive buffers: CommAll sparse ; { const unsigned * const s_size = & send_size[0] ; const unsigned * const r_size = & recv_size[0] ; sparse.allocate_buffers( mesh.parallel(), parallel_size / 4 , s_size, r_size); } // Send packing: for ( std::vector<Entity*>::const_iterator i = mesh.entity_comm().begin() ; i != mesh.entity_comm().end() ; ++i ) { Entity & e = **i ; if ( e.owner_rank() == parallel_rank ) { for ( fi = fb ; fi != fe ; ++fi ) { const FieldBase & f = **fi ; const unsigned size = field_data_size( f , e ); if ( size ) { unsigned char * ptr = reinterpret_cast<unsigned char *>(field_data( f , e )); for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) { if ( ghosts.ordinal() == ec->ghost_id ) { CommBuffer & b = sparse.send_buffer( ec->proc ); b.pack<unsigned char>( ptr , size ); } } } } } } // Communicate: sparse.communicate(); // Unpack for recv: for ( std::vector<Entity*>::const_iterator i = mesh.entity_comm().begin() ; i != mesh.entity_comm().end() ; ++i ) { Entity & e = **i ; if ( e.owner_rank() != parallel_rank ) { for ( fi = fb ; fi != fe ; ++fi ) { const FieldBase & f = **fi ; const unsigned size = field_data_size( f , e ); if ( size ) { unsigned char * ptr = reinterpret_cast<unsigned char *>(field_data( f , e )); for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) { if ( ghosts.ordinal() == ec->ghost_id ) { CommBuffer & b = sparse.recv_buffer( ec->proc ); b.unpack<unsigned char>( ptr , size ); } } } } } } }
//---------------------------------------------------------------------- void communicate_field_data( const Ghosting & ghosts , const std::vector< const FieldBase *> & fields ) { if ( fields.empty() ) { return; } const BulkData & mesh = ghosts.mesh(); const int parallel_size = mesh.parallel_size(); const int parallel_rank = mesh.parallel_rank(); const unsigned ghost_id = ghosts.ordinal(); const std::vector<const FieldBase *>::const_iterator fe = fields.end(); const std::vector<const FieldBase *>::const_iterator fb = fields.begin(); std::vector<const FieldBase *>::const_iterator fi ; // Sizing for send and receive const unsigned zero = 0 ; std::vector<unsigned> send_size( parallel_size , zero ); std::vector<unsigned> recv_size( parallel_size , zero ); for ( EntityCommListInfoVector::const_iterator i = mesh.internal_comm_list().begin() , iend = mesh.internal_comm_list().end(); i != iend ; ++i ) { Entity e = i->entity; const MeshIndex meshIdx = mesh.mesh_index(e); const unsigned bucketId = meshIdx.bucket->bucket_id(); const bool owned = i->owner == parallel_rank ; unsigned e_size = 0 ; for ( fi = fb ; fi != fe ; ++fi ) { const FieldBase & f = **fi ; if(is_matching_rank(f, *meshIdx.bucket)) { e_size += field_bytes_per_entity( f , bucketId ); } } if (e_size == 0) { continue; } const EntityCommInfoVector& infovec = i->entity_comm->comm_map; PairIterEntityComm ec(infovec.begin(), infovec.end()); if ( owned ) { for ( ; ! ec.empty() ; ++ec ) { if (ec->ghost_id == ghost_id) { send_size[ ec->proc ] += e_size ; } } } else { for ( ; ! ec.empty() ; ++ec ) { if (ec->ghost_id == ghost_id) { recv_size[ i->owner ] += e_size ; break;//jump out since we know we're only recving 1 msg from the 1-and-only owner } } } } // Allocate send and receive buffers: CommAll sparse ; { const unsigned * const snd_size = send_size.data() ; const unsigned * const rcv_size = recv_size.data() ; sparse.allocate_buffers( mesh.parallel(), snd_size, rcv_size); } // Send packing: for (int phase = 0; phase < 2; ++phase) { for ( EntityCommListInfoVector::const_iterator i = mesh.internal_comm_list().begin(), iend = mesh.internal_comm_list().end() ; i != iend ; ++i ) { if ( (i->owner == parallel_rank && phase == 0) || (i->owner != parallel_rank && phase == 1) ) { Entity e = i->entity; const MeshIndex meshIdx = mesh.mesh_index(e); const unsigned bucketId = meshIdx.bucket->bucket_id(); for ( fi = fb ; fi != fe ; ++fi ) { const FieldBase & f = **fi ; if(!is_matching_rank(f, e)) continue; const unsigned size = field_bytes_per_entity( f , e ); if ( size ) { unsigned char * ptr = reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , bucketId, meshIdx.bucket_ordinal, size )); const EntityCommInfoVector& infovec = i->entity_comm->comm_map; PairIterEntityComm ec(infovec.begin(), infovec.end()); if (phase == 0) { // send for ( ; !ec.empty() ; ++ec ) { if (ec->ghost_id == ghost_id) { CommBuffer & b = sparse.send_buffer( ec->proc ); b.pack<unsigned char>( ptr , size ); } } } else { //recv for ( ; !ec.empty(); ++ec ) { if (ec->ghost_id == ghost_id) { CommBuffer & b = sparse.recv_buffer( i->owner ); b.unpack<unsigned char>( ptr , size ); break; } } } } } } } if (phase == 0) { sparse.communicate(); } } }