void comm_procs( const Ghosting & ghost ,
const Entity & entity , std::vector<unsigned> & procs )
{
procs.clear();
for ( PairIterEntityComm ec = entity.comm(); ! ec.empty() ; ++ec ) {
if ( ec->ghost_id == ghost.ordinal() ) {
procs.push_back( ec->proc );
}
}
}
bool in_ghost( const Ghosting & ghost , const Entity & entity , unsigned p )
{
// Ghost communication from owner.
EntityCommInfo tmp( ghost.ordinal() , p );
std::vector<EntityCommInfo>::const_iterator i =
std::lower_bound( entity.comm().begin() , entity.comm().end() , tmp );
return i != entity.comm().end() && tmp == *i ;
}
void communicate_field_data(
const Ghosting & ghosts ,
const std::vector< const FieldBase *> & fields )
{
if ( fields.empty() ) { return; }
const BulkData & mesh = BulkData::get(ghosts);
const unsigned parallel_size = mesh.parallel_size();
const unsigned parallel_rank = mesh.parallel_rank();
const std::vector<const FieldBase *>::const_iterator fe = fields.end();
const std::vector<const FieldBase *>::const_iterator fb = fields.begin();
std::vector<const FieldBase *>::const_iterator fi ;
// Sizing for send and receive
const unsigned zero = 0 ;
std::vector<unsigned> send_size( parallel_size , zero );
std::vector<unsigned> recv_size( parallel_size , zero );
for ( std::vector<Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
Entity & e = **i ;
const bool owned = e.owner_rank() == parallel_rank ;
unsigned e_size = 0 ;
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
e_size += field_data_size( f , e );
}
for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) {
if ( ghosts.ordinal() == ec->ghost_id ) {
if ( owned ) {
send_size[ ec->proc ] += e_size ;
}
else {
recv_size[ ec->proc ] += e_size ;
}
}
}
}
// Allocate send and receive buffers:
CommAll sparse ;
{
const unsigned * const s_size = & send_size[0] ;
const unsigned * const r_size = & recv_size[0] ;
sparse.allocate_buffers( mesh.parallel(), parallel_size / 4 , s_size, r_size);
}
// Send packing:
for ( std::vector<Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
Entity & e = **i ;
if ( e.owner_rank() == parallel_rank ) {
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
const unsigned size = field_data_size( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(field_data( f , e ));
for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) {
if ( ghosts.ordinal() == ec->ghost_id ) {
CommBuffer & b = sparse.send_buffer( ec->proc );
b.pack<unsigned char>( ptr , size );
}
}
}
}
}
}
// Communicate:
sparse.communicate();
// Unpack for recv:
for ( std::vector<Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
Entity & e = **i ;
if ( e.owner_rank() != parallel_rank ) {
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
const unsigned size = field_data_size( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(field_data( f , e ));
for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) {
if ( ghosts.ordinal() == ec->ghost_id ) {
CommBuffer & b = sparse.recv_buffer( ec->proc );
b.unpack<unsigned char>( ptr , size );
}
}
}
}
}
}
}
//----------------------------------------------------------------------
void communicate_field_data(
const Ghosting & ghosts ,
const std::vector< const FieldBase *> & fields )
{
if ( fields.empty() ) {
return;
}
const BulkData & mesh = ghosts.mesh();
const int parallel_size = mesh.parallel_size();
const int parallel_rank = mesh.parallel_rank();
const unsigned ghost_id = ghosts.ordinal();
const std::vector<const FieldBase *>::const_iterator fe = fields.end();
const std::vector<const FieldBase *>::const_iterator fb = fields.begin();
std::vector<const FieldBase *>::const_iterator fi ;
// Sizing for send and receive
const unsigned zero = 0 ;
std::vector<unsigned> send_size( parallel_size , zero );
std::vector<unsigned> recv_size( parallel_size , zero );
for ( EntityCommListInfoVector::const_iterator
i = mesh.internal_comm_list().begin() , iend = mesh.internal_comm_list().end(); i != iend ; ++i ) {
Entity e = i->entity;
const MeshIndex meshIdx = mesh.mesh_index(e);
const unsigned bucketId = meshIdx.bucket->bucket_id();
const bool owned = i->owner == parallel_rank ;
unsigned e_size = 0 ;
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
if(is_matching_rank(f, *meshIdx.bucket)) {
e_size += field_bytes_per_entity( f , bucketId );
}
}
if (e_size == 0) {
continue;
}
const EntityCommInfoVector& infovec = i->entity_comm->comm_map;
PairIterEntityComm ec(infovec.begin(), infovec.end());
if ( owned ) {
for ( ; ! ec.empty() ; ++ec ) {
if (ec->ghost_id == ghost_id) {
send_size[ ec->proc ] += e_size ;
}
}
}
else {
for ( ; ! ec.empty() ; ++ec ) {
if (ec->ghost_id == ghost_id) {
recv_size[ i->owner ] += e_size ;
break;//jump out since we know we're only recving 1 msg from the 1-and-only owner
}
}
}
}
// Allocate send and receive buffers:
CommAll sparse ;
{
const unsigned * const snd_size = send_size.data() ;
const unsigned * const rcv_size = recv_size.data() ;
sparse.allocate_buffers( mesh.parallel(), snd_size, rcv_size);
}
// Send packing:
for (int phase = 0; phase < 2; ++phase) {
for ( EntityCommListInfoVector::const_iterator i = mesh.internal_comm_list().begin(), iend = mesh.internal_comm_list().end() ; i != iend ; ++i ) {
if ( (i->owner == parallel_rank && phase == 0) || (i->owner != parallel_rank && phase == 1) ) {
Entity e = i->entity;
const MeshIndex meshIdx = mesh.mesh_index(e);
const unsigned bucketId = meshIdx.bucket->bucket_id();
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
if(!is_matching_rank(f, e)) continue;
const unsigned size = field_bytes_per_entity( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , bucketId, meshIdx.bucket_ordinal, size ));
const EntityCommInfoVector& infovec = i->entity_comm->comm_map;
PairIterEntityComm ec(infovec.begin(), infovec.end());
if (phase == 0) { // send
for ( ; !ec.empty() ; ++ec ) {
if (ec->ghost_id == ghost_id) {
CommBuffer & b = sparse.send_buffer( ec->proc );
b.pack<unsigned char>( ptr , size );
}
}
}
else { //recv
for ( ; !ec.empty(); ++ec ) {
if (ec->ghost_id == ghost_id) {
CommBuffer & b = sparse.recv_buffer( i->owner );
b.unpack<unsigned char>( ptr , size );
break;
}
}
}
}
}
}
}
if (phase == 0) {
sparse.communicate();
}
}
}
