bool in_send_ghost( const Entity & entity )
{
// Ghost communication with non-owner.
PairIterEntityComm ec = entity.comm();
return ! ec.empty() && ec.back().ghost_id != 0 &&
ec.back().proc != entity.owner_rank();
}
static void print_comm_list( const BulkData & mesh , bool doit )
{
if ( doit ) {
std::ostringstream msg ;
msg << std::endl ;
for ( std::vector<Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
Entity & entity = **i ;
msg << "P" << mesh.parallel_rank() << ": " ;
print_entity_key( msg , MetaData::get(mesh) , entity.key() );
msg << " owner(" << entity.owner_rank() << ")" ;
if ( EntityLogModified == entity.log_query() ) { msg << " mod" ; }
else if ( EntityLogDeleted == entity.log_query() ) { msg << " del" ; }
else { msg << " " ; }
for ( PairIterEntityComm ec = mesh.entity_comm(entity.key()); ! ec.empty() ; ++ec ) {
msg << " gid, proc (" << ec->ghost_id << "," << ec->proc << ")" ;
}
msg << std::endl ;
}
std::cout << msg.str();
}
}
bool in_receive_ghost( const Entity & entity )
{
// Ghost communication with owner.
PairIterEntityComm ec = entity.comm();
return ! ec.empty() && ec.front().ghost_id != 0 &&
ec.front().proc == entity.owner_rank();
}
void printEntity(std::ostringstream& msg, Entity *entity)
{
msg << " :: " << print_entity_key(entity) << ":o[" << entity->owner_rank() << "]:l[" << entity->log_query()
<< "]:ec[";
for ( PairIterEntityComm ec = entity->comm() ; ! ec.empty() ; ++ec ) {
msg << "(" << ec->ghost_id << "," << ec->proc << ")";
}
msg << "]";
}
bool in_shared( const Entity & entity , unsigned proc )
{
for ( PairIterEntityComm ec = entity.comm();
! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
if ( proc == ec->proc ) {
return true ;
}
}
return false ;
}
void comm_procs( const Ghosting & ghost ,
const Entity & entity , std::vector<unsigned> & procs )
{
procs.clear();
for ( PairIterEntityComm ec = entity.comm(); ! ec.empty() ; ++ec ) {
if ( ec->ghost_id == ghost.ordinal() ) {
procs.push_back( ec->proc );
}
}
}
bool in_send_ghost( const Entity & entity , unsigned proc )
{
for ( PairIterEntityComm ec = entity.comm(); ! ec.empty() ; ++ec ) {
if ( ec->ghost_id != 0 &&
ec->proc != entity.owner_rank() &&
ec->proc == proc ) {
return true ;
}
}
return false ;
}
void comm_procs( const Entity & entity , std::vector<unsigned> & procs )
{
procs.clear();
for ( PairIterEntityComm ec = entity.comm(); ! ec.empty() ; ++ec ) {
procs.push_back( ec->proc );
}
std::sort( procs.begin() , procs.end() );
std::vector<unsigned>::iterator
i = std::unique( procs.begin() , procs.end() );
procs.erase( i , procs.end() );
}
std::ostream& Ghosting::operator<<(std::ostream& out) const
{
out << "Ghosting object: name: " << name()
<< ", ordinal: " << ordinal() << "\n";
out << " Locally owned entities ghosted on other processors (send list):\n";
for ( EntityCommListInfoVector::const_iterator
i = m_mesh.internal_comm_list().begin() ;
i != m_mesh.internal_comm_list().end() ; ++i ){
if ( i->owner == m_mesh.parallel_rank() ) {
for ( PairIterEntityComm ec = m_mesh.internal_entity_comm_map(i->key) ; ! ec.empty() ; ++ec ) {
if ( ec->ghost_id == m_ordinal ) {
out << " ";
out << i->key.id();
out << ", sending ghost to " << ec->proc << ", status is: "
<< m_mesh.state(i->entity) << "\n";
}
}
}
}
out << " Entities ghosted on this processor from the owner (recv list):\n";
for ( EntityCommListInfoVector::const_iterator
i = m_mesh.internal_comm_list().begin() ;
i != m_mesh.internal_comm_list().end() ; ++i ) {
if ( i->owner != m_mesh.parallel_rank() ) {
for ( PairIterEntityComm ec = m_mesh.internal_entity_comm_map(i->key); !ec.empty(); ++ec ) {
if ( ec->ghost_id == m_ordinal ) {
out << " ";
out << i->key.id();
out << ", owner of ghost is " << i->owner
<< ", status is: " << m_mesh.state(i->entity) << "\n";
}
}
}
}
return out;
}
void Ghosting::receive_list( std::vector<EntityKey> & v ) const
{
for ( EntityCommListInfoVector::const_iterator
i = m_mesh.internal_comm_list().begin() ;
i != m_mesh.internal_comm_list().end() ; ++i ){
if ( i->owner != m_mesh.parallel_rank() ) {
for ( PairIterEntityComm ec = m_mesh.internal_entity_comm_map(i->key) ; ! ec.empty() ; ++ec ) {
if ( ec->ghost_id == m_ordinal ) {
v.push_back(i->key);
}
}
}
}
}
void communicate_field_data(
const BulkData & mesh ,
const unsigned field_count ,
const FieldBase * fields[] ,
CommAll & sparse )
{
const std::vector<Entity*> & entity_comm = mesh.entity_comm();
const unsigned parallel_size = mesh.parallel_size();
// Sizing for send and receive
const unsigned zero = 0 ;
std::vector<unsigned> msg_size( parallel_size , zero );
size_t j = 0;
for ( j = 0 ; j < field_count ; ++j ) {
const FieldBase & f = * fields[j] ;
for ( std::vector<Entity*>::const_iterator
i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
Entity & e = **i ;
const unsigned size = field_data_size( f , e );
if ( size ) {
for ( PairIterEntityComm
ec = e.comm() ; ! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
msg_size[ ec->proc ] += size ;
}
}
}
}
// Allocate send and receive buffers:
{
const unsigned * const s_size = & msg_size[0] ;
sparse.allocate_buffers( mesh.parallel(), parallel_size / 4 , s_size, s_size);
}
// Pack for send:
for ( j = 0 ; j < field_count ; ++j ) {
const FieldBase & f = * fields[j] ;
for ( std::vector<Entity*>::const_iterator
i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
Entity & e = **i ;
const unsigned size = field_data_size( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(field_data( f , e ));
for ( PairIterEntityComm
ec = e.comm() ; ! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
CommBuffer & b = sparse.send_buffer( ec->proc );
b.pack<unsigned char>( ptr , size );
}
}
}
}
// Communicate:
sparse.communicate();
}
void communicate_field_data(
const Ghosting & ghosts ,
const std::vector< const FieldBase *> & fields )
{
if ( fields.empty() ) { return; }
const BulkData & mesh = BulkData::get(ghosts);
const unsigned parallel_size = mesh.parallel_size();
const unsigned parallel_rank = mesh.parallel_rank();
const std::vector<const FieldBase *>::const_iterator fe = fields.end();
const std::vector<const FieldBase *>::const_iterator fb = fields.begin();
std::vector<const FieldBase *>::const_iterator fi ;
// Sizing for send and receive
const unsigned zero = 0 ;
std::vector<unsigned> send_size( parallel_size , zero );
std::vector<unsigned> recv_size( parallel_size , zero );
for ( std::vector<Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
Entity & e = **i ;
const bool owned = e.owner_rank() == parallel_rank ;
unsigned e_size = 0 ;
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
e_size += field_data_size( f , e );
}
for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) {
if ( ghosts.ordinal() == ec->ghost_id ) {
if ( owned ) {
send_size[ ec->proc ] += e_size ;
}
else {
recv_size[ ec->proc ] += e_size ;
}
}
}
}
// Allocate send and receive buffers:
CommAll sparse ;
{
const unsigned * const s_size = & send_size[0] ;
const unsigned * const r_size = & recv_size[0] ;
sparse.allocate_buffers( mesh.parallel(), parallel_size / 4 , s_size, r_size);
}
// Send packing:
for ( std::vector<Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
Entity & e = **i ;
if ( e.owner_rank() == parallel_rank ) {
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
const unsigned size = field_data_size( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(field_data( f , e ));
for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) {
if ( ghosts.ordinal() == ec->ghost_id ) {
CommBuffer & b = sparse.send_buffer( ec->proc );
b.pack<unsigned char>( ptr , size );
}
}
}
}
}
}
// Communicate:
sparse.communicate();
// Unpack for recv:
for ( std::vector<Entity*>::const_iterator
i = mesh.entity_comm().begin() ;
i != mesh.entity_comm().end() ; ++i ) {
Entity & e = **i ;
if ( e.owner_rank() != parallel_rank ) {
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
const unsigned size = field_data_size( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(field_data( f , e ));
for ( PairIterEntityComm ec = e.comm() ; ! ec.empty() ; ++ec ) {
if ( ghosts.ordinal() == ec->ghost_id ) {
CommBuffer & b = sparse.recv_buffer( ec->proc );
b.unpack<unsigned char>( ptr , size );
}
}
}
}
}
}
}
bool in_shared( const Entity & entity )
{
PairIterEntityComm ec = entity.comm();
return ! ec.empty() && ec.front().ghost_id == 0 ;
}