//----------------------------------------------------------------------
void communicate_field_data(
const Ghosting & ghosts ,
const std::vector< const FieldBase *> & fields )
{
if ( fields.empty() ) {
return;
}
const BulkData & mesh = ghosts.mesh();
const int parallel_size = mesh.parallel_size();
const int parallel_rank = mesh.parallel_rank();
const unsigned ghost_id = ghosts.ordinal();
const std::vector<const FieldBase *>::const_iterator fe = fields.end();
const std::vector<const FieldBase *>::const_iterator fb = fields.begin();
std::vector<const FieldBase *>::const_iterator fi ;
// Sizing for send and receive
const unsigned zero = 0 ;
std::vector<unsigned> send_size( parallel_size , zero );
std::vector<unsigned> recv_size( parallel_size , zero );
for ( EntityCommListInfoVector::const_iterator
i = mesh.internal_comm_list().begin() , iend = mesh.internal_comm_list().end(); i != iend ; ++i ) {
Entity e = i->entity;
const MeshIndex meshIdx = mesh.mesh_index(e);
const unsigned bucketId = meshIdx.bucket->bucket_id();
const bool owned = i->owner == parallel_rank ;
unsigned e_size = 0 ;
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
if(is_matching_rank(f, *meshIdx.bucket)) {
e_size += field_bytes_per_entity( f , bucketId );
}
}
if (e_size == 0) {
continue;
}
const EntityCommInfoVector& infovec = i->entity_comm->comm_map;
PairIterEntityComm ec(infovec.begin(), infovec.end());
if ( owned ) {
for ( ; ! ec.empty() ; ++ec ) {
if (ec->ghost_id == ghost_id) {
send_size[ ec->proc ] += e_size ;
}
}
}
else {
for ( ; ! ec.empty() ; ++ec ) {
if (ec->ghost_id == ghost_id) {
recv_size[ i->owner ] += e_size ;
break;//jump out since we know we're only recving 1 msg from the 1-and-only owner
}
}
}
}
// Allocate send and receive buffers:
CommAll sparse ;
{
const unsigned * const snd_size = send_size.data() ;
const unsigned * const rcv_size = recv_size.data() ;
sparse.allocate_buffers( mesh.parallel(), snd_size, rcv_size);
}
// Send packing:
for (int phase = 0; phase < 2; ++phase) {
for ( EntityCommListInfoVector::const_iterator i = mesh.internal_comm_list().begin(), iend = mesh.internal_comm_list().end() ; i != iend ; ++i ) {
if ( (i->owner == parallel_rank && phase == 0) || (i->owner != parallel_rank && phase == 1) ) {
Entity e = i->entity;
const MeshIndex meshIdx = mesh.mesh_index(e);
const unsigned bucketId = meshIdx.bucket->bucket_id();
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
if(!is_matching_rank(f, e)) continue;
const unsigned size = field_bytes_per_entity( f , e );
if ( size ) {
unsigned char * ptr =
reinterpret_cast<unsigned char *>(stk::mesh::field_data( f , bucketId, meshIdx.bucket_ordinal, size ));
const EntityCommInfoVector& infovec = i->entity_comm->comm_map;
PairIterEntityComm ec(infovec.begin(), infovec.end());
if (phase == 0) { // send
for ( ; !ec.empty() ; ++ec ) {
if (ec->ghost_id == ghost_id) {
CommBuffer & b = sparse.send_buffer( ec->proc );
b.pack<unsigned char>( ptr , size );
}
}
}
else { //recv
for ( ; !ec.empty(); ++ec ) {
if (ec->ghost_id == ghost_id) {
CommBuffer & b = sparse.recv_buffer( i->owner );
b.unpack<unsigned char>( ptr , size );
break;
}
}
}
}
}
}
}
if (phase == 0) {
sparse.communicate();
}
}
}
inline void parallel_sum_including_ghosts(
const BulkData & mesh ,
const std::vector< const FieldBase *> & fields )
{
if ( fields.empty() ) { return; }
const int parallel_size = mesh.parallel_size();
const int parallel_rank = mesh.parallel_rank();
const std::vector<const FieldBase *>::const_iterator fe = fields.end();
const std::vector<const FieldBase *>::const_iterator fb = fields.begin();
std::vector<const FieldBase *>::const_iterator fi ;
// Sizing for send and receive
const unsigned zero = 0 ;
std::vector<unsigned> send_size( parallel_size , zero );
std::vector<unsigned> recv_size( parallel_size , zero );
const EntityCommListInfoVector& comm_info_vec = mesh.internal_comm_list();
size_t comm_info_vec_size = comm_info_vec.size();
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
for (size_t i=0; i<comm_info_vec_size; ++i) {
if (!mesh.is_valid(comm_info_vec[i].entity))
{
ThrowAssertMsg(mesh.is_valid(comm_info_vec[i].entity),"parallel_sum_including_ghosts found invalid entity");
}
const Bucket* bucket = comm_info_vec[i].bucket;
unsigned e_size = 0 ;
if(is_matching_rank(f, *bucket)) {
const unsigned bucketId = bucket->bucket_id();
e_size += field_bytes_per_entity( f , bucketId );
}
if (e_size == 0) {
continue;
}
const bool owned = comm_info_vec[i].owner == parallel_rank ;
if ( !owned ) {
send_size[ comm_info_vec[i].owner ] += e_size ;
}
else {
const EntityCommInfoVector& infovec = comm_info_vec[i].entity_comm->comm_map;
size_t info_vec_size = infovec.size();
for (size_t j=0; j<info_vec_size; ++j ) {
recv_size[ infovec[j].proc ] += e_size ;
}
}
}
}
// Allocate send and receive buffers:
CommAll sparse ;
{
const unsigned * const snd_size = & send_size[0] ;
const unsigned * const rcv_size = & recv_size[0] ;
sparse.allocate_buffers( mesh.parallel(), snd_size, rcv_size);
}
// Send packing:
for (int phase = 0; phase < 2; ++phase) {
for ( fi = fb ; fi != fe ; ++fi ) {
const FieldBase & f = **fi ;
for (size_t i=0; i<comm_info_vec_size; ++i) {
const bool owned = comm_info_vec[i].owner == parallel_rank;
if ( (!owned && phase == 0) || (owned && phase == 1) )
{
const Bucket* bucket = comm_info_vec[i].bucket;
if(!is_matching_rank(f, *bucket)) continue;
const unsigned bucketId = bucket->bucket_id();
const size_t bucket_ordinal = comm_info_vec[i].bucket_ordinal;
const unsigned scalars_per_entity = field_scalars_per_entity(f, bucketId);
if ( scalars_per_entity > 0 ) {
int owner = comm_info_vec[i].owner;
if (f.data_traits().is_floating_point && f.data_traits().size_of == 8)
{
send_or_recv_field_data_for_assembly<double>(sparse, phase, f, owner, comm_info_vec[i].entity_comm->comm_map, scalars_per_entity, bucketId, bucket_ordinal);
}
else if (f.data_traits().is_floating_point && f.data_traits().size_of == 4)
{
send_or_recv_field_data_for_assembly<float>(sparse, phase, f, owner, comm_info_vec[i].entity_comm->comm_map, scalars_per_entity, bucketId, bucket_ordinal);
}
else if (f.data_traits().is_integral && f.data_traits().size_of == 4 && f.data_traits().is_unsigned)
{
send_or_recv_field_data_for_assembly<unsigned>(sparse, phase, f, owner, comm_info_vec[i].entity_comm->comm_map, scalars_per_entity, bucketId, bucket_ordinal);
}
else if (f.data_traits().is_integral && f.data_traits().size_of == 4 && f.data_traits().is_signed)
{
send_or_recv_field_data_for_assembly<int>(sparse, phase, f, owner, comm_info_vec[i].entity_comm->comm_map, scalars_per_entity, bucketId, bucket_ordinal);
}
else
{
ThrowRequireMsg(false,"Unsupported field type in parallel_sum_including_ghosts");
}
}
}
}
}
if (phase == 0) { sparse.communicate(); }
}
copy_from_owned(mesh, fields);
}