void ElementCellLayout_BuildShadowInfo( ElementCellLayout* self ) {
unsigned nDims;
Comm* comm;
int nIncProcs;
int* incProcs;
unsigned n_i;
nDims = Mesh_GetDimSize( self->mesh );
comm = Mesh_GetCommTopology( self->mesh, nDims );
Comm_GetNeighbours( comm, &nIncProcs, &incProcs );
/* Extract neighbouring proc information. */
self->cellShadowInfo.procNbrInfo = Memory_Alloc_Unnamed( ProcNbrInfo );
self->cellShadowInfo.procNbrInfo->procNbrCnt = nIncProcs;
self->cellShadowInfo.procNbrInfo->procNbrTbl = AllocArray( unsigned, nIncProcs );
memcpy( self->cellShadowInfo.procNbrInfo->procNbrTbl, incProcs, nIncProcs * sizeof(unsigned) );
/* Count shadow info. */
if( nIncProcs ) {
self->cellShadowInfo.procShadowedCnt = AllocArray( unsigned, nIncProcs );
memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) );
self->cellShadowInfo.procShadowCnt = AllocArray( unsigned, nIncProcs );
memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) );
}
for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) {
unsigned nSharers;
unsigned* sharers;
unsigned s_i;
Mesh_GetSharers( self->mesh, nDims, n_i,
&nSharers, &sharers );
for( s_i = 0; s_i < nSharers; s_i++ )
self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++;
}
for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) {
unsigned owner;
owner = Mesh_GetOwner( self->mesh, nDims, n_i );
self->cellShadowInfo.procShadowCnt[owner]++;
}
/* Build shadow info indices. */
if( nIncProcs ) {
self->cellShadowInfo.procShadowedTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs,
self->cellShadowInfo.procShadowedCnt );
self->cellShadowInfo.procShadowTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs,
self->cellShadowInfo.procShadowCnt );
memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) );
memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) );
}
for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) {
unsigned local;
unsigned curInd;
unsigned nSharers;
unsigned* sharers;
unsigned s_i;
local = Mesh_SharedToLocal( self->mesh, nDims, n_i );
Mesh_GetSharers( self->mesh, nDims, n_i,
&nSharers, &sharers );
for( s_i = 0; s_i < nSharers; s_i++ ) {
curInd = self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++;
self->cellShadowInfo.procShadowedTbl[sharers[s_i]][curInd] = local;
}
}
for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) {
unsigned domain;
unsigned curInd;
unsigned owner;
domain = Mesh_GetLocalSize( self->mesh, nDims ) + n_i;
owner = Mesh_GetOwner( self->mesh, nDims, n_i );
curInd = self->cellShadowInfo.procShadowCnt[owner]++;
self->cellShadowInfo.procShadowTbl[owner][curInd] = domain;
}
}
void IGraph_SetElements( void* _self, int dim, int nEls, const int* globals ) {
IGraph* self = (IGraph*)_self;
int rank;
int nNbrs;
const int *nbrs;
int nSubEls;
const int *subEls;
int rem, netRem;
int *nNbrEls, **nbrEls;
IArray** isects;
ISet localsObj, *locals = &localsObj;
ISet remotesObj, *remotes = &remotesObj;
int nCurEls, *curEls;
MPI_Comm mpiComm;
int n_i, e_i;
assert( self && dim < self->nTDims );
assert( !nEls || globals );
assert( self->comm );
Comm_GetNeighbours( self->comm, &nNbrs, &nbrs );
if( !nNbrs ) {
IGraph_SetLocalElements( self, dim, nEls, globals );
return;
}
ISet_Init( locals );
mpiComm = Comm_GetMPIComm( self->comm );
insist( MPI_Comm_rank( mpiComm, &rank ), == MPI_SUCCESS );
isects = AllocArray( IArray*, nNbrs );
for( n_i = 0; n_i < nNbrs; n_i++ )
isects[n_i] = IArray_New();
ISet_UseArray( locals, nEls, globals );
nSubEls = (nEls < 1000) ? nEls : 1000;
rem = nEls;
subEls = globals;
nNbrEls = AllocArray( int, nNbrs );
nbrEls = AllocArray( int*, nNbrs );
do {
Comm_AllgatherInit( self->comm, nSubEls, nNbrEls, sizeof(int) );
for( n_i = 0; n_i < nNbrs; n_i++ )
nbrEls[n_i] = AllocArray( int, nNbrEls[n_i] );
Comm_AllgatherBegin( self->comm, subEls, (void**)nbrEls );
Comm_AllgatherEnd( self->comm );
for( n_i = 0; n_i < nNbrs; n_i++ ) {
for( e_i = 0; e_i < nNbrEls[n_i]; e_i++ ) {
if( ISet_Has( locals, nbrEls[n_i][e_i] ) )
IArray_Append( isects[n_i], nbrEls[n_i][e_i] );
}
FreeArray( nbrEls[n_i] );
}
subEls += nSubEls;
rem -= nSubEls;
nSubEls = (rem < 1000) ? rem : 1000;
insist( MPI_Allreduce( &rem, &netRem, 1, MPI_INT, MPI_SUM, mpiComm ), == MPI_SUCCESS );
} while( netRem );
FreeArray( nNbrEls );
FreeArray( nbrEls );
ISet_Init( remotes );
ISet_SetMaxSize( remotes, nEls );
for( n_i = 0; n_i < nNbrs; n_i++ ) {
IArray_GetArray( isects[n_i], &nCurEls, (const int**)&curEls );
if( nbrs[n_i] < rank ) {
for( e_i = 0; e_i < nCurEls; e_i++ ) {
ISet_TryRemove( locals, curEls[e_i] );
ISet_TryInsert( remotes, curEls[e_i] );
}
}
Stg_Class_Delete( isects[n_i] );
}
FreeArray( isects );
nCurEls = ISet_GetSize( locals );
curEls = AllocArray( int, nCurEls );
ISet_GetArray( locals, curEls );
ISet_Destruct( locals );
qsort( curEls, nCurEls, sizeof(int), IGraph_Cmp );
IGraph_SetLocalElements( self, dim, nCurEls, curEls );
FreeArray( curEls );
nCurEls = ISet_GetSize( remotes );
curEls = AllocArray( int, nCurEls );
ISet_GetArray( remotes, curEls );
ISet_Destruct( remotes );
qsort( curEls, nCurEls, sizeof(int), IGraph_Cmp );
IGraph_SetRemoteElements( self, dim, nCurEls, curEls );
FreeArray( curEls );
}
