void ElementCellLayout_BuildShadowInfo( ElementCellLayout* self ) { unsigned nDims; Comm* comm; int nIncProcs; int* incProcs; unsigned n_i; nDims = Mesh_GetDimSize( self->mesh ); comm = Mesh_GetCommTopology( self->mesh, nDims ); Comm_GetNeighbours( comm, &nIncProcs, &incProcs ); /* Extract neighbouring proc information. */ self->cellShadowInfo.procNbrInfo = Memory_Alloc_Unnamed( ProcNbrInfo ); self->cellShadowInfo.procNbrInfo->procNbrCnt = nIncProcs; self->cellShadowInfo.procNbrInfo->procNbrTbl = AllocArray( unsigned, nIncProcs ); memcpy( self->cellShadowInfo.procNbrInfo->procNbrTbl, incProcs, nIncProcs * sizeof(unsigned) ); /* Count shadow info. */ if( nIncProcs ) { self->cellShadowInfo.procShadowedCnt = AllocArray( unsigned, nIncProcs ); memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) ); self->cellShadowInfo.procShadowCnt = AllocArray( unsigned, nIncProcs ); memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) ); } for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) { unsigned nSharers; unsigned* sharers; unsigned s_i; Mesh_GetSharers( self->mesh, nDims, n_i, &nSharers, &sharers ); for( s_i = 0; s_i < nSharers; s_i++ ) self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++; } for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) { unsigned owner; owner = Mesh_GetOwner( self->mesh, nDims, n_i ); self->cellShadowInfo.procShadowCnt[owner]++; } /* Build shadow info indices. */ if( nIncProcs ) { self->cellShadowInfo.procShadowedTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs, self->cellShadowInfo.procShadowedCnt ); self->cellShadowInfo.procShadowTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs, self->cellShadowInfo.procShadowCnt ); memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) ); memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) ); } for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) { unsigned local; unsigned curInd; unsigned nSharers; unsigned* sharers; unsigned s_i; local = Mesh_SharedToLocal( self->mesh, nDims, n_i ); Mesh_GetSharers( self->mesh, nDims, n_i, &nSharers, &sharers ); for( s_i = 0; s_i < nSharers; s_i++ ) { curInd = self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++; self->cellShadowInfo.procShadowedTbl[sharers[s_i]][curInd] = local; } } for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) { unsigned domain; unsigned curInd; unsigned owner; domain = Mesh_GetLocalSize( self->mesh, nDims ) + n_i; owner = Mesh_GetOwner( self->mesh, nDims, n_i ); curInd = self->cellShadowInfo.procShadowCnt[owner]++; self->cellShadowInfo.procShadowTbl[owner][curInd] = domain; } }
void IGraph_SetElements( void* _self, int dim, int nEls, const int* globals ) { IGraph* self = (IGraph*)_self; int rank; int nNbrs; const int *nbrs; int nSubEls; const int *subEls; int rem, netRem; int *nNbrEls, **nbrEls; IArray** isects; ISet localsObj, *locals = &localsObj; ISet remotesObj, *remotes = &remotesObj; int nCurEls, *curEls; MPI_Comm mpiComm; int n_i, e_i; assert( self && dim < self->nTDims ); assert( !nEls || globals ); assert( self->comm ); Comm_GetNeighbours( self->comm, &nNbrs, &nbrs ); if( !nNbrs ) { IGraph_SetLocalElements( self, dim, nEls, globals ); return; } ISet_Init( locals ); mpiComm = Comm_GetMPIComm( self->comm ); insist( MPI_Comm_rank( mpiComm, &rank ), == MPI_SUCCESS ); isects = AllocArray( IArray*, nNbrs ); for( n_i = 0; n_i < nNbrs; n_i++ ) isects[n_i] = IArray_New(); ISet_UseArray( locals, nEls, globals ); nSubEls = (nEls < 1000) ? nEls : 1000; rem = nEls; subEls = globals; nNbrEls = AllocArray( int, nNbrs ); nbrEls = AllocArray( int*, nNbrs ); do { Comm_AllgatherInit( self->comm, nSubEls, nNbrEls, sizeof(int) ); for( n_i = 0; n_i < nNbrs; n_i++ ) nbrEls[n_i] = AllocArray( int, nNbrEls[n_i] ); Comm_AllgatherBegin( self->comm, subEls, (void**)nbrEls ); Comm_AllgatherEnd( self->comm ); for( n_i = 0; n_i < nNbrs; n_i++ ) { for( e_i = 0; e_i < nNbrEls[n_i]; e_i++ ) { if( ISet_Has( locals, nbrEls[n_i][e_i] ) ) IArray_Append( isects[n_i], nbrEls[n_i][e_i] ); } FreeArray( nbrEls[n_i] ); } subEls += nSubEls; rem -= nSubEls; nSubEls = (rem < 1000) ? rem : 1000; insist( MPI_Allreduce( &rem, &netRem, 1, MPI_INT, MPI_SUM, mpiComm ), == MPI_SUCCESS ); } while( netRem ); FreeArray( nNbrEls ); FreeArray( nbrEls ); ISet_Init( remotes ); ISet_SetMaxSize( remotes, nEls ); for( n_i = 0; n_i < nNbrs; n_i++ ) { IArray_GetArray( isects[n_i], &nCurEls, (const int**)&curEls ); if( nbrs[n_i] < rank ) { for( e_i = 0; e_i < nCurEls; e_i++ ) { ISet_TryRemove( locals, curEls[e_i] ); ISet_TryInsert( remotes, curEls[e_i] ); } } Stg_Class_Delete( isects[n_i] ); } FreeArray( isects ); nCurEls = ISet_GetSize( locals ); curEls = AllocArray( int, nCurEls ); ISet_GetArray( locals, curEls ); ISet_Destruct( locals ); qsort( curEls, nCurEls, sizeof(int), IGraph_Cmp ); IGraph_SetLocalElements( self, dim, nCurEls, curEls ); FreeArray( curEls ); nCurEls = ISet_GetSize( remotes ); curEls = AllocArray( int, nCurEls ); ISet_GetArray( remotes, curEls ); ISet_Destruct( remotes ); qsort( curEls, nCurEls, sizeof(int), IGraph_Cmp ); IGraph_SetRemoteElements( self, dim, nCurEls, curEls ); FreeArray( curEls ); }