void _ParticleShadowSync_FinishReceiveOfIncomingParticles( ParticleCommHandler* pCommHandler ) {
ParticleShadowSync *self = (ParticleShadowSync*)pCommHandler;
MPI_Status status;
Processor_Index proc_I;
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Neighbour_Index nbr_I;
int i = 0, j = 0;
int shadowParticleCounter;
int shadowCell = 0;
shadowParticleCounter = 0;
/* TODO: may be worth converting the below into an MPI_Test loop */
for ( nbr_I=0; nbr_I < procNbrInfo->procNbrCnt; nbr_I++ ) {
proc_I = procNbrInfo->procNbrTbl[nbr_I];
if( self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] > 0 ){
(void)MPI_Wait( self->particlesArrivingFromNbrShadowCellsHandles[nbr_I], &status );
}
}
for ( nbr_I=0; nbr_I < procNbrInfo->procNbrCnt; nbr_I++ ) {
for( i=0; i<cellShadowInfo->procShadowCnt[nbr_I]; i++ ){
shadowCell = cellShadowInfo->procShadowTbl[nbr_I][i];
for( j=0; j<self->particlesArrivingFromNbrShadowCellCounts[nbr_I][i]; j++ ){
Swarm_AddShadowParticleToShadowCell( self->swarm, shadowCell, shadowParticleCounter );
shadowParticleCounter++;
}
}
}
}
void _ParticleShadowSync_FinishReceiveOfIncomingParticleCounts( ParticleCommHandler* self ) {
MPI_Status status;
Processor_Index proc_I;
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Neighbour_Index nbr_I;
int i = 0;
self->swarm->shadowParticleCount = 0;
/* TODO: may be worth converting the below into an MPI_Test loop */
for ( nbr_I=0; nbr_I < procNbrInfo->procNbrCnt; nbr_I++ ) {
proc_I = procNbrInfo->procNbrTbl[nbr_I];
(void)MPI_Wait( self->particlesArrivingFromNbrShadowCellCountsHandles[nbr_I], &status );
self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] = 0;
for( i=0; i<cellShadowInfo->procShadowCnt[nbr_I]; i++ ){
Index shadowCell = 0;
shadowCell = cellShadowInfo->procShadowTbl[nbr_I][i];
self->swarm->shadowCellParticleCountTbl[shadowCell-self->swarm->cellLocalCount] = 0;
self->swarm->shadowCellParticleTbl[shadowCell] = Memory_Realloc_Array( self->swarm->shadowCellParticleTbl[shadowCell], Particle_Index,
(self->particlesArrivingFromNbrShadowCellCounts[nbr_I][i]) );
self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] += self->particlesArrivingFromNbrShadowCellCounts[nbr_I][i];
}
self->swarm->shadowParticleCount+= self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I];
}
}
void _ParticleShadowSync_SendParticleTotalsInShadowCellsToNbrs( ParticleCommHandler* self )
{
Processor_Index proc_I;
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Neighbour_Index nbr_I, nbrCount;
Cell_Index lCellID, cellParticleCount, shadowedCell_I;
Journal_DPrintfL( self->debug, 1, "In %s():\n", __func__ );
Stream_IndentBranch( Swarm_Debug );
self->shadowParticlesLeavingMeTotalCount = 0;
nbrCount = procNbrInfo->procNbrCnt;
for( nbr_I = 0 ; nbr_I < nbrCount ; nbr_I++ ) {
proc_I = procNbrInfo->procNbrTbl[nbr_I];
self->shadowParticlesLeavingMeTotalCounts[nbr_I] = 0;
for( shadowedCell_I = 0 ; shadowedCell_I < cellShadowInfo->procShadowedCnt[ nbr_I ]; shadowedCell_I++ ) {
lCellID = cellShadowInfo->procShadowedTbl[nbr_I][shadowedCell_I];
cellParticleCount = self->swarm->cellParticleCountTbl[ lCellID ];
self->shadowParticlesLeavingMeCountsPerCell[nbr_I][shadowedCell_I] = cellParticleCount;
self->shadowParticlesLeavingMeTotalCounts[nbr_I] += cellParticleCount;
}
MPI_Ssend( self->shadowParticlesLeavingMeCountsPerCell[nbr_I], cellShadowInfo->procShadowCnt[nbr_I], MPI_UNSIGNED,
proc_I, SHADOW_PARTICLE_COUNTS_PER_CELL, self->swarm->comm );
}
Stream_UnIndentBranch( Swarm_Debug );
}
void _ParticleShadowSync_SendShadowParticles( ParticleCommHandler *self )
{
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Processor_Index proc_I;
int i = 0, j = 0, k = 0, cell = 0;
unsigned int arrayIndex = 0;
long arraySize = 0;
unsigned int pIndex = 0;
self->shadowParticlesLeavingMeHandles = Memory_Alloc_Array_Unnamed( MPI_Request*, procNbrInfo->procNbrCnt );
self->shadowParticlesLeavingMe = Memory_Alloc_Array_Unnamed( Particle*, procNbrInfo->procNbrCnt );
for( i=0; i<procNbrInfo->procNbrCnt; i++ ){
proc_I = procNbrInfo->procNbrTbl[i];
if( self->shadowParticlesLeavingMeTotalCounts[i] != 0 ){
self->shadowParticlesLeavingMeHandles[i] = Memory_Alloc_Array_Unnamed( MPI_Request, 1 );
arraySize = self->swarm->particleExtensionMgr->finalSize * self->shadowParticlesLeavingMeTotalCounts[i];
self->shadowParticlesLeavingMe[i] = Memory_Alloc_Bytes( arraySize, "Particle", "pCommHandler->outgoingPArray" );
memset( self->shadowParticlesLeavingMe[i], 0, arraySize );
arrayIndex = 0;
for( j=0; j<cellShadowInfo->procShadowedCnt[i]; j++ ){
cell = cellShadowInfo->procShadowedTbl[i][j];
for( k=0; k<self->swarm->cellParticleCountTbl[cell]; k++ ){
pIndex = self->swarm->cellParticleTbl[cell][k];
Swarm_CopyParticleOffSwarm( self->swarm,
self->shadowParticlesLeavingMe[i], arrayIndex++,
pIndex );
}
}
/*printf( "sending %ld bytes\n", self->shadowParticlesLeavingMeTotalCounts[i] * self->swarm->particleExtensionMgr->finalSize );*/
MPI_Issend( self->shadowParticlesLeavingMe[i],
self->shadowParticlesLeavingMeTotalCounts[i] * self->swarm->particleExtensionMgr->finalSize,
MPI_BYTE, proc_I, SHADOW_PARTICLES, self->swarm->comm,
self->shadowParticlesLeavingMeHandles[i] );
}
}
}
void _ParticleShadowSync_BeginReceiveOfIncomingParticles( ParticleCommHandler* pCommHandler ) {
ParticleShadowSync *self = (ParticleShadowSync*)pCommHandler;
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
long incomingViaShadowArrayBytes = 0;
Neighbour_Index nbr_I;
Processor_Index proc_I;
int i = 0;
char* recvLocation = NULL;
self->particlesArrivingFromNbrShadowCellsHandles = Memory_Alloc_Array_Unnamed( MPI_Request*, procNbrInfo->procNbrCnt );
for( i=0; i<procNbrInfo->procNbrCnt; i++ ){
self->particlesArrivingFromNbrShadowCellsHandles[i] = Memory_Alloc_Array_Unnamed( MPI_Request, 1 );
}
self->swarm->shadowParticles = Memory_Realloc( self->swarm->shadowParticles,
self->swarm->particleExtensionMgr->finalSize*(self->swarm->shadowParticleCount) );
recvLocation = (char*)self->swarm->shadowParticles;
for ( nbr_I=0; nbr_I < procNbrInfo->procNbrCnt; nbr_I++ ) {
if ( self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] != 0 ) {
proc_I = procNbrInfo->procNbrTbl[nbr_I];
/* start non-blocking recv of particles */
incomingViaShadowArrayBytes = self->swarm->particleExtensionMgr->finalSize *
self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I];
/*printf( "receiving %ld bytes\n", incomingViaShadowArrayBytes );*/
(void)MPI_Irecv( recvLocation, incomingViaShadowArrayBytes, MPI_BYTE,
proc_I, SHADOW_PARTICLES, self->swarm->comm,
self->particlesArrivingFromNbrShadowCellsHandles[nbr_I] );
recvLocation += incomingViaShadowArrayBytes;
}
}
}
/* TODO: look at using MPI_Indexed instead */
void ParticleMovementHandler_FinishReceiveAndUpdateShadowParticlesEnteringMyDomain( ParticleCommHandler* self ) {
MPI_Status status;
Cell_ShadowTransferIndex stCell_I;
Cell_LocalIndex lCell_I;
Neighbour_Index nbr_I;
Cell_ShadowTransferIndex shadowCellsFromProcCount;
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Neighbour_Index nbrCount = procNbrInfo->procNbrCnt;
Particle_InCellIndex incomingCellParticleCount;
Particle_InCellIndex cParticle_I;
Particle_Index lParticle_I;
Index incomingParticle_I=0; /*Index into the array of all leaving particle indices */
Index incomingParticleSetsNotYetReceivedCount;
Bool* incomingParticlesReceived;
#if DEBUG
GlobalParticle* currParticle;
#endif
Journal_DPrintf( self->debug, "In %s():\n", __func__ );
Stream_IndentBranch( Swarm_Debug );
incomingParticlesReceived = Memory_Alloc_Array_Unnamed( Bool, nbrCount );
/* Calculate how many particle sets we have to receive */
incomingParticleSetsNotYetReceivedCount = 0;
for ( nbr_I=0; nbr_I < nbrCount; nbr_I++ ) {
incomingParticlesReceived[nbr_I] = False;
if (self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] > 0) {
incomingParticleSetsNotYetReceivedCount++;
}
}
while ( incomingParticleSetsNotYetReceivedCount > 0 ) {
int flag = 0;
Journal_DPrintfL( self->debug, 3, "%d particle sets still to go...\n", incomingParticleSetsNotYetReceivedCount );
for ( nbr_I=0; nbr_I < nbrCount; nbr_I++ ) {
if ( (self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] > 0) &&
(False == incomingParticlesReceived[nbr_I]) )
{
MPI_Test( self->particlesArrivingFromNbrShadowCellsHandles[nbr_I], &flag, &status );
if ( False == flag ) {
/* No results yet from this proc -> continue to next. */
continue;
}
else {
Journal_DPrintfL( self->debug, 3, "Received particles from nbr %d (proc %d):\n",
nbr_I, procNbrInfo->procNbrTbl[nbr_I] );
Stream_Indent( self->debug );
incomingParticle_I = 0;
shadowCellsFromProcCount = cellShadowInfo->procShadowedCnt[nbr_I];
for ( stCell_I=0; stCell_I < shadowCellsFromProcCount; stCell_I++ ) {
lCell_I = cellShadowInfo->procShadowedTbl[nbr_I][stCell_I];
Journal_DPrintfL( self->debug, 3, "Incoming cell %d (local index %d):\n",
stCell_I, lCell_I );
Stream_Indent( self->debug );
incomingCellParticleCount =
self->particlesArrivingFromNbrShadowCellCounts[nbr_I][stCell_I];
for ( cParticle_I=0; cParticle_I < incomingCellParticleCount; cParticle_I++ ) {
#if DEBUG
currParticle = (GlobalParticle*)ParticleAt(
self->particlesArrivingFromNbrShadowCells[nbr_I],
incomingParticle_I,
self->swarm->particleExtensionMgr->finalSize );
Journal_DPrintfL( self->debug, 3, "Handling its PIC %d: - at "
"(%.2f,%.2f,%.2f)\n", cParticle_I,
currParticle->coord[0], currParticle->coord[1],
currParticle->coord[2] );
#endif
Stream_Indent( self->debug );
lParticle_I = ParticleMovementHandler_FindFreeSlotAndPrepareForInsertion( self );
Swarm_CopyParticleOntoSwarm(
self->swarm,
lParticle_I,
self->particlesArrivingFromNbrShadowCells[nbr_I], incomingParticle_I++ );
Swarm_AddParticleToCell( self->swarm, lCell_I, lParticle_I );
Stream_UnIndent( self->debug );
}
Stream_UnIndent( self->debug );
}
incomingParticlesReceived[nbr_I] = True;
incomingParticleSetsNotYetReceivedCount--;
Stream_UnIndent( self->debug );
}
}
}
}
Memory_Free( incomingParticlesReceived );
Stream_UnIndentBranch( Swarm_Debug );
}
