void _ParticleShadowSync_SendShadowParticles( ParticleCommHandler *self )
{
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Processor_Index proc_I;
int i = 0, j = 0, k = 0, cell = 0;
unsigned int arrayIndex = 0;
long arraySize = 0;
unsigned int pIndex = 0;
self->shadowParticlesLeavingMeHandles = Memory_Alloc_Array_Unnamed( MPI_Request*, procNbrInfo->procNbrCnt );
self->shadowParticlesLeavingMe = Memory_Alloc_Array_Unnamed( Particle*, procNbrInfo->procNbrCnt );
for( i=0; i<procNbrInfo->procNbrCnt; i++ ){
proc_I = procNbrInfo->procNbrTbl[i];
if( self->shadowParticlesLeavingMeTotalCounts[i] != 0 ){
self->shadowParticlesLeavingMeHandles[i] = Memory_Alloc_Array_Unnamed( MPI_Request, 1 );
arraySize = self->swarm->particleExtensionMgr->finalSize * self->shadowParticlesLeavingMeTotalCounts[i];
self->shadowParticlesLeavingMe[i] = Memory_Alloc_Bytes( arraySize, "Particle", "pCommHandler->outgoingPArray" );
memset( self->shadowParticlesLeavingMe[i], 0, arraySize );
arrayIndex = 0;
for( j=0; j<cellShadowInfo->procShadowedCnt[i]; j++ ){
cell = cellShadowInfo->procShadowedTbl[i][j];
for( k=0; k<self->swarm->cellParticleCountTbl[cell]; k++ ){
pIndex = self->swarm->cellParticleTbl[cell][k];
Swarm_CopyParticleOffSwarm( self->swarm,
self->shadowParticlesLeavingMe[i], arrayIndex++,
pIndex );
}
}
/*printf( "sending %ld bytes\n", self->shadowParticlesLeavingMeTotalCounts[i] * self->swarm->particleExtensionMgr->finalSize );*/
MPI_Issend( self->shadowParticlesLeavingMe[i],
self->shadowParticlesLeavingMeTotalCounts[i] * self->swarm->particleExtensionMgr->finalSize,
MPI_BYTE, proc_I, SHADOW_PARTICLES, self->swarm->comm,
self->shadowParticlesLeavingMeHandles[i] );
}
}
}
void ParticleMovementHandler_ShareAndUpdateParticlesThatHaveMovedOutsideDomains(
ParticleMovementHandler* self,
Particle_Index* globalParticlesArrivingMyDomainCountPtr,
Particle_Index* globalParticlesOutsideDomainTotalPtr )
{
Particle_Index* globalParticlesOutsideDomainCounts = NULL;
Particle_Index maxGlobalParticlesOutsideDomainCount = 0;
Processor_Index proc_I = 0;
Particle_Index lParticle_I = 0;
Particle_Index particle_I = 0;
Journal_DPrintfL( self->debug, 2, "In %s():\n", __func__ );
Stream_IndentBranch( Swarm_Debug );
(*globalParticlesArrivingMyDomainCountPtr) = 0;
(*globalParticlesOutsideDomainTotalPtr) = 0;
/* Find the counts of particles outside domain... */
ParticleMovementHandler_GetCountOfParticlesOutsideDomainPerProcessor(
self,
&globalParticlesOutsideDomainCounts,
&maxGlobalParticlesOutsideDomainCount,
globalParticlesOutsideDomainTotalPtr );
if ( (*globalParticlesOutsideDomainTotalPtr) > 0 ) {
Particle* particlesLeavingMyDomain = NULL;
Particle* globalParticlesLeavingDomains = NULL;
SizeT particlesLeavingDomainSizeBytes = 0;
Cell_DomainIndex lCell_I = 0;
GlobalParticle* currParticle = NULL;
Particle_Index currProcParticlesOutsideDomainCount = 0;
Particle_Index currProcOffset = 0;
particlesLeavingDomainSizeBytes = self->swarm->particleExtensionMgr->finalSize
* maxGlobalParticlesOutsideDomainCount;
particlesLeavingMyDomain = Memory_Alloc_Bytes( particlesLeavingDomainSizeBytes, "Particle",
"particlesLeavingMyDomain" );
// TODO: investigate doing this with an MPI_Indexed datatype instead...
Journal_DPrintfL( self->debug, 2, "Copying particles leaving my domain to temp. transfer array\n" );
Stream_IndentBranch( Swarm_Debug );
#if 0
MPI_Type_indexed(
self->particlesOutsideDomainTotalCount,
blocklens,
self->particlesOutsideDomainIndices,//change to contiguous indices?
MPI_BYTE,
ParticlesLeavingDomainTransferIndexed
);
#endif
for ( particle_I=0; particle_I < self->particlesOutsideDomainTotalCount; particle_I++ ) {
Journal_DPrintfL( self->debug, 3, "Copying particle %d to particlesLeavingMyDomain[%d]\n",
self->particlesOutsideDomainIndices[particle_I], particle_I );
Swarm_CopyParticleOffSwarm( self->swarm,
particlesLeavingMyDomain, particle_I,
self->particlesOutsideDomainIndices[particle_I] );
}
Stream_UnIndentBranch( Swarm_Debug );
/* allocate the big global receive buffer */
globalParticlesLeavingDomains = Memory_Alloc_Bytes( particlesLeavingDomainSizeBytes * self->swarm->nProc,
"Particle", "globalParticlesLeavingDomains" );
Journal_DPrintfL( self->debug, 2, "Getting the global array of particles leaving domains\n" );
(void)MPI_Allgather( particlesLeavingMyDomain, particlesLeavingDomainSizeBytes, MPI_BYTE,
globalParticlesLeavingDomains, particlesLeavingDomainSizeBytes, MPI_BYTE,
self->swarm->comm );
Journal_DPrintfL( self->debug, 2, "Checking through the global array of particles leaving domains, "
"and snaffling those moving into my domain:\n" );
Stream_IndentBranch( Swarm_Debug );
for ( proc_I=0; proc_I < self->swarm->nProc; proc_I++ ) {
if ( proc_I == self->swarm->myRank ) continue;
currProcOffset = proc_I * maxGlobalParticlesOutsideDomainCount;
currProcParticlesOutsideDomainCount = globalParticlesOutsideDomainCounts[proc_I];
Journal_DPrintfL( self->debug, 3, "Checking particles that left proc. %d:\n", proc_I );
for ( particle_I=0; particle_I < currProcParticlesOutsideDomainCount; particle_I++ ) {
currParticle = (GlobalParticle*)ParticleAt( globalParticlesLeavingDomains,
(currProcOffset + particle_I),
self->swarm->particleExtensionMgr->finalSize );
lCell_I = CellLayout_CellOf( self->swarm->cellLayout, currParticle );
if ( lCell_I < self->swarm->cellLocalCount ) {
#if DEBUG
Journal_DPrintfL( self->debug, 3, "Found particle at (%.2f,%.2f,%.2f) that's moved "
"into my local cell %d...\n", currParticle->coord[0],
currParticle->coord[1], currParticle->coord[2], lCell_I );
#endif
/* copy particle to the lowest available slot in my particles array */
lParticle_I = ParticleMovementHandler_FindFreeSlotAndPrepareForInsertion( (ParticleCommHandler*)self );
Swarm_CopyParticleOntoSwarm( self->swarm, lParticle_I,
globalParticlesLeavingDomains, (currProcOffset + particle_I) );
Swarm_AddParticleToCell( self->swarm, lCell_I, lParticle_I );
(*globalParticlesArrivingMyDomainCountPtr)++;
}
#if DEBUG
else {
currParticle = (GlobalParticle*)ParticleAt( globalParticlesLeavingDomains,
(currProcOffset + particle_I),
self->swarm->particleExtensionMgr->finalSize );
Journal_DPrintfL( self->debug, 3, "Ignoring particle at (%.2f,%.2f,%.2f) since "
"not in my local cells...\n", currParticle->coord[0],
currParticle->coord[1], currParticle->coord[2] );
}
#endif
}
}
Stream_UnIndentBranch( Swarm_Debug );
Memory_Free( particlesLeavingMyDomain );
Memory_Free( globalParticlesLeavingDomains );
/* Defensive check to make sure particles not lost/created accidentally somehow */
if( self->defensive == True ) {
ParticleMovementHandler_EnsureParticleCountLeavingDomainsEqualsCountEnteringGlobally( self );
}
}
Memory_Free( globalParticlesOutsideDomainCounts );
Stream_UnIndentBranch( Swarm_Debug );
}
