void _ParticleShadowSync_SendShadowParticles( ParticleCommHandler *self ) { ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout ); ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo; Processor_Index proc_I; int i = 0, j = 0, k = 0, cell = 0; unsigned int arrayIndex = 0; long arraySize = 0; unsigned int pIndex = 0; self->shadowParticlesLeavingMeHandles = Memory_Alloc_Array_Unnamed( MPI_Request*, procNbrInfo->procNbrCnt ); self->shadowParticlesLeavingMe = Memory_Alloc_Array_Unnamed( Particle*, procNbrInfo->procNbrCnt ); for( i=0; i<procNbrInfo->procNbrCnt; i++ ){ proc_I = procNbrInfo->procNbrTbl[i]; if( self->shadowParticlesLeavingMeTotalCounts[i] != 0 ){ self->shadowParticlesLeavingMeHandles[i] = Memory_Alloc_Array_Unnamed( MPI_Request, 1 ); arraySize = self->swarm->particleExtensionMgr->finalSize * self->shadowParticlesLeavingMeTotalCounts[i]; self->shadowParticlesLeavingMe[i] = Memory_Alloc_Bytes( arraySize, "Particle", "pCommHandler->outgoingPArray" ); memset( self->shadowParticlesLeavingMe[i], 0, arraySize ); arrayIndex = 0; for( j=0; j<cellShadowInfo->procShadowedCnt[i]; j++ ){ cell = cellShadowInfo->procShadowedTbl[i][j]; for( k=0; k<self->swarm->cellParticleCountTbl[cell]; k++ ){ pIndex = self->swarm->cellParticleTbl[cell][k]; Swarm_CopyParticleOffSwarm( self->swarm, self->shadowParticlesLeavingMe[i], arrayIndex++, pIndex ); } } /*printf( "sending %ld bytes\n", self->shadowParticlesLeavingMeTotalCounts[i] * self->swarm->particleExtensionMgr->finalSize );*/ MPI_Issend( self->shadowParticlesLeavingMe[i], self->shadowParticlesLeavingMeTotalCounts[i] * self->swarm->particleExtensionMgr->finalSize, MPI_BYTE, proc_I, SHADOW_PARTICLES, self->swarm->comm, self->shadowParticlesLeavingMeHandles[i] ); } } }
void ParticleMovementHandler_ShareAndUpdateParticlesThatHaveMovedOutsideDomains( ParticleMovementHandler* self, Particle_Index* globalParticlesArrivingMyDomainCountPtr, Particle_Index* globalParticlesOutsideDomainTotalPtr ) { Particle_Index* globalParticlesOutsideDomainCounts = NULL; Particle_Index maxGlobalParticlesOutsideDomainCount = 0; Processor_Index proc_I = 0; Particle_Index lParticle_I = 0; Particle_Index particle_I = 0; Journal_DPrintfL( self->debug, 2, "In %s():\n", __func__ ); Stream_IndentBranch( Swarm_Debug ); (*globalParticlesArrivingMyDomainCountPtr) = 0; (*globalParticlesOutsideDomainTotalPtr) = 0; /* Find the counts of particles outside domain... */ ParticleMovementHandler_GetCountOfParticlesOutsideDomainPerProcessor( self, &globalParticlesOutsideDomainCounts, &maxGlobalParticlesOutsideDomainCount, globalParticlesOutsideDomainTotalPtr ); if ( (*globalParticlesOutsideDomainTotalPtr) > 0 ) { Particle* particlesLeavingMyDomain = NULL; Particle* globalParticlesLeavingDomains = NULL; SizeT particlesLeavingDomainSizeBytes = 0; Cell_DomainIndex lCell_I = 0; GlobalParticle* currParticle = NULL; Particle_Index currProcParticlesOutsideDomainCount = 0; Particle_Index currProcOffset = 0; particlesLeavingDomainSizeBytes = self->swarm->particleExtensionMgr->finalSize * maxGlobalParticlesOutsideDomainCount; particlesLeavingMyDomain = Memory_Alloc_Bytes( particlesLeavingDomainSizeBytes, "Particle", "particlesLeavingMyDomain" ); // TODO: investigate doing this with an MPI_Indexed datatype instead... Journal_DPrintfL( self->debug, 2, "Copying particles leaving my domain to temp. transfer array\n" ); Stream_IndentBranch( Swarm_Debug ); #if 0 MPI_Type_indexed( self->particlesOutsideDomainTotalCount, blocklens, self->particlesOutsideDomainIndices,//change to contiguous indices? MPI_BYTE, ParticlesLeavingDomainTransferIndexed ); #endif for ( particle_I=0; particle_I < self->particlesOutsideDomainTotalCount; particle_I++ ) { Journal_DPrintfL( self->debug, 3, "Copying particle %d to particlesLeavingMyDomain[%d]\n", self->particlesOutsideDomainIndices[particle_I], particle_I ); Swarm_CopyParticleOffSwarm( self->swarm, particlesLeavingMyDomain, particle_I, self->particlesOutsideDomainIndices[particle_I] ); } Stream_UnIndentBranch( Swarm_Debug ); /* allocate the big global receive buffer */ globalParticlesLeavingDomains = Memory_Alloc_Bytes( particlesLeavingDomainSizeBytes * self->swarm->nProc, "Particle", "globalParticlesLeavingDomains" ); Journal_DPrintfL( self->debug, 2, "Getting the global array of particles leaving domains\n" ); (void)MPI_Allgather( particlesLeavingMyDomain, particlesLeavingDomainSizeBytes, MPI_BYTE, globalParticlesLeavingDomains, particlesLeavingDomainSizeBytes, MPI_BYTE, self->swarm->comm ); Journal_DPrintfL( self->debug, 2, "Checking through the global array of particles leaving domains, " "and snaffling those moving into my domain:\n" ); Stream_IndentBranch( Swarm_Debug ); for ( proc_I=0; proc_I < self->swarm->nProc; proc_I++ ) { if ( proc_I == self->swarm->myRank ) continue; currProcOffset = proc_I * maxGlobalParticlesOutsideDomainCount; currProcParticlesOutsideDomainCount = globalParticlesOutsideDomainCounts[proc_I]; Journal_DPrintfL( self->debug, 3, "Checking particles that left proc. %d:\n", proc_I ); for ( particle_I=0; particle_I < currProcParticlesOutsideDomainCount; particle_I++ ) { currParticle = (GlobalParticle*)ParticleAt( globalParticlesLeavingDomains, (currProcOffset + particle_I), self->swarm->particleExtensionMgr->finalSize ); lCell_I = CellLayout_CellOf( self->swarm->cellLayout, currParticle ); if ( lCell_I < self->swarm->cellLocalCount ) { #if DEBUG Journal_DPrintfL( self->debug, 3, "Found particle at (%.2f,%.2f,%.2f) that's moved " "into my local cell %d...\n", currParticle->coord[0], currParticle->coord[1], currParticle->coord[2], lCell_I ); #endif /* copy particle to the lowest available slot in my particles array */ lParticle_I = ParticleMovementHandler_FindFreeSlotAndPrepareForInsertion( (ParticleCommHandler*)self ); Swarm_CopyParticleOntoSwarm( self->swarm, lParticle_I, globalParticlesLeavingDomains, (currProcOffset + particle_I) ); Swarm_AddParticleToCell( self->swarm, lCell_I, lParticle_I ); (*globalParticlesArrivingMyDomainCountPtr)++; } #if DEBUG else { currParticle = (GlobalParticle*)ParticleAt( globalParticlesLeavingDomains, (currProcOffset + particle_I), self->swarm->particleExtensionMgr->finalSize ); Journal_DPrintfL( self->debug, 3, "Ignoring particle at (%.2f,%.2f,%.2f) since " "not in my local cells...\n", currParticle->coord[0], currParticle->coord[1], currParticle->coord[2] ); } #endif } } Stream_UnIndentBranch( Swarm_Debug ); Memory_Free( particlesLeavingMyDomain ); Memory_Free( globalParticlesLeavingDomains ); /* Defensive check to make sure particles not lost/created accidentally somehow */ if( self->defensive == True ) { ParticleMovementHandler_EnsureParticleCountLeavingDomainsEqualsCountEnteringGlobally( self ); } } Memory_Free( globalParticlesOutsideDomainCounts ); Stream_UnIndentBranch( Swarm_Debug ); }