void _ParticleShadowSync_Init( ParticleShadowSync* self ) { _ParticleCommHandler_Init( (ParticleCommHandler*)self ); self->particlesOutsideDomainIndices = NULL; _ParticleCommHandler_ZeroShadowCommStrategyCounters( (ParticleCommHandler*)self ); }
void ParticleShadowSync_HandleParticleMovementBetweenProcs( ParticleCommHandler* pCommsHandler ) { ParticleShadowSync* self = (ParticleShadowSync*)pCommsHandler; Journal_DPrintfL( self->debug, 1, "In %s(), for swarm \"%s\":\n", __func__, self->swarm->name ); if ( 1 == self->swarm->nProc ) { Journal_DPrintfL( self->debug, 1, "Serial run -> nothing to communicate in %s, returning.\n", __func__ ); Stream_UnIndentBranch( Swarm_Debug ); return; } Stream_IndentBranch( Swarm_Debug ); if ( self->swarm->cellShadowCount > 0 ) { /* Allocate the recv count arrays and handles */ /*---_ParticleCommHandler_AllocateIncomingCountArrays( (ParticleCommHandler*)self ); _ParticleCommHandler_AllocateOutgoingCountArrays( (ParticleCommHandler*)self );*/ self->allocateIncomingCountArrays( (ParticleCommHandler*)self ); self->allocateOutgoingCountArrays( (ParticleCommHandler*)self ); /* First thing to do is begin non-blocking receive of incoming particles (for latency hiding) */ /*---_ParticleCommHandler_BeginReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );*/ self->beginReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self ); /* Do a blocking send of outgoing counts, so our nbrs know what to receive */ //_ParticleCommHandler_SendParticleTotalsInShadowCellsToNbrs( (ParticleCommHandler*)self ); /*---_ParticleShadowSync_SendParticleTotalsInShadowCellsToNbrs( (ParticleCommHandler*)self );*/ self->sendOutgoingParticleCounts( (ParticleCommHandler*)self ); /* Now need to make sure that incoming particle counts are here, then begin receiving particles (We do this as early as possible for latency hiding purposes */ /*---_ParticleShadowSync_FinishReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );*/ self->finishReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self ); /*---_ParticleShadowSync_BeginReceiveOfIncomingParticles( (ParticleCommHandler*)self ); _ParticleShadowSync_SendShadowParticles( (ParticleCommHandler*)self );*/ self->beginReceiveOfIncomingParticles( (ParticleCommHandler*)self ); self->beginSendingParticles( (ParticleCommHandler*)self ); /*---_ParticleCommHandler_ConfirmOutgoingSendsCompleted( (ParticleCommHandler*)self ); _ParticleShadowSync_FinishReceiveOfIncomingParticles( self );*/ self->confirmOutgoingSendsCompleted( (ParticleCommHandler*)self ); self->finishReceiveOfIncomingParticlesAndUpdateIndices( (ParticleCommHandler*)self ); /*---_ParticleCommHandler_FreeOutgoingArrays( (ParticleCommHandler*)self ); _ParticleCommHandler_FreeIncomingArrays( (ParticleCommHandler*)self );*/ self->freeIncomingArrays( (ParticleCommHandler*)self ); self->freeOutgoingArrays( (ParticleCommHandler*)self ); /* This is where our own Data structure comes into play Jules and Raq*/ //----_ParticleCommHandler_AllocateIncomingParticlesArrays( (ParticleCommHandler*)self ); //----_ParticleCommHandler_BeginReceiveOfIncomingParticles( (ParticleCommHandler*)self ); /* OK, now begin sending out particles we know need to go to nbrs */ //----_ParticleCommHandler_AllocateOutgoingParticlesArrays( (ParticleCommHandler*)self ); //----_ParticleCommHandler_BeginSendingParticlesInShadowCellsToNbrs( (ParticleCommHandler*)self ); } MPI_Barrier( self->swarm->comm ); _ParticleCommHandler_ZeroShadowCommStrategyCounters( (ParticleCommHandler*)self ); Stream_UnIndentBranch( Swarm_Debug ); }
void ParticleMovementHandler_HandleParticleMovementBetweenProcs( ParticleCommHandler* pCommsHandler ) { ParticleMovementHandler* self = (ParticleMovementHandler*)pCommsHandler; double startTime = 0; Stream* info = Journal_Register( Info_Type, (Name)self->type ); Journal_DPrintfL( self->debug, 1, "In %s(), for swarm \"%s\":\n", __func__, self->swarm->name ); if ( 1 == self->swarm->nProc ) { Journal_DPrintfL( self->debug, 1, "Serial run -> nothing to communicate, returning.\n" ); Stream_UnIndentBranch( Swarm_Debug ); return; } Stream_IndentBranch( Swarm_Debug ); startTime = MPI_Wtime(); if ( self->swarm->cellShadowCount > 0 ) { /* Allocate the recv count arrays and handles */ self->allocateIncomingCountArrays( (ParticleCommHandler*)self ); self->allocateOutgoingCountArrays( (ParticleCommHandler*)self ); /* First thing to do is begin non-blocking receive of incoming particles (for latency hiding) */ self->beginReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self ); /* Do a blocking send of outgoing counts, so our nbrs know what to receive */ self->sendOutgoingParticleCounts( (ParticleCommHandler*)self ); /* Now need to make sure that incoming particle counts are here, then begin receiving particles (We do this as early as possible for latency hiding purposes */ self->finishReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self ); self->allocateIncomingParticleArrays( (ParticleCommHandler*)self ); self->beginReceiveOfIncomingParticles( (ParticleCommHandler*)self ); /* OK, now begin sending out particles we know need to go to nbrs */ self->allocateOutgoingParticleArrays( (ParticleCommHandler*)self ); self->beginSendingParticles( (ParticleCommHandler*)self ); } if ( self->useGlobalFallbackCommStrategy ) { self->particlesOutsideDomainIndices = Memory_Alloc_Array( Particle_Index, self->swarm->particlesArrayDelta, "self->particlesOutsideDomainIndices" ); ParticleMovementHandler_DoGlobalFallbackCommunication( self ); } if ( self->swarm->cellShadowCount > 0 ) { /* Ok, at this point make sure our non-blocking receive of incoming via domain boundary * particles have completed: then immediately insert these particles into our local swarm */ self->finishReceiveOfIncomingParticlesAndUpdateIndices( (ParticleCommHandler*)self ); } /* final update of the 'holes' in my particles list (some may be left if we sent more than we recv'd overall */ ParticleMovementHandler_FillRemainingHolesInLocalParticlesArray( (ParticleCommHandler*)self ); /* NB: Can't delete these until here, as they're needed by the "fillRemainingHoles" function above. */ if ( self->useGlobalFallbackCommStrategy ) { Memory_Free( self->particlesOutsideDomainIndices ); self->particlesOutsideDomainIndices = NULL; } if ( self->swarm->cellShadowCount > 0 ) { self->confirmOutgoingSendsCompleted( (ParticleCommHandler*)self ); } _ParticleCommHandler_PrintCommunicationVolumeStats( (ParticleCommHandler*)self, startTime, info ); MPI_Barrier( self->swarm->comm ); /* clean up allocated memory, and zero counters, ready for next timestep */ if ( self->swarm->cellShadowCount > 0 ) { self->freeIncomingArrays( (ParticleCommHandler*)self ); self->freeOutgoingArrays( (ParticleCommHandler*)self ); } _ParticleCommHandler_ZeroShadowCommStrategyCounters( (ParticleCommHandler*)self ); Stream_UnIndentBranch( Swarm_Debug ); }