void _ParticleShadowSync_Init(
ParticleShadowSync* self )
{
_ParticleCommHandler_Init( (ParticleCommHandler*)self );
self->particlesOutsideDomainIndices = NULL;
_ParticleCommHandler_ZeroShadowCommStrategyCounters( (ParticleCommHandler*)self );
}
void ParticleShadowSync_HandleParticleMovementBetweenProcs( ParticleCommHandler* pCommsHandler ) {
ParticleShadowSync* self = (ParticleShadowSync*)pCommsHandler;
Journal_DPrintfL( self->debug, 1, "In %s(), for swarm \"%s\":\n", __func__, self->swarm->name );
if ( 1 == self->swarm->nProc ) {
Journal_DPrintfL( self->debug, 1, "Serial run -> nothing to communicate in %s, returning.\n", __func__ );
Stream_UnIndentBranch( Swarm_Debug );
return;
}
Stream_IndentBranch( Swarm_Debug );
if ( self->swarm->cellShadowCount > 0 ) {
/* Allocate the recv count arrays and handles */
/*---_ParticleCommHandler_AllocateIncomingCountArrays( (ParticleCommHandler*)self );
_ParticleCommHandler_AllocateOutgoingCountArrays( (ParticleCommHandler*)self );*/
self->allocateIncomingCountArrays( (ParticleCommHandler*)self );
self->allocateOutgoingCountArrays( (ParticleCommHandler*)self );
/* First thing to do is begin non-blocking receive of incoming particles (for latency hiding) */
/*---_ParticleCommHandler_BeginReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );*/
self->beginReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );
/* Do a blocking send of outgoing counts, so our nbrs know what to receive */
//_ParticleCommHandler_SendParticleTotalsInShadowCellsToNbrs( (ParticleCommHandler*)self );
/*---_ParticleShadowSync_SendParticleTotalsInShadowCellsToNbrs( (ParticleCommHandler*)self );*/
self->sendOutgoingParticleCounts( (ParticleCommHandler*)self );
/* Now need to make sure that incoming particle counts are here, then begin receiving particles
(We do this as early as possible for latency hiding purposes */
/*---_ParticleShadowSync_FinishReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );*/
self->finishReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );
/*---_ParticleShadowSync_BeginReceiveOfIncomingParticles( (ParticleCommHandler*)self );
_ParticleShadowSync_SendShadowParticles( (ParticleCommHandler*)self );*/
self->beginReceiveOfIncomingParticles( (ParticleCommHandler*)self );
self->beginSendingParticles( (ParticleCommHandler*)self );
/*---_ParticleCommHandler_ConfirmOutgoingSendsCompleted( (ParticleCommHandler*)self );
_ParticleShadowSync_FinishReceiveOfIncomingParticles( self );*/
self->confirmOutgoingSendsCompleted( (ParticleCommHandler*)self );
self->finishReceiveOfIncomingParticlesAndUpdateIndices( (ParticleCommHandler*)self );
/*---_ParticleCommHandler_FreeOutgoingArrays( (ParticleCommHandler*)self );
_ParticleCommHandler_FreeIncomingArrays( (ParticleCommHandler*)self );*/
self->freeIncomingArrays( (ParticleCommHandler*)self );
self->freeOutgoingArrays( (ParticleCommHandler*)self );
/* This is where our own Data structure comes into play Jules and Raq*/
//----_ParticleCommHandler_AllocateIncomingParticlesArrays( (ParticleCommHandler*)self );
//----_ParticleCommHandler_BeginReceiveOfIncomingParticles( (ParticleCommHandler*)self );
/* OK, now begin sending out particles we know need to go to nbrs */
//----_ParticleCommHandler_AllocateOutgoingParticlesArrays( (ParticleCommHandler*)self );
//----_ParticleCommHandler_BeginSendingParticlesInShadowCellsToNbrs( (ParticleCommHandler*)self );
}
MPI_Barrier( self->swarm->comm );
_ParticleCommHandler_ZeroShadowCommStrategyCounters( (ParticleCommHandler*)self );
Stream_UnIndentBranch( Swarm_Debug );
}
void ParticleMovementHandler_HandleParticleMovementBetweenProcs( ParticleCommHandler* pCommsHandler ) {
ParticleMovementHandler* self = (ParticleMovementHandler*)pCommsHandler;
double startTime = 0;
Stream* info = Journal_Register( Info_Type, (Name)self->type );
Journal_DPrintfL( self->debug, 1, "In %s(), for swarm \"%s\":\n", __func__, self->swarm->name );
if ( 1 == self->swarm->nProc ) {
Journal_DPrintfL( self->debug, 1, "Serial run -> nothing to communicate, returning.\n" );
Stream_UnIndentBranch( Swarm_Debug );
return;
}
Stream_IndentBranch( Swarm_Debug );
startTime = MPI_Wtime();
if ( self->swarm->cellShadowCount > 0 ) {
/* Allocate the recv count arrays and handles */
self->allocateIncomingCountArrays( (ParticleCommHandler*)self );
self->allocateOutgoingCountArrays( (ParticleCommHandler*)self );
/* First thing to do is begin non-blocking receive of incoming particles (for latency hiding) */
self->beginReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );
/* Do a blocking send of outgoing counts, so our nbrs know what to receive */
self->sendOutgoingParticleCounts( (ParticleCommHandler*)self );
/* Now need to make sure that incoming particle counts are here, then begin receiving particles
(We do this as early as possible for latency hiding purposes */
self->finishReceiveOfIncomingParticleCounts( (ParticleCommHandler*)self );
self->allocateIncomingParticleArrays( (ParticleCommHandler*)self );
self->beginReceiveOfIncomingParticles( (ParticleCommHandler*)self );
/* OK, now begin sending out particles we know need to go to nbrs */
self->allocateOutgoingParticleArrays( (ParticleCommHandler*)self );
self->beginSendingParticles( (ParticleCommHandler*)self );
}
if ( self->useGlobalFallbackCommStrategy ) {
self->particlesOutsideDomainIndices = Memory_Alloc_Array( Particle_Index, self->swarm->particlesArrayDelta,
"self->particlesOutsideDomainIndices" );
ParticleMovementHandler_DoGlobalFallbackCommunication( self );
}
if ( self->swarm->cellShadowCount > 0 ) {
/* Ok, at this point make sure our non-blocking receive of incoming via domain boundary
* particles have completed: then immediately insert these particles into our local swarm */
self->finishReceiveOfIncomingParticlesAndUpdateIndices( (ParticleCommHandler*)self );
}
/* final update of the 'holes' in my particles list (some may be left if we sent more than we recv'd overall */
ParticleMovementHandler_FillRemainingHolesInLocalParticlesArray( (ParticleCommHandler*)self );
/* NB: Can't delete these until here, as they're needed by the "fillRemainingHoles" function above. */
if ( self->useGlobalFallbackCommStrategy ) {
Memory_Free( self->particlesOutsideDomainIndices );
self->particlesOutsideDomainIndices = NULL;
}
if ( self->swarm->cellShadowCount > 0 ) {
self->confirmOutgoingSendsCompleted( (ParticleCommHandler*)self );
}
_ParticleCommHandler_PrintCommunicationVolumeStats( (ParticleCommHandler*)self, startTime, info );
MPI_Barrier( self->swarm->comm );
/* clean up allocated memory, and zero counters, ready for next timestep */
if ( self->swarm->cellShadowCount > 0 ) {
self->freeIncomingArrays( (ParticleCommHandler*)self );
self->freeOutgoingArrays( (ParticleCommHandler*)self );
}
_ParticleCommHandler_ZeroShadowCommStrategyCounters( (ParticleCommHandler*)self );
Stream_UnIndentBranch( Swarm_Debug );
}
