Particle_Index ParticleMovementHandler_FindFreeSlotAndPrepareForInsertion( ParticleCommHandler* self )
{
Particle_Index lParticle_I = 0;
if ( self->shadowParticlesLeavingMeUnfilledCount > 0 ) {
Journal_DPrintfL( self->debug, 3, "Still %d holes available from "
"particles leaving via shadow cells\n-> free slot to add into is %d\n",
self->shadowParticlesLeavingMeUnfilledCount,
self->shadowParticlesLeavingMeIndices[self->currShadowParticleLeavingMeIndex]);
lParticle_I = self->shadowParticlesLeavingMeIndices[self->currShadowParticleLeavingMeIndex];
self->currShadowParticleLeavingMeIndex++;
self->shadowParticlesLeavingMeUnfilledCount--;
}
else if ( self->particlesOutsideDomainUnfilledCount ) {
Journal_DPrintfL( self->debug, 3, "Still %d holes available from "
"particles leaving domain direct\n-> free slot to add into is %d\n",
self->particlesOutsideDomainUnfilledCount,
self->particlesOutsideDomainIndices[self->currParticleLeavingMyDomainIndex]);
lParticle_I = self->particlesOutsideDomainIndices[self->currParticleLeavingMyDomainIndex];
self->currParticleLeavingMyDomainIndex++;
self->particlesOutsideDomainUnfilledCount--;
}
else {
Journal_DPrintfL( self->debug, 3, "No holes left from leaving "
"particles\n-> slot to insert into is end of array %d\n",
self->swarm->particleLocalCount );
lParticle_I = self->swarm->particleLocalCount;
/*
if ( self->swarm->particleLocalCount == self->swarm->particlesArraySize ) {
Journal_DPrintfL( self->debug, 3, "Particles array memory used up "
"-> increasing from %d entries by %d\n",
self->swarm->particlesArraySize, self->swarm->particlesArrayDelta );
self->swarm->particlesArraySize += self->swarm->particlesArrayDelta;
self->swarm->particles = Memory_Realloc_Array_Bytes(
self->swarm->particles,
self->swarm->particleExtensionMgr->finalSize,
self->swarm->particlesArraySize );
}
*/
self->swarm->particleLocalCount++;
Swarm_Realloc( self->swarm );
}
return lParticle_I;
}
PyObject* GeneralSwarm_AddParticlesFromCoordArray( void* swarm, Index count, Index dim, double* array )
{
GeneralSwarm* self = (GeneralSwarm*)swarm;
unsigned* cellArray = Memory_Alloc_Array( unsigned, count, "GeneralSwarm_AddParticlesFromCoordArray_CellArray" );
GlobalParticle localParticle;
GlobalParticle* particle = &localParticle;
int cellLocalCount = self->cellLocalCount;
int oldParticleCount = self->particleLocalCount;
int ii;
int totsLocalParticles=0;
// find which particles are local. we do this to avoid swarm reallocs.
for (ii=0; ii<count; ii++) {
memcpy(&(particle->coord), array + dim*ii, dim*sizeof(double));
cellArray[ii] = CellLayout_CellOf( self->cellLayout, particle );
if( cellArray[ii] < cellLocalCount )
totsLocalParticles++;
}
// alloc particle local index array (to be returned)
int* partLocalIndex = Memory_Alloc_Array( int, count, "GeneralSwarm_AddParticlesFromCoordArray_CellArray" );
// ok, lets add them to the swarm, now that we know how many are required
self->particleLocalCount += totsLocalParticles;
Swarm_Realloc( self );
int newPartIndex = oldParticleCount;
for (ii=0; ii<count; ii++) {
if( cellArray[ii] < cellLocalCount ){
particle = (GlobalParticle*)Swarm_ParticleAt( self, newPartIndex );
memcpy(&(particle->coord), array + dim*ii, dim*sizeof(double));
Swarm_AddParticleToCell( swarm, cellArray[ii], newPartIndex );
partLocalIndex[ii] = newPartIndex;
newPartIndex++;
} else {
partLocalIndex[ii] = -1;
}
}
/* create numpy array to return */
npy_intp dims[1] = { count };
PyObject* pyobj = PyArray_New(&PyArray_Type, 1, dims, NPY_INT, NULL, (void*)partLocalIndex, sizeof(int), 0, NULL);
/* enable the owndata flag.. this tells numpy to dealloc the data when it is finished with it */
#if NPY_API_VERSION < 0x00000007
(((PyArrayObject*)pyobj)->flags) = NPY_ARRAY_OWNDATA;
#else
PyArray_ENABLEFLAGS((PyArrayObject*)pyobj, NPY_ARRAY_OWNDATA);
#endif
return pyobj;
}
int GeneralSwarm_AddParticle( void* swarm, Index dim, double xI, double xJ, double xK )
{
GeneralSwarm* self = (GeneralSwarm*)swarm;
GlobalParticle localParticle;
GlobalParticle* particle = &localParticle;
unsigned cell;
int newPartIndex;
/* If the location is not local, return immediately with index value -1 */
particle->coord[ I_AXIS ] = xI;
particle->coord[ J_AXIS ] = xJ;
if(dim == 3)
particle->coord[ K_AXIS ] = xK;
cell = CellLayout_CellOf( self->cellLayout, particle );
if( cell >= self->cellLocalCount )
return (-1);
/* Otherwise we allocate the particle to the Swarm and return the new index */
newPartIndex = self->particleLocalCount;
self->particleLocalCount ++;
Swarm_Realloc( self );
particle = (GlobalParticle*)Swarm_ParticleAt( self, newPartIndex );
particle->coord[ I_AXIS ] = xI;
particle->coord[ J_AXIS ] = xJ;
if(dim==3)
particle->coord[ K_AXIS ] = xK;
Swarm_AddParticleToCell( self, cell, newPartIndex );
return(newPartIndex);
}
void ParticleMovementHandler_FillRemainingHolesInLocalParticlesArray( ParticleCommHandler* self )
{
Particle_Index prevParticlesArraySize = self->swarm->particlesArraySize;
Particle_Index numHolesToFill;
Particle_InCellIndex cParticle_I;
StandardParticle* oldPtrToMovedParticle;
Cell_LocalIndex owningCell;
Particle_Index indexToInsertAt;
Particle_Index* leavingParticlesArray = NULL;
Index currLeavingParticleArrayEntry = 0;
Index highestLeavingParticleArrayEntry;
Index leavingParticlesUnfilledCount = 0;
Particle_Index highestLeavingParticleIndex;
Particle_Index candidateParticleToMove;
Bool finishedFlag = False;
Bool mergedArrayCreated = False;
Particle_Index prevParticleCount = self->swarm->particleLocalCount;
Journal_DPrintf( self->debug, "In %s():\n", __func__ );
Stream_IndentBranch( Swarm_Debug );
numHolesToFill = self->particlesOutsideDomainUnfilledCount + self->shadowParticlesLeavingMeUnfilledCount;
if ( numHolesToFill == 0 ) {
Journal_DPrintfL( self->debug, 2, "No holes to fill -> nothing to do, returning.\n" );
Stream_UnIndentBranch( Swarm_Debug );
return;
}
#if DEBUG
if ( Stream_IsPrintableLevel( self->debug, 2 ) ) {
ParticleMovementHandler_PrintParticleSlotsYetToFill( self );
}
#endif
/* work out the list we have to iterate over: */
if ( self->shadowParticlesLeavingMeUnfilledCount && !self->particlesOutsideDomainUnfilledCount ) {
Journal_DPrintfL( self->debug, 2, "Particles have only left via shadow cells -> no need to merge lists\n" );
leavingParticlesArray = &self->shadowParticlesLeavingMeIndices[self->currShadowParticleLeavingMeIndex];
}
else if ( self->particlesOutsideDomainUnfilledCount && !self->shadowParticlesLeavingMeUnfilledCount ) {
Journal_DPrintfL( self->debug, 2, "Particles have only left domain directly -> no need to merge lists\n" );
leavingParticlesArray = &self->particlesOutsideDomainIndices[self->currParticleLeavingMyDomainIndex];
}
else {
Journal_DPrintfL( self->debug, 2, "Particles have left both via shadow cells and domain directly -> merge lists\n" );
leavingParticlesArray = ParticleMovementHandler_MergeListsOfUnfilledParticleSlots( self );
mergedArrayCreated = True;
}
/* Ok: while there are holes left to fill, find the highest candidate, move it, and reduce the count. */
Journal_DPrintfL( self->debug, 2, "Starting run through the %d particles to fill:\n", numHolesToFill );
Stream_IndentBranch( Swarm_Debug );
currLeavingParticleArrayEntry = 0;
highestLeavingParticleArrayEntry = numHolesToFill-1;
leavingParticlesUnfilledCount = numHolesToFill;
while ( leavingParticlesUnfilledCount > 0 ) {
indexToInsertAt = leavingParticlesArray[currLeavingParticleArrayEntry];
Journal_DPrintfL( self->debug, 3, "Attempting to fill leaving slot %d (at particle index %d):\n",
currLeavingParticleArrayEntry, indexToInsertAt );
Stream_Indent( self->debug );
/* This is where we work out the index of which particle to move into the free slot.
* We Start from the end of the particles array, then decrement by 1 until we find a candidate that
* hasn't itself already left.
* We also need to consider the possibility that every candidate higher than the current index
* has also left, in which case we are done, and finish the while process.
* See the ParticleCommHandler Twiki page for diagrams illustrating this algorithm.
*/
candidateParticleToMove = self->swarm->particleLocalCount-1;
highestLeavingParticleIndex = leavingParticlesArray[highestLeavingParticleArrayEntry];
Journal_DPrintfL( self->debug, 3, "Searching for highest particle that hasn't also moved:\n" );
Stream_Indent( self->debug );
while ( candidateParticleToMove == leavingParticlesArray[highestLeavingParticleArrayEntry] ) {
/* Check if that was the last candidate particle above the current one: */
/* This test needs to be at the top of this loop to handle the case where we have one
particle that's leaving */
if ( candidateParticleToMove <= indexToInsertAt ) {
Journal_DPrintfL( self->debug, 3, "** No more particles above current "
"hole %d to fill: we're done. **\n", indexToInsertAt );
/* Need the line below to mark the fact we failed to fill the current indexToInsertAt hole */
self->swarm->particleLocalCount--;
finishedFlag = True;
break;
}
Journal_DPrintfL( self->debug, 3, "Candidate particle %d has also left...\n",
candidateParticleToMove );
highestLeavingParticleArrayEntry--;
highestLeavingParticleIndex = leavingParticlesArray[highestLeavingParticleArrayEntry];
leavingParticlesUnfilledCount--;
self->swarm->particleLocalCount--;
candidateParticleToMove--;
}
Stream_UnIndent( self->debug );
if ( True == finishedFlag ) {
/* We must have hit the "no more candidate particles" criterion in the search loop, so
* quit trying to fill empty holes entirely. */
Stream_UnIndent( self->debug );
break;
}
Journal_DPrintfL( self->debug, 3, "Highest valid particle found at index %d:\n",
candidateParticleToMove );
Journal_DFirewall( (candidateParticleToMove > indexToInsertAt), Swarm_Error,
"Error in %s: Empty hole filling\nalgorithm has stuffed up somehow,"
" since particle to be moved %d is <= slot to insert into %d.\n",
__func__, candidateParticleToMove, indexToInsertAt );
Stream_Indent( self->debug );
Journal_DPrintfL( self->debug, 3, "Copying particle data from %d to %d\n",
candidateParticleToMove, indexToInsertAt );
Swarm_CopyParticleWithinSwarm( self->swarm, indexToInsertAt, candidateParticleToMove );
/* update the cell that the moved particle lives in to have the correct index into the
* particle array for it. */
oldPtrToMovedParticle = Swarm_ParticleAt( self->swarm, candidateParticleToMove );
owningCell = oldPtrToMovedParticle->owningCell;
cParticle_I = Swarm_GetParticleIndexWithinCell( self->swarm, owningCell, candidateParticleToMove );
Journal_DPrintfL( self->debug, 3, "Updating owning cell: (Cell %d, PIC index %d) now -> p.i. %d\n",
owningCell, cParticle_I, indexToInsertAt );
self->swarm->cellParticleTbl[owningCell][cParticle_I] = indexToInsertAt;
Stream_UnIndent( self->debug );
/* update the counters/indices */
currLeavingParticleArrayEntry++;
leavingParticlesUnfilledCount--;
self->swarm->particleLocalCount--;
Stream_UnIndent( self->debug );
}
Stream_UnIndentBranch( Swarm_Debug );
/* we only need to free the array of leaving particle slots if its a new merged list */
if ( mergedArrayCreated == True ) {
Memory_Free( leavingParticlesArray );
}
/* ------------------------- */
Journal_DPrintfL( self->debug, 2, "Local particle count reduced from %d to %d\n", prevParticleCount,
self->swarm->particleLocalCount );
/* Update the memory allocated to the particles array if particle count has reduced significantly */
while ( self->swarm->particlesArraySize > self->swarm->particleLocalCount + self->swarm->particlesArrayDelta ) {
self->swarm->particlesArraySize -= self->swarm->particlesArrayDelta;
}
if ( self->swarm->particlesArraySize < prevParticlesArraySize ) {
Journal_DPrintfL( self->debug, 2, "Reducing particles array entries from %d to %d\n",
prevParticlesArraySize, self->swarm->particlesArraySize );
Swarm_Realloc( self->swarm );
/*
self->swarm->particles = Memory_Realloc_Array_Bytes(
self->swarm->particles,
self->swarm->particleExtensionMgr->finalSize,
self->swarm->particlesArraySize );
*/
}
Stream_UnIndentBranch( Swarm_Debug );
}
void _GlobalParticleLayout_InitialiseParticles( void* particleLayout, void* _swarm )
{
GlobalParticleLayout* self = (GlobalParticleLayout*)particleLayout;
Swarm* swarm = (Swarm*)_swarm;
GlobalParticle* particle = NULL;
Particle_Index lParticle_I=0;
Particle_Index newParticle_I=0;
Cell_Index cell_I;
Particle_Index globalParticlesInitialisedCount=0;
Stream* errorStream = Journal_Register( Error_Type, self->type );
Journal_DPrintf( self->debug, "In %s(): for ParticleLayout \"%s\" (of type %s):\n",
__func__, self->name, self->type );
Stream_IndentBranch( Swarm_Debug );
Journal_DPrintf( self->debug, "For each of the %u total global requested particles, "
"generating a particle, and checking if it's in this processor's domain. If so, "
"adding it to the appropriate local cell.\n", self->totalInitialParticles );
Stream_IndentBranch( Swarm_Debug );
while( newParticle_I < self->totalInitialParticles ) {
particle = (GlobalParticle*)Swarm_ParticleAt( swarm, lParticle_I );
GlobalParticleLayout_InitialiseParticle( self, swarm, newParticle_I, particle );
/* Work out which cell the new particle is in */
/* First specify the particle doesn't have an owning cell yet, so as
not to confuse the search algorithm if its an irregular cell/mesh layout */
particle->owningCell = swarm->cellDomainCount;
cell_I = CellLayout_CellOf( swarm->cellLayout, particle );
/* If we found a further particle inside our domain, add it to a cell */
if ( cell_I < swarm->cellLocalCount ) {
Journal_DPrintfL( self->debug, 3, "global particle %u at (%.2f,%.2f,%.2f) inside local cell %u\n"
"adding it to cell and saving it as local particle %u.\n",
newParticle_I, particle->coord[0], particle->coord[1], particle->coord[2],
cell_I, lParticle_I );
Stream_IndentBranch( Swarm_Debug );
/* Add it to that cell */
Swarm_AddParticleToCell( swarm, cell_I, lParticle_I );
lParticle_I++;
swarm->particleLocalCount++;
Swarm_Realloc( swarm );
Stream_UnIndentBranch( Swarm_Debug );
}
else {
Journal_DPrintfL( self->debug, 4, "global particle %u at (%.2f,%.2f,%.2f) outside this proc's domain:\n"
"ignoring.\n", newParticle_I, particle->coord[0], particle->coord[1], particle->coord[2] );
}
newParticle_I++;
}
Stream_UnIndentBranch( Swarm_Debug );
/* Do a test to make sure that the total particles assigned across all processors ==
totalInitialParticles count */
MPI_Allreduce( &swarm->particleLocalCount, &globalParticlesInitialisedCount, 1, MPI_UNSIGNED, MPI_SUM, swarm->comm );
Journal_Firewall( globalParticlesInitialisedCount == self->totalInitialParticles, errorStream,
"Error - in %s() - for GlobalParticleLayout \"%s\", of type %s: after initialising particles, "
"actual global count of particles initialised was %u, whereas requested global total "
"totalInitialParticles was %u. If actual is < requested, it means some particles were not "
"identified by any processor as inside their domain. If actual > requested, it means that "
"some particles were identified by _multiple_ processors as belonging to their domain. Both "
"these states are erroneous.\n",
__func__, self->name, self->type, globalParticlesInitialisedCount, self->totalInitialParticles );
Stream_UnIndentBranch( Swarm_Debug );
}