PyObject* GeneralSwarm_AddParticlesFromCoordArray( void* swarm, Index count, Index dim, double* array )
{
GeneralSwarm* self = (GeneralSwarm*)swarm;
unsigned* cellArray = Memory_Alloc_Array( unsigned, count, "GeneralSwarm_AddParticlesFromCoordArray_CellArray" );
GlobalParticle localParticle;
GlobalParticle* particle = &localParticle;
int cellLocalCount = self->cellLocalCount;
int oldParticleCount = self->particleLocalCount;
int ii;
int totsLocalParticles=0;
// find which particles are local. we do this to avoid swarm reallocs.
for (ii=0; ii<count; ii++) {
memcpy(&(particle->coord), array + dim*ii, dim*sizeof(double));
cellArray[ii] = CellLayout_CellOf( self->cellLayout, particle );
if( cellArray[ii] < cellLocalCount )
totsLocalParticles++;
}
// alloc particle local index array (to be returned)
int* partLocalIndex = Memory_Alloc_Array( int, count, "GeneralSwarm_AddParticlesFromCoordArray_CellArray" );
// ok, lets add them to the swarm, now that we know how many are required
self->particleLocalCount += totsLocalParticles;
Swarm_Realloc( self );
int newPartIndex = oldParticleCount;
for (ii=0; ii<count; ii++) {
if( cellArray[ii] < cellLocalCount ){
particle = (GlobalParticle*)Swarm_ParticleAt( self, newPartIndex );
memcpy(&(particle->coord), array + dim*ii, dim*sizeof(double));
Swarm_AddParticleToCell( swarm, cellArray[ii], newPartIndex );
partLocalIndex[ii] = newPartIndex;
newPartIndex++;
} else {
partLocalIndex[ii] = -1;
}
}
/* create numpy array to return */
npy_intp dims[1] = { count };
PyObject* pyobj = PyArray_New(&PyArray_Type, 1, dims, NPY_INT, NULL, (void*)partLocalIndex, sizeof(int), 0, NULL);
/* enable the owndata flag.. this tells numpy to dealloc the data when it is finished with it */
#if NPY_API_VERSION < 0x00000007
(((PyArrayObject*)pyobj)->flags) = NPY_ARRAY_OWNDATA;
#else
PyArray_ENABLEFLAGS((PyArrayObject*)pyobj, NPY_ARRAY_OWNDATA);
#endif
return pyobj;
}
int GeneralSwarm_AddParticle( void* swarm, Index dim, double xI, double xJ, double xK )
{
GeneralSwarm* self = (GeneralSwarm*)swarm;
GlobalParticle localParticle;
GlobalParticle* particle = &localParticle;
unsigned cell;
int newPartIndex;
/* If the location is not local, return immediately with index value -1 */
particle->coord[ I_AXIS ] = xI;
particle->coord[ J_AXIS ] = xJ;
if(dim == 3)
particle->coord[ K_AXIS ] = xK;
cell = CellLayout_CellOf( self->cellLayout, particle );
if( cell >= self->cellLocalCount )
return (-1);
/* Otherwise we allocate the particle to the Swarm and return the new index */
newPartIndex = self->particleLocalCount;
self->particleLocalCount ++;
Swarm_Realloc( self );
particle = (GlobalParticle*)Swarm_ParticleAt( self, newPartIndex );
particle->coord[ I_AXIS ] = xI;
particle->coord[ J_AXIS ] = xJ;
if(dim==3)
particle->coord[ K_AXIS ] = xK;
Swarm_AddParticleToCell( self, cell, newPartIndex );
return(newPartIndex);
}
/* TODO: look at using MPI_Indexed instead */
void ParticleMovementHandler_FinishReceiveAndUpdateShadowParticlesEnteringMyDomain( ParticleCommHandler* self ) {
MPI_Status status;
Cell_ShadowTransferIndex stCell_I;
Cell_LocalIndex lCell_I;
Neighbour_Index nbr_I;
Cell_ShadowTransferIndex shadowCellsFromProcCount;
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Neighbour_Index nbrCount = procNbrInfo->procNbrCnt;
Particle_InCellIndex incomingCellParticleCount;
Particle_InCellIndex cParticle_I;
Particle_Index lParticle_I;
Index incomingParticle_I=0; /*Index into the array of all leaving particle indices */
Index incomingParticleSetsNotYetReceivedCount;
Bool* incomingParticlesReceived;
#if DEBUG
GlobalParticle* currParticle;
#endif
Journal_DPrintf( self->debug, "In %s():\n", __func__ );
Stream_IndentBranch( Swarm_Debug );
incomingParticlesReceived = Memory_Alloc_Array_Unnamed( Bool, nbrCount );
/* Calculate how many particle sets we have to receive */
incomingParticleSetsNotYetReceivedCount = 0;
for ( nbr_I=0; nbr_I < nbrCount; nbr_I++ ) {
incomingParticlesReceived[nbr_I] = False;
if (self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] > 0) {
incomingParticleSetsNotYetReceivedCount++;
}
}
while ( incomingParticleSetsNotYetReceivedCount > 0 ) {
int flag = 0;
Journal_DPrintfL( self->debug, 3, "%d particle sets still to go...\n", incomingParticleSetsNotYetReceivedCount );
for ( nbr_I=0; nbr_I < nbrCount; nbr_I++ ) {
if ( (self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] > 0) &&
(False == incomingParticlesReceived[nbr_I]) )
{
MPI_Test( self->particlesArrivingFromNbrShadowCellsHandles[nbr_I], &flag, &status );
if ( False == flag ) {
/* No results yet from this proc -> continue to next. */
continue;
}
else {
Journal_DPrintfL( self->debug, 3, "Received particles from nbr %d (proc %d):\n",
nbr_I, procNbrInfo->procNbrTbl[nbr_I] );
Stream_Indent( self->debug );
incomingParticle_I = 0;
shadowCellsFromProcCount = cellShadowInfo->procShadowedCnt[nbr_I];
for ( stCell_I=0; stCell_I < shadowCellsFromProcCount; stCell_I++ ) {
lCell_I = cellShadowInfo->procShadowedTbl[nbr_I][stCell_I];
Journal_DPrintfL( self->debug, 3, "Incoming cell %d (local index %d):\n",
stCell_I, lCell_I );
Stream_Indent( self->debug );
incomingCellParticleCount =
self->particlesArrivingFromNbrShadowCellCounts[nbr_I][stCell_I];
for ( cParticle_I=0; cParticle_I < incomingCellParticleCount; cParticle_I++ ) {
#if DEBUG
currParticle = (GlobalParticle*)ParticleAt(
self->particlesArrivingFromNbrShadowCells[nbr_I],
incomingParticle_I,
self->swarm->particleExtensionMgr->finalSize );
Journal_DPrintfL( self->debug, 3, "Handling its PIC %d: - at "
"(%.2f,%.2f,%.2f)\n", cParticle_I,
currParticle->coord[0], currParticle->coord[1],
currParticle->coord[2] );
#endif
Stream_Indent( self->debug );
lParticle_I = ParticleMovementHandler_FindFreeSlotAndPrepareForInsertion( self );
Swarm_CopyParticleOntoSwarm(
self->swarm,
lParticle_I,
self->particlesArrivingFromNbrShadowCells[nbr_I], incomingParticle_I++ );
Swarm_AddParticleToCell( self->swarm, lCell_I, lParticle_I );
Stream_UnIndent( self->debug );
}
Stream_UnIndent( self->debug );
}
incomingParticlesReceived[nbr_I] = True;
incomingParticleSetsNotYetReceivedCount--;
Stream_UnIndent( self->debug );
}
}
}
}
Memory_Free( incomingParticlesReceived );
Stream_UnIndentBranch( Swarm_Debug );
}
void ParticleMovementHandler_ShareAndUpdateParticlesThatHaveMovedOutsideDomains(
ParticleMovementHandler* self,
Particle_Index* globalParticlesArrivingMyDomainCountPtr,
Particle_Index* globalParticlesOutsideDomainTotalPtr )
{
Particle_Index* globalParticlesOutsideDomainCounts = NULL;
Particle_Index maxGlobalParticlesOutsideDomainCount = 0;
Processor_Index proc_I = 0;
Particle_Index lParticle_I = 0;
Particle_Index particle_I = 0;
Journal_DPrintfL( self->debug, 2, "In %s():\n", __func__ );
Stream_IndentBranch( Swarm_Debug );
(*globalParticlesArrivingMyDomainCountPtr) = 0;
(*globalParticlesOutsideDomainTotalPtr) = 0;
/* Find the counts of particles outside domain... */
ParticleMovementHandler_GetCountOfParticlesOutsideDomainPerProcessor(
self,
&globalParticlesOutsideDomainCounts,
&maxGlobalParticlesOutsideDomainCount,
globalParticlesOutsideDomainTotalPtr );
if ( (*globalParticlesOutsideDomainTotalPtr) > 0 ) {
Particle* particlesLeavingMyDomain = NULL;
Particle* globalParticlesLeavingDomains = NULL;
SizeT particlesLeavingDomainSizeBytes = 0;
Cell_DomainIndex lCell_I = 0;
GlobalParticle* currParticle = NULL;
Particle_Index currProcParticlesOutsideDomainCount = 0;
Particle_Index currProcOffset = 0;
particlesLeavingDomainSizeBytes = self->swarm->particleExtensionMgr->finalSize
* maxGlobalParticlesOutsideDomainCount;
particlesLeavingMyDomain = Memory_Alloc_Bytes( particlesLeavingDomainSizeBytes, "Particle",
"particlesLeavingMyDomain" );
// TODO: investigate doing this with an MPI_Indexed datatype instead...
Journal_DPrintfL( self->debug, 2, "Copying particles leaving my domain to temp. transfer array\n" );
Stream_IndentBranch( Swarm_Debug );
#if 0
MPI_Type_indexed(
self->particlesOutsideDomainTotalCount,
blocklens,
self->particlesOutsideDomainIndices,//change to contiguous indices?
MPI_BYTE,
ParticlesLeavingDomainTransferIndexed
);
#endif
for ( particle_I=0; particle_I < self->particlesOutsideDomainTotalCount; particle_I++ ) {
Journal_DPrintfL( self->debug, 3, "Copying particle %d to particlesLeavingMyDomain[%d]\n",
self->particlesOutsideDomainIndices[particle_I], particle_I );
Swarm_CopyParticleOffSwarm( self->swarm,
particlesLeavingMyDomain, particle_I,
self->particlesOutsideDomainIndices[particle_I] );
}
Stream_UnIndentBranch( Swarm_Debug );
/* allocate the big global receive buffer */
globalParticlesLeavingDomains = Memory_Alloc_Bytes( particlesLeavingDomainSizeBytes * self->swarm->nProc,
"Particle", "globalParticlesLeavingDomains" );
Journal_DPrintfL( self->debug, 2, "Getting the global array of particles leaving domains\n" );
(void)MPI_Allgather( particlesLeavingMyDomain, particlesLeavingDomainSizeBytes, MPI_BYTE,
globalParticlesLeavingDomains, particlesLeavingDomainSizeBytes, MPI_BYTE,
self->swarm->comm );
Journal_DPrintfL( self->debug, 2, "Checking through the global array of particles leaving domains, "
"and snaffling those moving into my domain:\n" );
Stream_IndentBranch( Swarm_Debug );
for ( proc_I=0; proc_I < self->swarm->nProc; proc_I++ ) {
if ( proc_I == self->swarm->myRank ) continue;
currProcOffset = proc_I * maxGlobalParticlesOutsideDomainCount;
currProcParticlesOutsideDomainCount = globalParticlesOutsideDomainCounts[proc_I];
Journal_DPrintfL( self->debug, 3, "Checking particles that left proc. %d:\n", proc_I );
for ( particle_I=0; particle_I < currProcParticlesOutsideDomainCount; particle_I++ ) {
currParticle = (GlobalParticle*)ParticleAt( globalParticlesLeavingDomains,
(currProcOffset + particle_I),
self->swarm->particleExtensionMgr->finalSize );
lCell_I = CellLayout_CellOf( self->swarm->cellLayout, currParticle );
if ( lCell_I < self->swarm->cellLocalCount ) {
#if DEBUG
Journal_DPrintfL( self->debug, 3, "Found particle at (%.2f,%.2f,%.2f) that's moved "
"into my local cell %d...\n", currParticle->coord[0],
currParticle->coord[1], currParticle->coord[2], lCell_I );
#endif
/* copy particle to the lowest available slot in my particles array */
lParticle_I = ParticleMovementHandler_FindFreeSlotAndPrepareForInsertion( (ParticleCommHandler*)self );
Swarm_CopyParticleOntoSwarm( self->swarm, lParticle_I,
globalParticlesLeavingDomains, (currProcOffset + particle_I) );
Swarm_AddParticleToCell( self->swarm, lCell_I, lParticle_I );
(*globalParticlesArrivingMyDomainCountPtr)++;
}
#if DEBUG
else {
currParticle = (GlobalParticle*)ParticleAt( globalParticlesLeavingDomains,
(currProcOffset + particle_I),
self->swarm->particleExtensionMgr->finalSize );
Journal_DPrintfL( self->debug, 3, "Ignoring particle at (%.2f,%.2f,%.2f) since "
"not in my local cells...\n", currParticle->coord[0],
currParticle->coord[1], currParticle->coord[2] );
}
#endif
}
}
Stream_UnIndentBranch( Swarm_Debug );
Memory_Free( particlesLeavingMyDomain );
Memory_Free( globalParticlesLeavingDomains );
/* Defensive check to make sure particles not lost/created accidentally somehow */
if( self->defensive == True ) {
ParticleMovementHandler_EnsureParticleCountLeavingDomainsEqualsCountEnteringGlobally( self );
}
}
Memory_Free( globalParticlesOutsideDomainCounts );
Stream_UnIndentBranch( Swarm_Debug );
}
void _GlobalParticleLayout_InitialiseParticles( void* particleLayout, void* _swarm )
{
GlobalParticleLayout* self = (GlobalParticleLayout*)particleLayout;
Swarm* swarm = (Swarm*)_swarm;
GlobalParticle* particle = NULL;
Particle_Index lParticle_I=0;
Particle_Index newParticle_I=0;
Cell_Index cell_I;
Particle_Index globalParticlesInitialisedCount=0;
Stream* errorStream = Journal_Register( Error_Type, self->type );
Journal_DPrintf( self->debug, "In %s(): for ParticleLayout \"%s\" (of type %s):\n",
__func__, self->name, self->type );
Stream_IndentBranch( Swarm_Debug );
Journal_DPrintf( self->debug, "For each of the %u total global requested particles, "
"generating a particle, and checking if it's in this processor's domain. If so, "
"adding it to the appropriate local cell.\n", self->totalInitialParticles );
Stream_IndentBranch( Swarm_Debug );
while( newParticle_I < self->totalInitialParticles ) {
particle = (GlobalParticle*)Swarm_ParticleAt( swarm, lParticle_I );
GlobalParticleLayout_InitialiseParticle( self, swarm, newParticle_I, particle );
/* Work out which cell the new particle is in */
/* First specify the particle doesn't have an owning cell yet, so as
not to confuse the search algorithm if its an irregular cell/mesh layout */
particle->owningCell = swarm->cellDomainCount;
cell_I = CellLayout_CellOf( swarm->cellLayout, particle );
/* If we found a further particle inside our domain, add it to a cell */
if ( cell_I < swarm->cellLocalCount ) {
Journal_DPrintfL( self->debug, 3, "global particle %u at (%.2f,%.2f,%.2f) inside local cell %u\n"
"adding it to cell and saving it as local particle %u.\n",
newParticle_I, particle->coord[0], particle->coord[1], particle->coord[2],
cell_I, lParticle_I );
Stream_IndentBranch( Swarm_Debug );
/* Add it to that cell */
Swarm_AddParticleToCell( swarm, cell_I, lParticle_I );
lParticle_I++;
swarm->particleLocalCount++;
Swarm_Realloc( swarm );
Stream_UnIndentBranch( Swarm_Debug );
}
else {
Journal_DPrintfL( self->debug, 4, "global particle %u at (%.2f,%.2f,%.2f) outside this proc's domain:\n"
"ignoring.\n", newParticle_I, particle->coord[0], particle->coord[1], particle->coord[2] );
}
newParticle_I++;
}
Stream_UnIndentBranch( Swarm_Debug );
/* Do a test to make sure that the total particles assigned across all processors ==
totalInitialParticles count */
MPI_Allreduce( &swarm->particleLocalCount, &globalParticlesInitialisedCount, 1, MPI_UNSIGNED, MPI_SUM, swarm->comm );
Journal_Firewall( globalParticlesInitialisedCount == self->totalInitialParticles, errorStream,
"Error - in %s() - for GlobalParticleLayout \"%s\", of type %s: after initialising particles, "
"actual global count of particles initialised was %u, whereas requested global total "
"totalInitialParticles was %u. If actual is < requested, it means some particles were not "
"identified by any processor as inside their domain. If actual > requested, it means that "
"some particles were identified by _multiple_ processors as belonging to their domain. Both "
"these states are erroneous.\n",
__func__, self->name, self->type, globalParticlesInitialisedCount, self->totalInitialParticles );
Stream_UnIndentBranch( Swarm_Debug );
}
void _GeneralSwarm_Initialise( void* swarm, void* data )
{
GeneralSwarm* self = (GeneralSwarm*) swarm;
AbstractContext* context = (AbstractContext*)self->context;
Index var_I = 0;
_Swarm_Initialise( self, data );
if( self->escapedRoutine != NULL) Stg_Component_Initialise( self->escapedRoutine, data , False );
for( var_I = 0 ; var_I < self->nSwarmVars ; var_I++ )
{
Stg_Component_Initialise( self->swarmVars[var_I], data , False );
}
/** if loading from checkpoint, particle materials etc have already been loaded in Swarm_Build() - */
/** possibly need to check for empty cells (and populate) if performing a interpolation restart */
if ( context && True == context->loadFromCheckPoint )
{
if ( (True == self->isSwarmTypeToCheckPointAndReload) && (True == context->interpolateRestart) )
{
Particle_InCellIndex cParticle_I = 0;
Particle_InCellIndex particle_I = 0;
GlobalParticle* particle = NULL;
double minDistance = HUGE_VAL;
double distanceToParticle;
Dimension_Index dim = self->dim;
Cell_DomainIndex dCell_I;
Cell_LocalIndex lCell_I;
Cell_DomainIndex belongsToCell_I = 0;
GlobalParticle* matNewParticle;
GlobalParticle* matParticleToSplit;
Particle_Index matNewParticle_IndexOnCPU;
Coord xi;
Coord coord;
unsigned nEmptyCells = 0;
Index* cellID = NULL;
Index* particleCPUID = NULL;
unsigned count;
unsigned ii;
/** first determine how many local cells are empty */
for( lCell_I = 0 ; lCell_I < self->cellLocalCount ; lCell_I++ )
if (self->cellParticleCountTbl[lCell_I] == 0) nEmptyCells++;
/** create arrays which will be later used to populate cells */
cellID = Memory_Alloc_Array( Index, nEmptyCells, "Cell ID for cell to be populated" );
particleCPUID = Memory_Alloc_Array( Index, nEmptyCells, "particle ID for particle to populate cell" );
count = 0;
for( lCell_I = 0 ; lCell_I < self->cellLocalCount ; lCell_I++ )
{
minDistance = HUGE_VAL;
if (self->cellParticleCountTbl[lCell_I] == 0)
{
/** select the centre of the current cell */
xi[0] = 0;
xi[1] = 0;
xi[2] = 0;
Journal_Firewall(
Stg_Class_IsInstance( self->cellLayout, ElementCellLayout_Type ),
NULL,
"Error In func %s: When performing interpolation restart, cellLayout must be of type ElementCellLayout.",
__func__ );
/** get global coord */
FeMesh_CoordLocalToGlobal( ((ElementCellLayout*)self->cellLayout)->mesh, lCell_I, xi, coord );
for( dCell_I = 0 ; dCell_I < self->cellDomainCount ; dCell_I++ )
{
/** Loop over particles find closest to cell centre */
for( cParticle_I = 0 ; cParticle_I < self->cellParticleCountTbl[dCell_I] ; cParticle_I++ )
{
particle = (GlobalParticle*)Swarm_ParticleInCellAt( self, dCell_I, cParticle_I );
/** Calculate distance to particle */
distanceToParticle =
(particle->coord[ I_AXIS ] - coord[ I_AXIS ]) *
(particle->coord[ I_AXIS ] - coord[ I_AXIS ]) +
(particle->coord[ J_AXIS ] - coord[ J_AXIS ]) *
(particle->coord[ J_AXIS ] - coord[ J_AXIS ]) ;
if (dim == 3)
{
distanceToParticle +=
(particle->coord[ K_AXIS ] - coord[ K_AXIS ]) *
(particle->coord[ K_AXIS ] - coord[ K_AXIS ]) ;
}
/** Don't do square root here because it is unnessesary: i.e. a < b <=> sqrt(a) < sqrt(b) */
/** Check if this is the closest particle */
if (minDistance > distanceToParticle)
{
particle_I = cParticle_I;
minDistance = distanceToParticle;
belongsToCell_I = dCell_I;
}
}
}
/** create new particle which will be placed at centre of empty cell */
matNewParticle = (GlobalParticle*) Swarm_CreateNewParticle( self, &matNewParticle_IndexOnCPU );
/** grab closest particle, which we will copy */
matParticleToSplit = (GlobalParticle*) Swarm_ParticleInCellAt( self, belongsToCell_I, particle_I );
/** copy - even though self->particleExtensionMgr->finalSize maybe biger than sizeof(GlobalParticle)
the addressing of the copy is the important part
*/
memcpy( matNewParticle, matParticleToSplit, self->particleExtensionMgr->finalSize );
/** set the owningCell to the cellDomainCount so that its owningCell will be reinitialised (not sure if necessary) */
matNewParticle->owningCell = self->cellDomainCount;
/** Copy new global position (cell centre) to coord on new mat particle */
memcpy( matNewParticle->coord, coord, sizeof(Coord) );
/** we now store the required information to populate empty cells */
/** note that cells are not populated at this point, as this may interfere
with the 0th order interpolation we are performing */
cellID[count] = lCell_I;
particleCPUID[count] = matNewParticle_IndexOnCPU;
count++;
}
}
/** populate empty cells */
for(ii = 0 ; ii < count ; ii++)
Swarm_AddParticleToCell( self, cellID[ii], particleCPUID[ii] );
Memory_Free( cellID );
Memory_Free( particleCPUID );
}
/* TODO: print info / debug message */
}
}
