void _ParticleShadowSync_FinishReceiveOfIncomingParticleCounts( ParticleCommHandler* self ) {
MPI_Status status;
Processor_Index proc_I;
ShadowInfo* cellShadowInfo = CellLayout_GetShadowInfo( self->swarm->cellLayout );
ProcNbrInfo* procNbrInfo = cellShadowInfo->procNbrInfo;
Neighbour_Index nbr_I;
int i = 0;
self->swarm->shadowParticleCount = 0;
/* TODO: may be worth converting the below into an MPI_Test loop */
for ( nbr_I=0; nbr_I < procNbrInfo->procNbrCnt; nbr_I++ ) {
proc_I = procNbrInfo->procNbrTbl[nbr_I];
(void)MPI_Wait( self->particlesArrivingFromNbrShadowCellCountsHandles[nbr_I], &status );
self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] = 0;
for( i=0; i<cellShadowInfo->procShadowCnt[nbr_I]; i++ ){
Index shadowCell = 0;
shadowCell = cellShadowInfo->procShadowTbl[nbr_I][i];
self->swarm->shadowCellParticleCountTbl[shadowCell-self->swarm->cellLocalCount] = 0;
self->swarm->shadowCellParticleTbl[shadowCell] = Memory_Realloc_Array( self->swarm->shadowCellParticleTbl[shadowCell], Particle_Index,
(self->particlesArrivingFromNbrShadowCellCounts[nbr_I][i]) );
self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I] += self->particlesArrivingFromNbrShadowCellCounts[nbr_I][i];
}
self->swarm->shadowParticleCount+= self->particlesArrivingFromNbrShadowCellsTotalCounts[nbr_I];
}
}
void _Stg_ObjectList_AllocMoreMemory( void* namedObjectList ) {
Stg_ObjectList* self = (Stg_ObjectList*) namedObjectList;
self->_size += self->_delta;
self->data = Memory_Realloc_Array( self->data, Stg_ObjectPtr, self->_size );
assert( self->data );
}
void ParticleMovementHandler_FindParticlesThatHaveMovedOutsideMyDomain( ParticleMovementHandler* self )
{
Particle_Index particlesOutsideDomainSize = 0;
GlobalParticle* currParticle = NULL;
Particle_Index lParticle_I = 0;
Journal_DPrintfL( self->debug, 1, "In %s():\n", __func__ );
Stream_IndentBranch( Swarm_Debug );
self->particlesOutsideDomainTotalCount = 0;
self->currParticleLeavingMyDomainIndex = 0;
particlesOutsideDomainSize = self->swarm->particlesArrayDelta;
Journal_DPrintfL( self->debug, 1, "Checking the owning cell of each of my swarm's %d particles:\n",
self->swarm->particleLocalCount );
Stream_IndentBranch( Swarm_Debug );
for ( lParticle_I=0; lParticle_I < self->swarm->particleLocalCount; lParticle_I++ ) {
currParticle = (GlobalParticle*)Swarm_ParticleAt( self->swarm, lParticle_I );
if ( currParticle->owningCell == self->swarm->cellDomainCount ) {
Journal_DPrintfL( self->debug, 3, "particle %d has moved outside domain to (%.2f,%.2f,%.2f): "
"saving index\n", lParticle_I, currParticle->coord[0], currParticle->coord[1],
currParticle->coord[2] );
if ( self->particlesOutsideDomainTotalCount == particlesOutsideDomainSize ) {
particlesOutsideDomainSize += self->swarm->particlesArrayDelta;
Journal_DPrintfL( self->debug, 3, "(Need more memory to save indexes: increasing from %d to %d.)\n",
self->particlesOutsideDomainTotalCount, particlesOutsideDomainSize );
self->particlesOutsideDomainIndices = Memory_Realloc_Array( self->particlesOutsideDomainIndices,
Particle_Index, particlesOutsideDomainSize );
}
self->particlesOutsideDomainIndices[self->particlesOutsideDomainTotalCount++] = lParticle_I;
}
}
Stream_UnIndentBranch( Swarm_Debug );
self->particlesOutsideDomainUnfilledCount = self->particlesOutsideDomainTotalCount;
#if DEBUG
{
Particle_Index particle_I = 0;
if ( Stream_IsPrintableLevel( self->debug, 2 ) ) {
Journal_DPrintf( self->debug, "%d Particles have moved outside my domain:\n\t[",
self->particlesOutsideDomainTotalCount );
for ( ; particle_I < self->particlesOutsideDomainTotalCount; particle_I++ ) {
Journal_DPrintf( self->debug, "%d, ", self->particlesOutsideDomainIndices[particle_I] );
}
Journal_DPrintf( self->debug, "]\n" );
}
}
#endif
Stream_UnIndentBranch( Swarm_Debug );
}
void PeriodicBoundariesManager_AddPeriodicBoundary( void* periodicBCsManager, Axis axis ) {
PeriodicBoundariesManager* self = (PeriodicBoundariesManager*)periodicBCsManager;
PeriodicBoundary* newPeriodicBoundary;
double min[3], max[3];
Mesh_GetGlobalCoordRange( self->mesh, min, max );
if ( self->count == self->size ) {
self->size += self->delta;
self->boundaries = Memory_Realloc_Array( self->boundaries, PeriodicBoundary, self->size );
}
newPeriodicBoundary = &self->boundaries[self->count];
newPeriodicBoundary->axis = axis;
newPeriodicBoundary->minWall = min[axis];
newPeriodicBoundary->maxWall = max[axis];
newPeriodicBoundary->particlesUpdatedMinEndCount = 0;
newPeriodicBoundary->particlesUpdatedMaxEndCount = 0;
self->count++;
}
void CreateTriangle(lucIsosurface* self, Vertex* point1, Vertex* point2, Vertex* point3, Bool wall)
{
Vertex* points[3] = {point1, point2, point3};
int i;
if (self->triangleCount >= self->trianglesAlloced)
{
self->trianglesAlloced += 100;
self->triangleList = Memory_Realloc_Array( self->triangleList, Surface_Triangle, self->trianglesAlloced );
}
/* Wall flag */
self->triangleList[self->triangleCount].wall = wall;
for (i=0; i<3; i++)
{
double value = 0;
float *pos = self->triangleList[self->triangleCount].pos[i];
if (self->sampleGlobal)
{
/* Check maskField values */
if ( self->maskField )
{
double mvalue;
FieldVariable_InterpolateValueAt( self->maskField, points[i]->pos, &mvalue);
/* Don't bother calculating if vertex is masked */
if (lucDrawingObjectMask_Test( &self->mask, mvalue ) == False) return;
}
/* Get global coordinates */
pos[0] = points[i]->pos[0];
pos[1] = points[i]->pos[1];
pos[2] = points[i]->pos[2];
if (self->isosurfaceField->dim == 2) pos[2] = 0;
/* Get colourField value */
if ( self->colourField && self->colourMap)
FieldVariable_InterpolateValueAt( self->colourField, points[i]->pos, &value);
}
else
{
FeVariable* field = (FeVariable*)self->isosurfaceField;
double dpos[3] = {0};
/* Check maskField values */
if ( self->maskField )
{
double mvalue;
FeVariable_InterpolateWithinElement( self->maskField, points[i]->element_I, points[i]->pos, &mvalue);
/* Don't bother calculating if vertex is masked */
if (lucDrawingObjectMask_Test( &self->mask, mvalue ) == False) return;
}
/* Get global coordinates */
FeMesh_CoordLocalToGlobal( field->feMesh, points[i]->element_I, points[i]->pos, dpos );
pos[0] = dpos[0];
pos[1] = dpos[1];
pos[2] = dpos[2];
/* Get colourField value */
if ( self->colourField && self->colourMap)
FeVariable_InterpolateWithinElement( self->colourField, points[i]->element_I, points[i]->pos, &value);
}
/* Copy value */
self->triangleList[self->triangleCount].value[i] = value;
}
self->triangleCount++;
}
void SolutionVector_UpdateSolutionOntoNodes( void* solutionVector ) {
SolutionVector* self = (SolutionVector *)solutionVector;
double* localSolnVecValues;
Node_LocalIndex lNode_I = 0;
Dof_Index currNodeNumDofs;
Dof_Index nodeLocalDof_I;
Partition_Index ownerProc;
FeVariable* feVar = self->feVariable;
FeMesh* feMesh = feVar->feMesh;
MPI_Comm mpiComm;
FeEquationNumber* eqNum = self->eqNum;
Dof_EquationNumber currEqNum;
Index indexIntoLocalSolnVecValues;
Index* reqFromOthersCounts;
Index* reqFromOthersSizes;
RequestInfo** reqFromOthersInfos;
Dof_EquationNumber** reqFromOthers;
Comm* comm;
Partition_Index nProc;
Partition_Index myRank;
Partition_Index proc_I;
double initialGuessAtNonLocalEqNumsRatio = 0.1;
double ratioToIncreaseRequestArraySize = 1.5;
Index newReqFromOthersSize;
Journal_DPrintf( self->debug, "In %s - for \"%s\"\n", __func__, self->name );
Stream_IndentBranch( StgFEM_Debug );
#if DEBUG
if ( Stream_IsPrintableLevel( self->debug, 3 ) ) {
Journal_DPrintf( self->debug, "Vector data:\n" );
_SolutionVector_VectorView( self->vector, self->debug );
}
#endif
comm = Mesh_GetCommTopology( feMesh, MT_VERTEX );
mpiComm = Comm_GetMPIComm( comm );
MPI_Comm_size( mpiComm, (int*)&nProc );
MPI_Comm_rank( mpiComm, (int*)&myRank );
/* allocate arrays for nodes that I want on each processor */
reqFromOthersCounts = Memory_Alloc_Array( Index, nProc, "reqFromOthersCounts" );
reqFromOthersSizes = Memory_Alloc_Array( Index, nProc, "reqFromOthersSizes" );
reqFromOthersInfos = Memory_Alloc_Array( RequestInfo*, nProc, "reqFromOthersInfos" );
reqFromOthers = Memory_Alloc_Array( Dof_EquationNumber*, nProc, "reqFromOthers" );
/* Allocate the arrays of req. values from others independently, as we don't know how large they'll be */
for ( proc_I=0; proc_I < nProc; proc_I++ ) {
reqFromOthersCounts[proc_I] = 0;
if (proc_I == myRank) continue;
/* Our initial guess at number of non-local eqNums is a small ratio of the number of local dofs */
reqFromOthersSizes[proc_I] = eqNum->localEqNumsOwnedCount * initialGuessAtNonLocalEqNumsRatio;
/* Special case for really small meshes: make sure it's at least 1 */
if (0 == reqFromOthersSizes[proc_I] ) {
reqFromOthersSizes[proc_I]++;
}
reqFromOthersInfos[proc_I] = Memory_Alloc_Array( RequestInfo, reqFromOthersSizes[proc_I],
"reqFromOthersInfos[proc_I]" );
reqFromOthers[proc_I] = Memory_Alloc_Array( Dof_EquationNumber, reqFromOthersSizes[proc_I],
"reqFromOthers[proc_I]" );
}
/* Get the locally held part of the vector */
//Vector_GetArray( self->vector, &localSolnVecValues );
VecGetArray( self->vector, &localSolnVecValues );
for( lNode_I=0; lNode_I < Mesh_GetLocalSize( feMesh, MT_VERTEX ); lNode_I++ ) {
currNodeNumDofs = feVar->dofLayout->dofCounts[ lNode_I ];
Journal_DPrintfL( self->debug, 3, "getting solutions for local node %d, has %d dofs.\n", lNode_I, currNodeNumDofs );
/* process each dof */
for ( nodeLocalDof_I = 0; nodeLocalDof_I < currNodeNumDofs; nodeLocalDof_I++ ) {
Journal_DPrintfL( self->debug, 3, "\tdof %d: ", nodeLocalDof_I );
currEqNum = eqNum->mapNodeDof2Eq[lNode_I][nodeLocalDof_I];
if( currEqNum != -1 ) {
Journal_DPrintfL( self->debug, 3, "is unconstrained, eqNum %d:", currEqNum );
if( STreeMap_HasKey( eqNum->ownedMap, &currEqNum ) ) {
indexIntoLocalSolnVecValues = *(int*)STreeMap_Map( eqNum->ownedMap, &currEqNum );
Journal_DPrintfL( self->debug, 3, "local -> just copying value %f\n",
localSolnVecValues[indexIntoLocalSolnVecValues] );
DofLayout_SetValueDouble( feVar->dofLayout, lNode_I, nodeLocalDof_I,
localSolnVecValues[indexIntoLocalSolnVecValues] );
}
else {
RequestInfo* requestInfo;
Journal_DPrintfL( self->debug, 3, "nonlocal -> add to req list " );
ownerProc = FeEquationNumber_CalculateOwningProcessorOfEqNum( eqNum, currEqNum );
Journal_DPrintfL( self->debug, 3, "from proc %d\n", ownerProc );
/* first check count & realloc if necessary */
if (reqFromOthersCounts[ownerProc] == reqFromOthersSizes[ownerProc] ) {
newReqFromOthersSize = reqFromOthersSizes[ownerProc] * ratioToIncreaseRequestArraySize;
if ( newReqFromOthersSize == reqFromOthersSizes[ownerProc] ) {
/* Special case: always increase by at least 1 */
newReqFromOthersSize++;
}
reqFromOthersSizes[ownerProc] = newReqFromOthersSize;
Journal_DPrintfL( self->debug, 3, "req list from proc %d count %d now "
"equal to size, so reallocing to size %d\n",
ownerProc, reqFromOthersCounts[ownerProc],
reqFromOthersSizes[ownerProc] );
reqFromOthersInfos[ownerProc] = Memory_Realloc_Array(
reqFromOthersInfos[ownerProc], RequestInfo, reqFromOthersSizes[ownerProc] );
reqFromOthers[ownerProc] = Memory_Realloc_Array(
reqFromOthers[ownerProc], Dof_EquationNumber, reqFromOthersSizes[ownerProc] );
}
requestInfo = &reqFromOthersInfos[ownerProc][ reqFromOthersCounts[ownerProc] ];
requestInfo->lNode_I = lNode_I;
requestInfo->nodeLocalDof_I = nodeLocalDof_I;
reqFromOthers[ownerProc][reqFromOthersCounts[ownerProc]] = currEqNum;
(reqFromOthersCounts[ownerProc])++;
}
}
else {
Journal_DPrintfL( self->debug, 3, "is a BC, so skipping...\n" );
}
}
}
if ( nProc > 1 ) {
_SolutionVector_ShareValuesNotStoredLocally( self, reqFromOthersCounts, reqFromOthersInfos, reqFromOthers,
localSolnVecValues );
}
for ( proc_I=0; proc_I < nProc; proc_I++ ) {
if (proc_I == myRank) continue;
Memory_Free( reqFromOthers[proc_I] );
Memory_Free( reqFromOthersInfos[proc_I] );
}
Memory_Free( reqFromOthers );
Memory_Free( reqFromOthersInfos );
Memory_Free( reqFromOthersCounts );
Memory_Free( reqFromOthersSizes );
//Vector_RestoreArray( self->vector, &localSolnVecValues );
VecRestoreArray( self->vector, &localSolnVecValues );
/*
** Syncronise the FEVariable in question.
*/
FeVariable_SyncShadowValues( feVar );
Stream_UnIndentBranch( StgFEM_Debug );
}
