void lucMeshCrossSection_Sample( void* drawingObject, Bool reverse)
{
lucMeshCrossSection* self = (lucMeshCrossSection*)drawingObject;
FeVariable* fieldVariable = (FeVariable*) self->fieldVariable;
Mesh* mesh = (Mesh*) fieldVariable->feMesh;
Grid* vertGrid;
Node_LocalIndex crossSection_I;
IJK node_ijk;
Node_GlobalIndex node_gI;
Node_DomainIndex node_dI;
int i,j, d, sizes[3] = {1,1,1};
Coord globalMin, globalMax, min, max;
int localcount = 0;
vertGrid = *(Grid**)ExtensionManager_Get( mesh->info, mesh, self->vertexGridHandle );
for (d=0; d<fieldVariable->dim; d++) sizes[d] = vertGrid->sizes[d];
self->dim[0] = sizes[ self->axis ];
self->dim[1] = sizes[ self->axis1 ];
self->dim[2] = sizes[ self->axis2 ];
crossSection_I = lucCrossSection_GetValue(self, 0, self->dim[0]-1);
FieldVariable_GetMinAndMaxLocalCoords( fieldVariable, min, max );
FieldVariable_GetMinAndMaxGlobalCoords( fieldVariable, globalMin, globalMax );
Journal_Printf( lucDebug, "%s called on field %s, with axis of cross section as %d, crossSection_I as %d (dims %d,%d,%d) field dim %d\n",
__func__, fieldVariable->name, self->axis, crossSection_I, self->dim[0], self->dim[1], self->dim[2], self->fieldDim);
/* Get mesh cross section self->vertices and values */
self->resolutionA = self->dim[1];
self->resolutionB = self->dim[2];
lucCrossSection_AllocateSampleData(self, self->fieldDim);
int lSize = Mesh_GetLocalSize( mesh, MT_VERTEX );
double time = MPI_Wtime();
Journal_Printf(lucInfo, "Sampling mesh (%s) %d x %d... 0%", self->name, self->dim[1], self->dim[2]);
node_ijk[ self->axis ] = crossSection_I;
for ( i = 0 ; i < self->dim[1]; i++ )
{
int percent = 100 * (i + 1) / self->dim[1];
Journal_Printf(lucInfo, "\b\b\b\b%3d%%", percent);
fflush(stdout);
/* Reverse order if requested */
int i0 = i;
if (reverse) i0 = self->dim[1] - i - 1;
node_ijk[ self->axis1 ] = i0;
for ( j = 0 ; j < self->dim[2]; j++ )
{
self->vertices[i][j][0] = HUGE_VAL;
self->vertices[i][j][2] = 0;
node_ijk[ self->axis2 ] = j;
node_gI = Grid_Project( vertGrid, node_ijk );
/* Get coord and value if node is local... */
if (Mesh_GlobalToDomain( mesh, MT_VERTEX, node_gI, &node_dI ) && node_dI < lSize)
{
/* Found on this processor */
double value[self->fieldDim];
FeVariable_GetValueAtNode( fieldVariable, node_dI, value );
double* pos = Mesh_GetVertex( mesh, node_dI );
/*fprintf(stderr, "[%d] (%d,%d) Node %d %f,%f,%f value %f\n", self->context->rank, i, j, node_gI, pos[0], pos[1], pos[2], value);*/
for (d=0; d<fieldVariable->dim; d++)
self->vertices[i][j][d] = pos[d];
for (d=0; d<self->fieldDim; d++)
self->values[i][j][d] = (float)value[d];
localcount++;
}
}
}
Journal_Printf(lucInfo, " %f sec. ", MPI_Wtime() - time);
/* Collate */
time = MPI_Wtime();
for ( i=0 ; i < self->dim[1]; i++ )
{
for ( j=0 ; j < self->dim[2]; j++ )
{
/* Receive values at root */
if (self->context->rank == 0)
{
/* Already have value? */
if (self->vertices[i][j][0] != HUGE_VAL) {localcount--; continue; }
/* Recv (pos and value together = (3 + fevar dims)*float) */
float data[3 + self->fieldDim];
(void)MPI_Recv(data, 3+self->fieldDim, MPI_FLOAT, MPI_ANY_SOURCE, i*self->dim[2]+j, self->context->communicator, MPI_STATUS_IGNORE);
/* Copy */
memcpy(self->vertices[i][j], data, 3 * sizeof(float));
memcpy(self->values[i][j], &data[3], self->fieldDim * sizeof(float));
}
else
{
/* Found on this proc? */
if (self->vertices[i][j][0] == HUGE_VAL) continue;
/* Copy */
float data[3 + self->fieldDim];
memcpy(data, self->vertices[i][j], 3 * sizeof(float));
memcpy(&data[3], self->values[i][j], self->fieldDim * sizeof(float));
/* Send values to root (pos & value = 4 * float) */
MPI_Ssend(data, 3+self->fieldDim, MPI_FLOAT, 0, i*self->dim[2]+j, self->context->communicator);
localcount--;
}
}
}
MPI_Barrier(self->context->communicator); /* Barrier required, prevent subsequent MPI calls from interfering with transfer */
Journal_Printf(lucInfo, " Gather in %f sec.\n", MPI_Wtime() - time);
Journal_Firewall(localcount == 0, lucError,
"Error - in %s: count of values sampled compared to sent/received by mpi on proc %d does not match (balance = %d)\n",
__func__, self->context->rank, localcount);
}
void ElementCellLayout_BuildShadowInfo( ElementCellLayout* self ) {
unsigned nDims;
Comm* comm;
int nIncProcs;
int* incProcs;
unsigned n_i;
nDims = Mesh_GetDimSize( self->mesh );
comm = Mesh_GetCommTopology( self->mesh, nDims );
Comm_GetNeighbours( comm, &nIncProcs, &incProcs );
/* Extract neighbouring proc information. */
self->cellShadowInfo.procNbrInfo = Memory_Alloc_Unnamed( ProcNbrInfo );
self->cellShadowInfo.procNbrInfo->procNbrCnt = nIncProcs;
self->cellShadowInfo.procNbrInfo->procNbrTbl = AllocArray( unsigned, nIncProcs );
memcpy( self->cellShadowInfo.procNbrInfo->procNbrTbl, incProcs, nIncProcs * sizeof(unsigned) );
/* Count shadow info. */
if( nIncProcs ) {
self->cellShadowInfo.procShadowedCnt = AllocArray( unsigned, nIncProcs );
memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) );
self->cellShadowInfo.procShadowCnt = AllocArray( unsigned, nIncProcs );
memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) );
}
for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) {
unsigned nSharers;
unsigned* sharers;
unsigned s_i;
Mesh_GetSharers( self->mesh, nDims, n_i,
&nSharers, &sharers );
for( s_i = 0; s_i < nSharers; s_i++ )
self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++;
}
for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) {
unsigned owner;
owner = Mesh_GetOwner( self->mesh, nDims, n_i );
self->cellShadowInfo.procShadowCnt[owner]++;
}
/* Build shadow info indices. */
if( nIncProcs ) {
self->cellShadowInfo.procShadowedTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs,
self->cellShadowInfo.procShadowedCnt );
self->cellShadowInfo.procShadowTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs,
self->cellShadowInfo.procShadowCnt );
memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) );
memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) );
}
for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) {
unsigned local;
unsigned curInd;
unsigned nSharers;
unsigned* sharers;
unsigned s_i;
local = Mesh_SharedToLocal( self->mesh, nDims, n_i );
Mesh_GetSharers( self->mesh, nDims, n_i,
&nSharers, &sharers );
for( s_i = 0; s_i < nSharers; s_i++ ) {
curInd = self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++;
self->cellShadowInfo.procShadowedTbl[sharers[s_i]][curInd] = local;
}
}
for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) {
unsigned domain;
unsigned curInd;
unsigned owner;
domain = Mesh_GetLocalSize( self->mesh, nDims ) + n_i;
owner = Mesh_GetOwner( self->mesh, nDims, n_i );
curInd = self->cellShadowInfo.procShadowCnt[owner]++;
self->cellShadowInfo.procShadowTbl[owner][curInd] = domain;
}
}
void SolutionVector_UpdateSolutionOntoNodes( void* solutionVector ) {
SolutionVector* self = (SolutionVector *)solutionVector;
double* localSolnVecValues;
Node_LocalIndex lNode_I = 0;
Dof_Index currNodeNumDofs;
Dof_Index nodeLocalDof_I;
Partition_Index ownerProc;
FeVariable* feVar = self->feVariable;
FeMesh* feMesh = feVar->feMesh;
MPI_Comm mpiComm;
FeEquationNumber* eqNum = self->eqNum;
Dof_EquationNumber currEqNum;
Index indexIntoLocalSolnVecValues;
Index* reqFromOthersCounts;
Index* reqFromOthersSizes;
RequestInfo** reqFromOthersInfos;
Dof_EquationNumber** reqFromOthers;
Comm* comm;
Partition_Index nProc;
Partition_Index myRank;
Partition_Index proc_I;
double initialGuessAtNonLocalEqNumsRatio = 0.1;
double ratioToIncreaseRequestArraySize = 1.5;
Index newReqFromOthersSize;
Journal_DPrintf( self->debug, "In %s - for \"%s\"\n", __func__, self->name );
Stream_IndentBranch( StgFEM_Debug );
#if DEBUG
if ( Stream_IsPrintableLevel( self->debug, 3 ) ) {
Journal_DPrintf( self->debug, "Vector data:\n" );
_SolutionVector_VectorView( self->vector, self->debug );
}
#endif
comm = Mesh_GetCommTopology( feMesh, MT_VERTEX );
mpiComm = Comm_GetMPIComm( comm );
MPI_Comm_size( mpiComm, (int*)&nProc );
MPI_Comm_rank( mpiComm, (int*)&myRank );
/* allocate arrays for nodes that I want on each processor */
reqFromOthersCounts = Memory_Alloc_Array( Index, nProc, "reqFromOthersCounts" );
reqFromOthersSizes = Memory_Alloc_Array( Index, nProc, "reqFromOthersSizes" );
reqFromOthersInfos = Memory_Alloc_Array( RequestInfo*, nProc, "reqFromOthersInfos" );
reqFromOthers = Memory_Alloc_Array( Dof_EquationNumber*, nProc, "reqFromOthers" );
/* Allocate the arrays of req. values from others independently, as we don't know how large they'll be */
for ( proc_I=0; proc_I < nProc; proc_I++ ) {
reqFromOthersCounts[proc_I] = 0;
if (proc_I == myRank) continue;
/* Our initial guess at number of non-local eqNums is a small ratio of the number of local dofs */
reqFromOthersSizes[proc_I] = eqNum->localEqNumsOwnedCount * initialGuessAtNonLocalEqNumsRatio;
/* Special case for really small meshes: make sure it's at least 1 */
if (0 == reqFromOthersSizes[proc_I] ) {
reqFromOthersSizes[proc_I]++;
}
reqFromOthersInfos[proc_I] = Memory_Alloc_Array( RequestInfo, reqFromOthersSizes[proc_I],
"reqFromOthersInfos[proc_I]" );
reqFromOthers[proc_I] = Memory_Alloc_Array( Dof_EquationNumber, reqFromOthersSizes[proc_I],
"reqFromOthers[proc_I]" );
}
/* Get the locally held part of the vector */
//Vector_GetArray( self->vector, &localSolnVecValues );
VecGetArray( self->vector, &localSolnVecValues );
for( lNode_I=0; lNode_I < Mesh_GetLocalSize( feMesh, MT_VERTEX ); lNode_I++ ) {
currNodeNumDofs = feVar->dofLayout->dofCounts[ lNode_I ];
Journal_DPrintfL( self->debug, 3, "getting solutions for local node %d, has %d dofs.\n", lNode_I, currNodeNumDofs );
/* process each dof */
for ( nodeLocalDof_I = 0; nodeLocalDof_I < currNodeNumDofs; nodeLocalDof_I++ ) {
Journal_DPrintfL( self->debug, 3, "\tdof %d: ", nodeLocalDof_I );
currEqNum = eqNum->mapNodeDof2Eq[lNode_I][nodeLocalDof_I];
if( currEqNum != -1 ) {
Journal_DPrintfL( self->debug, 3, "is unconstrained, eqNum %d:", currEqNum );
if( STreeMap_HasKey( eqNum->ownedMap, &currEqNum ) ) {
indexIntoLocalSolnVecValues = *(int*)STreeMap_Map( eqNum->ownedMap, &currEqNum );
Journal_DPrintfL( self->debug, 3, "local -> just copying value %f\n",
localSolnVecValues[indexIntoLocalSolnVecValues] );
DofLayout_SetValueDouble( feVar->dofLayout, lNode_I, nodeLocalDof_I,
localSolnVecValues[indexIntoLocalSolnVecValues] );
}
else {
RequestInfo* requestInfo;
Journal_DPrintfL( self->debug, 3, "nonlocal -> add to req list " );
ownerProc = FeEquationNumber_CalculateOwningProcessorOfEqNum( eqNum, currEqNum );
Journal_DPrintfL( self->debug, 3, "from proc %d\n", ownerProc );
/* first check count & realloc if necessary */
if (reqFromOthersCounts[ownerProc] == reqFromOthersSizes[ownerProc] ) {
newReqFromOthersSize = reqFromOthersSizes[ownerProc] * ratioToIncreaseRequestArraySize;
if ( newReqFromOthersSize == reqFromOthersSizes[ownerProc] ) {
/* Special case: always increase by at least 1 */
newReqFromOthersSize++;
}
reqFromOthersSizes[ownerProc] = newReqFromOthersSize;
Journal_DPrintfL( self->debug, 3, "req list from proc %d count %d now "
"equal to size, so reallocing to size %d\n",
ownerProc, reqFromOthersCounts[ownerProc],
reqFromOthersSizes[ownerProc] );
reqFromOthersInfos[ownerProc] = Memory_Realloc_Array(
reqFromOthersInfos[ownerProc], RequestInfo, reqFromOthersSizes[ownerProc] );
reqFromOthers[ownerProc] = Memory_Realloc_Array(
reqFromOthers[ownerProc], Dof_EquationNumber, reqFromOthersSizes[ownerProc] );
}
requestInfo = &reqFromOthersInfos[ownerProc][ reqFromOthersCounts[ownerProc] ];
requestInfo->lNode_I = lNode_I;
requestInfo->nodeLocalDof_I = nodeLocalDof_I;
reqFromOthers[ownerProc][reqFromOthersCounts[ownerProc]] = currEqNum;
(reqFromOthersCounts[ownerProc])++;
}
}
else {
Journal_DPrintfL( self->debug, 3, "is a BC, so skipping...\n" );
}
}
}
if ( nProc > 1 ) {
_SolutionVector_ShareValuesNotStoredLocally( self, reqFromOthersCounts, reqFromOthersInfos, reqFromOthers,
localSolnVecValues );
}
for ( proc_I=0; proc_I < nProc; proc_I++ ) {
if (proc_I == myRank) continue;
Memory_Free( reqFromOthers[proc_I] );
Memory_Free( reqFromOthersInfos[proc_I] );
}
Memory_Free( reqFromOthers );
Memory_Free( reqFromOthersInfos );
Memory_Free( reqFromOthersCounts );
Memory_Free( reqFromOthersSizes );
//Vector_RestoreArray( self->vector, &localSolnVecValues );
VecRestoreArray( self->vector, &localSolnVecValues );
/*
** Syncronise the FEVariable in question.
*/
FeVariable_SyncShadowValues( feVar );
Stream_UnIndentBranch( StgFEM_Debug );
}
Cell_Index _ElementCellLayout_CellLocalCount( void* elementCellLayout ) {
ElementCellLayout* self = (ElementCellLayout*)elementCellLayout;
return Mesh_GetLocalSize( self->mesh, Mesh_GetDimSize( self->mesh ) );
}
Variable* Mesh_GenerateENMapVar( void* mesh ) {
/* Returns a Variable that stores the mapping of
* [local element] [global node indices]
* Assumes the mapping never changes.
*/
Mesh* self = (Mesh*)mesh;
char* name;
int n_i, e_i, nNbr, localElements, localtotal;
unsigned buffy_tvs; // buffer for global node indices
unsigned dim, *nbr, temp;
int *numberNodesPerEl = NULL;
IArray* inc = NULL;
Stream* error = Journal_Register( Error_Type, (Name)self->type );
// if variable already exists return it
if( self->enMapVar ) return self->enMapVar;
/* go over local elementNode map to get size of data */
inc = IArray_New( );
self->localtotalNodes=0;
dim = Mesh_GetDimSize( self );
localElements = Mesh_GetLocalSize( self, dim );
numberNodesPerEl = Memory_Alloc_Array( int, localElements, "Mesh::numberNodesPerEl" );
for( e_i=0 ; e_i < localElements; e_i++ ) {
Mesh_GetIncidence( self, dim, (unsigned)e_i, MT_VERTEX, inc );
nNbr = IArray_GetSize( inc );
nbr = IArray_GetPtr( inc );
numberNodesPerEl[e_i] = nNbr;
self->localtotalNodes += nNbr;
}
/* Create the Variable data structure, int[nbrNodesPerEl*local element count]
* Note: this is stored in a 1D array - so whatever read or writes to this variable
* needs to know how to parse it. */
self->e_n = Memory_Alloc_Array( int, self->localtotalNodes, "Mesh::nodeConn" );
Stg_asprintf( &name, "%s-%s", self->name, "nodeConn" );
self->enMapVar = Variable_NewScalar( name, NULL, Variable_DataType_Int, &self->localtotalNodes, NULL, (void**)&self->e_n, NULL );
Stg_Component_Build(self->enMapVar, NULL, False);
Stg_Component_Initialise(self->enMapVar, NULL, False);
free(numberNodesPerEl);
free(name);
// Evaluate the global indices for the local nodes
localtotal=0;
for( e_i=0; e_i<localElements; e_i++ ) {
Mesh_GetIncidence( self, dim, (unsigned)e_i, MT_VERTEX, inc );
nNbr = IArray_GetSize( inc );
nbr = IArray_GetPtr( inc );
for( n_i=0; n_i< nNbr; n_i++ ) {
buffy_tvs = Mesh_DomainToGlobal( self, MT_VERTEX, nbr[n_i] );
Variable_SetValue( self->enMapVar, localtotal, (void*)&buffy_tvs );
localtotal++;
}
}
Stg_Class_Delete( inc );
// return new variable
return self->enMapVar;
}
