void lucMeshCrossSection_Sample( void* drawingObject, Bool reverse) { lucMeshCrossSection* self = (lucMeshCrossSection*)drawingObject; FeVariable* fieldVariable = (FeVariable*) self->fieldVariable; Mesh* mesh = (Mesh*) fieldVariable->feMesh; Grid* vertGrid; Node_LocalIndex crossSection_I; IJK node_ijk; Node_GlobalIndex node_gI; Node_DomainIndex node_dI; int i,j, d, sizes[3] = {1,1,1}; Coord globalMin, globalMax, min, max; int localcount = 0; vertGrid = *(Grid**)ExtensionManager_Get( mesh->info, mesh, self->vertexGridHandle ); for (d=0; d<fieldVariable->dim; d++) sizes[d] = vertGrid->sizes[d]; self->dim[0] = sizes[ self->axis ]; self->dim[1] = sizes[ self->axis1 ]; self->dim[2] = sizes[ self->axis2 ]; crossSection_I = lucCrossSection_GetValue(self, 0, self->dim[0]-1); FieldVariable_GetMinAndMaxLocalCoords( fieldVariable, min, max ); FieldVariable_GetMinAndMaxGlobalCoords( fieldVariable, globalMin, globalMax ); Journal_Printf( lucDebug, "%s called on field %s, with axis of cross section as %d, crossSection_I as %d (dims %d,%d,%d) field dim %d\n", __func__, fieldVariable->name, self->axis, crossSection_I, self->dim[0], self->dim[1], self->dim[2], self->fieldDim); /* Get mesh cross section self->vertices and values */ self->resolutionA = self->dim[1]; self->resolutionB = self->dim[2]; lucCrossSection_AllocateSampleData(self, self->fieldDim); int lSize = Mesh_GetLocalSize( mesh, MT_VERTEX ); double time = MPI_Wtime(); Journal_Printf(lucInfo, "Sampling mesh (%s) %d x %d... 0%", self->name, self->dim[1], self->dim[2]); node_ijk[ self->axis ] = crossSection_I; for ( i = 0 ; i < self->dim[1]; i++ ) { int percent = 100 * (i + 1) / self->dim[1]; Journal_Printf(lucInfo, "\b\b\b\b%3d%%", percent); fflush(stdout); /* Reverse order if requested */ int i0 = i; if (reverse) i0 = self->dim[1] - i - 1; node_ijk[ self->axis1 ] = i0; for ( j = 0 ; j < self->dim[2]; j++ ) { self->vertices[i][j][0] = HUGE_VAL; self->vertices[i][j][2] = 0; node_ijk[ self->axis2 ] = j; node_gI = Grid_Project( vertGrid, node_ijk ); /* Get coord and value if node is local... */ if (Mesh_GlobalToDomain( mesh, MT_VERTEX, node_gI, &node_dI ) && node_dI < lSize) { /* Found on this processor */ double value[self->fieldDim]; FeVariable_GetValueAtNode( fieldVariable, node_dI, value ); double* pos = Mesh_GetVertex( mesh, node_dI ); /*fprintf(stderr, "[%d] (%d,%d) Node %d %f,%f,%f value %f\n", self->context->rank, i, j, node_gI, pos[0], pos[1], pos[2], value);*/ for (d=0; d<fieldVariable->dim; d++) self->vertices[i][j][d] = pos[d]; for (d=0; d<self->fieldDim; d++) self->values[i][j][d] = (float)value[d]; localcount++; } } } Journal_Printf(lucInfo, " %f sec. ", MPI_Wtime() - time); /* Collate */ time = MPI_Wtime(); for ( i=0 ; i < self->dim[1]; i++ ) { for ( j=0 ; j < self->dim[2]; j++ ) { /* Receive values at root */ if (self->context->rank == 0) { /* Already have value? */ if (self->vertices[i][j][0] != HUGE_VAL) {localcount--; continue; } /* Recv (pos and value together = (3 + fevar dims)*float) */ float data[3 + self->fieldDim]; (void)MPI_Recv(data, 3+self->fieldDim, MPI_FLOAT, MPI_ANY_SOURCE, i*self->dim[2]+j, self->context->communicator, MPI_STATUS_IGNORE); /* Copy */ memcpy(self->vertices[i][j], data, 3 * sizeof(float)); memcpy(self->values[i][j], &data[3], self->fieldDim * sizeof(float)); } else { /* Found on this proc? */ if (self->vertices[i][j][0] == HUGE_VAL) continue; /* Copy */ float data[3 + self->fieldDim]; memcpy(data, self->vertices[i][j], 3 * sizeof(float)); memcpy(&data[3], self->values[i][j], self->fieldDim * sizeof(float)); /* Send values to root (pos & value = 4 * float) */ MPI_Ssend(data, 3+self->fieldDim, MPI_FLOAT, 0, i*self->dim[2]+j, self->context->communicator); localcount--; } } } MPI_Barrier(self->context->communicator); /* Barrier required, prevent subsequent MPI calls from interfering with transfer */ Journal_Printf(lucInfo, " Gather in %f sec.\n", MPI_Wtime() - time); Journal_Firewall(localcount == 0, lucError, "Error - in %s: count of values sampled compared to sent/received by mpi on proc %d does not match (balance = %d)\n", __func__, self->context->rank, localcount); }
void ElementCellLayout_BuildShadowInfo( ElementCellLayout* self ) { unsigned nDims; Comm* comm; int nIncProcs; int* incProcs; unsigned n_i; nDims = Mesh_GetDimSize( self->mesh ); comm = Mesh_GetCommTopology( self->mesh, nDims ); Comm_GetNeighbours( comm, &nIncProcs, &incProcs ); /* Extract neighbouring proc information. */ self->cellShadowInfo.procNbrInfo = Memory_Alloc_Unnamed( ProcNbrInfo ); self->cellShadowInfo.procNbrInfo->procNbrCnt = nIncProcs; self->cellShadowInfo.procNbrInfo->procNbrTbl = AllocArray( unsigned, nIncProcs ); memcpy( self->cellShadowInfo.procNbrInfo->procNbrTbl, incProcs, nIncProcs * sizeof(unsigned) ); /* Count shadow info. */ if( nIncProcs ) { self->cellShadowInfo.procShadowedCnt = AllocArray( unsigned, nIncProcs ); memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) ); self->cellShadowInfo.procShadowCnt = AllocArray( unsigned, nIncProcs ); memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) ); } for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) { unsigned nSharers; unsigned* sharers; unsigned s_i; Mesh_GetSharers( self->mesh, nDims, n_i, &nSharers, &sharers ); for( s_i = 0; s_i < nSharers; s_i++ ) self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++; } for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) { unsigned owner; owner = Mesh_GetOwner( self->mesh, nDims, n_i ); self->cellShadowInfo.procShadowCnt[owner]++; } /* Build shadow info indices. */ if( nIncProcs ) { self->cellShadowInfo.procShadowedTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs, self->cellShadowInfo.procShadowedCnt ); self->cellShadowInfo.procShadowTbl = Memory_Alloc_2DComplex_Unnamed( unsigned, nIncProcs, self->cellShadowInfo.procShadowCnt ); memset( self->cellShadowInfo.procShadowedCnt, 0, nIncProcs * sizeof(unsigned) ); memset( self->cellShadowInfo.procShadowCnt, 0, nIncProcs * sizeof(unsigned) ); } for( n_i = 0; n_i < Mesh_GetSharedSize( self->mesh, nDims ); n_i++ ) { unsigned local; unsigned curInd; unsigned nSharers; unsigned* sharers; unsigned s_i; local = Mesh_SharedToLocal( self->mesh, nDims, n_i ); Mesh_GetSharers( self->mesh, nDims, n_i, &nSharers, &sharers ); for( s_i = 0; s_i < nSharers; s_i++ ) { curInd = self->cellShadowInfo.procShadowedCnt[sharers[s_i]]++; self->cellShadowInfo.procShadowedTbl[sharers[s_i]][curInd] = local; } } for( n_i = 0; n_i < Mesh_GetRemoteSize( self->mesh, nDims ); n_i++ ) { unsigned domain; unsigned curInd; unsigned owner; domain = Mesh_GetLocalSize( self->mesh, nDims ) + n_i; owner = Mesh_GetOwner( self->mesh, nDims, n_i ); curInd = self->cellShadowInfo.procShadowCnt[owner]++; self->cellShadowInfo.procShadowTbl[owner][curInd] = domain; } }
void SolutionVector_UpdateSolutionOntoNodes( void* solutionVector ) { SolutionVector* self = (SolutionVector *)solutionVector; double* localSolnVecValues; Node_LocalIndex lNode_I = 0; Dof_Index currNodeNumDofs; Dof_Index nodeLocalDof_I; Partition_Index ownerProc; FeVariable* feVar = self->feVariable; FeMesh* feMesh = feVar->feMesh; MPI_Comm mpiComm; FeEquationNumber* eqNum = self->eqNum; Dof_EquationNumber currEqNum; Index indexIntoLocalSolnVecValues; Index* reqFromOthersCounts; Index* reqFromOthersSizes; RequestInfo** reqFromOthersInfos; Dof_EquationNumber** reqFromOthers; Comm* comm; Partition_Index nProc; Partition_Index myRank; Partition_Index proc_I; double initialGuessAtNonLocalEqNumsRatio = 0.1; double ratioToIncreaseRequestArraySize = 1.5; Index newReqFromOthersSize; Journal_DPrintf( self->debug, "In %s - for \"%s\"\n", __func__, self->name ); Stream_IndentBranch( StgFEM_Debug ); #if DEBUG if ( Stream_IsPrintableLevel( self->debug, 3 ) ) { Journal_DPrintf( self->debug, "Vector data:\n" ); _SolutionVector_VectorView( self->vector, self->debug ); } #endif comm = Mesh_GetCommTopology( feMesh, MT_VERTEX ); mpiComm = Comm_GetMPIComm( comm ); MPI_Comm_size( mpiComm, (int*)&nProc ); MPI_Comm_rank( mpiComm, (int*)&myRank ); /* allocate arrays for nodes that I want on each processor */ reqFromOthersCounts = Memory_Alloc_Array( Index, nProc, "reqFromOthersCounts" ); reqFromOthersSizes = Memory_Alloc_Array( Index, nProc, "reqFromOthersSizes" ); reqFromOthersInfos = Memory_Alloc_Array( RequestInfo*, nProc, "reqFromOthersInfos" ); reqFromOthers = Memory_Alloc_Array( Dof_EquationNumber*, nProc, "reqFromOthers" ); /* Allocate the arrays of req. values from others independently, as we don't know how large they'll be */ for ( proc_I=0; proc_I < nProc; proc_I++ ) { reqFromOthersCounts[proc_I] = 0; if (proc_I == myRank) continue; /* Our initial guess at number of non-local eqNums is a small ratio of the number of local dofs */ reqFromOthersSizes[proc_I] = eqNum->localEqNumsOwnedCount * initialGuessAtNonLocalEqNumsRatio; /* Special case for really small meshes: make sure it's at least 1 */ if (0 == reqFromOthersSizes[proc_I] ) { reqFromOthersSizes[proc_I]++; } reqFromOthersInfos[proc_I] = Memory_Alloc_Array( RequestInfo, reqFromOthersSizes[proc_I], "reqFromOthersInfos[proc_I]" ); reqFromOthers[proc_I] = Memory_Alloc_Array( Dof_EquationNumber, reqFromOthersSizes[proc_I], "reqFromOthers[proc_I]" ); } /* Get the locally held part of the vector */ //Vector_GetArray( self->vector, &localSolnVecValues ); VecGetArray( self->vector, &localSolnVecValues ); for( lNode_I=0; lNode_I < Mesh_GetLocalSize( feMesh, MT_VERTEX ); lNode_I++ ) { currNodeNumDofs = feVar->dofLayout->dofCounts[ lNode_I ]; Journal_DPrintfL( self->debug, 3, "getting solutions for local node %d, has %d dofs.\n", lNode_I, currNodeNumDofs ); /* process each dof */ for ( nodeLocalDof_I = 0; nodeLocalDof_I < currNodeNumDofs; nodeLocalDof_I++ ) { Journal_DPrintfL( self->debug, 3, "\tdof %d: ", nodeLocalDof_I ); currEqNum = eqNum->mapNodeDof2Eq[lNode_I][nodeLocalDof_I]; if( currEqNum != -1 ) { Journal_DPrintfL( self->debug, 3, "is unconstrained, eqNum %d:", currEqNum ); if( STreeMap_HasKey( eqNum->ownedMap, &currEqNum ) ) { indexIntoLocalSolnVecValues = *(int*)STreeMap_Map( eqNum->ownedMap, &currEqNum ); Journal_DPrintfL( self->debug, 3, "local -> just copying value %f\n", localSolnVecValues[indexIntoLocalSolnVecValues] ); DofLayout_SetValueDouble( feVar->dofLayout, lNode_I, nodeLocalDof_I, localSolnVecValues[indexIntoLocalSolnVecValues] ); } else { RequestInfo* requestInfo; Journal_DPrintfL( self->debug, 3, "nonlocal -> add to req list " ); ownerProc = FeEquationNumber_CalculateOwningProcessorOfEqNum( eqNum, currEqNum ); Journal_DPrintfL( self->debug, 3, "from proc %d\n", ownerProc ); /* first check count & realloc if necessary */ if (reqFromOthersCounts[ownerProc] == reqFromOthersSizes[ownerProc] ) { newReqFromOthersSize = reqFromOthersSizes[ownerProc] * ratioToIncreaseRequestArraySize; if ( newReqFromOthersSize == reqFromOthersSizes[ownerProc] ) { /* Special case: always increase by at least 1 */ newReqFromOthersSize++; } reqFromOthersSizes[ownerProc] = newReqFromOthersSize; Journal_DPrintfL( self->debug, 3, "req list from proc %d count %d now " "equal to size, so reallocing to size %d\n", ownerProc, reqFromOthersCounts[ownerProc], reqFromOthersSizes[ownerProc] ); reqFromOthersInfos[ownerProc] = Memory_Realloc_Array( reqFromOthersInfos[ownerProc], RequestInfo, reqFromOthersSizes[ownerProc] ); reqFromOthers[ownerProc] = Memory_Realloc_Array( reqFromOthers[ownerProc], Dof_EquationNumber, reqFromOthersSizes[ownerProc] ); } requestInfo = &reqFromOthersInfos[ownerProc][ reqFromOthersCounts[ownerProc] ]; requestInfo->lNode_I = lNode_I; requestInfo->nodeLocalDof_I = nodeLocalDof_I; reqFromOthers[ownerProc][reqFromOthersCounts[ownerProc]] = currEqNum; (reqFromOthersCounts[ownerProc])++; } } else { Journal_DPrintfL( self->debug, 3, "is a BC, so skipping...\n" ); } } } if ( nProc > 1 ) { _SolutionVector_ShareValuesNotStoredLocally( self, reqFromOthersCounts, reqFromOthersInfos, reqFromOthers, localSolnVecValues ); } for ( proc_I=0; proc_I < nProc; proc_I++ ) { if (proc_I == myRank) continue; Memory_Free( reqFromOthers[proc_I] ); Memory_Free( reqFromOthersInfos[proc_I] ); } Memory_Free( reqFromOthers ); Memory_Free( reqFromOthersInfos ); Memory_Free( reqFromOthersCounts ); Memory_Free( reqFromOthersSizes ); //Vector_RestoreArray( self->vector, &localSolnVecValues ); VecRestoreArray( self->vector, &localSolnVecValues ); /* ** Syncronise the FEVariable in question. */ FeVariable_SyncShadowValues( feVar ); Stream_UnIndentBranch( StgFEM_Debug ); }
Cell_Index _ElementCellLayout_CellLocalCount( void* elementCellLayout ) { ElementCellLayout* self = (ElementCellLayout*)elementCellLayout; return Mesh_GetLocalSize( self->mesh, Mesh_GetDimSize( self->mesh ) ); }
Variable* Mesh_GenerateENMapVar( void* mesh ) { /* Returns a Variable that stores the mapping of * [local element] [global node indices] * Assumes the mapping never changes. */ Mesh* self = (Mesh*)mesh; char* name; int n_i, e_i, nNbr, localElements, localtotal; unsigned buffy_tvs; // buffer for global node indices unsigned dim, *nbr, temp; int *numberNodesPerEl = NULL; IArray* inc = NULL; Stream* error = Journal_Register( Error_Type, (Name)self->type ); // if variable already exists return it if( self->enMapVar ) return self->enMapVar; /* go over local elementNode map to get size of data */ inc = IArray_New( ); self->localtotalNodes=0; dim = Mesh_GetDimSize( self ); localElements = Mesh_GetLocalSize( self, dim ); numberNodesPerEl = Memory_Alloc_Array( int, localElements, "Mesh::numberNodesPerEl" ); for( e_i=0 ; e_i < localElements; e_i++ ) { Mesh_GetIncidence( self, dim, (unsigned)e_i, MT_VERTEX, inc ); nNbr = IArray_GetSize( inc ); nbr = IArray_GetPtr( inc ); numberNodesPerEl[e_i] = nNbr; self->localtotalNodes += nNbr; } /* Create the Variable data structure, int[nbrNodesPerEl*local element count] * Note: this is stored in a 1D array - so whatever read or writes to this variable * needs to know how to parse it. */ self->e_n = Memory_Alloc_Array( int, self->localtotalNodes, "Mesh::nodeConn" ); Stg_asprintf( &name, "%s-%s", self->name, "nodeConn" ); self->enMapVar = Variable_NewScalar( name, NULL, Variable_DataType_Int, &self->localtotalNodes, NULL, (void**)&self->e_n, NULL ); Stg_Component_Build(self->enMapVar, NULL, False); Stg_Component_Initialise(self->enMapVar, NULL, False); free(numberNodesPerEl); free(name); // Evaluate the global indices for the local nodes localtotal=0; for( e_i=0; e_i<localElements; e_i++ ) { Mesh_GetIncidence( self, dim, (unsigned)e_i, MT_VERTEX, inc ); nNbr = IArray_GetSize( inc ); nbr = IArray_GetPtr( inc ); for( n_i=0; n_i< nNbr; n_i++ ) { buffy_tvs = Mesh_DomainToGlobal( self, MT_VERTEX, nbr[n_i] ); Variable_SetValue( self->enMapVar, localtotal, (void*)&buffy_tvs ); localtotal++; } } Stg_Class_Delete( inc ); // return new variable return self->enMapVar; }