int main(int argc, char ** argv ) { int size, rank; unsigned int numPts; char pFile[50],oFile[50]; double gSize[3]; unsigned int ptsLen; unsigned int maxNumPts= 1; unsigned int dim=3; unsigned int maxDepth=30; DendroIntL locSz, totalSz; std::vector<ot::TreeNode> linOct, balOct; std::vector<double> pts; PetscInitialize(&argc,&argv,"optionsDS",help); ot::RegisterEvents(); ot::DAMG_Initialize(MPI_COMM_WORLD); #ifdef PETSC_USE_LOG int stages[3]; PetscLogStageRegister("P2O.",&stages[0]); PetscLogStageRegister("Bal",&stages[1]); PetscLogStageRegister("Solve",&stages[2]); #endif MPI_Comm_size(MPI_COMM_WORLD,&size); MPI_Comm_rank(MPI_COMM_WORLD,&rank); if(argc < 2) { std::cerr << "Usage: " << argv[0] << "inpfile maxDepth[30] \ dim[3] maxNumPtsPerOctant[1] " << std::endl; return -1; }
int main(int argc, char ** argv ) { int size, rank; bool incCorner = 1; unsigned int numPts; unsigned int solveU = 0; unsigned int writeB = 0; char Kstr[20]; char pFile[50],bFile[50],uFile[50]; double gSize[3]; unsigned int ptsLen; unsigned int maxNumPts= 1; unsigned int dim=3; unsigned int maxDepth=30; bool compressLut=true; double localTime, totalTime; double startTime, endTime; DendroIntL locSz, totalSz; std::vector<ot::TreeNode> linOct, balOct; std::vector<double> pts; double mgLoadFac = 2.0; PetscInitialize(&argc,&argv,"options",help); ot::RegisterEvents(); ot::DAMG_Initialize(MPI_COMM_WORLD); #ifdef PETSC_USE_LOG PetscClassId classid; PetscClassIdRegister("Dendro",&classid); PetscLogEventRegister("ODAmatDiag",classid, &Jac1DiagEvent); PetscLogEventRegister("ODAmatMult",classid, &Jac1MultEvent); PetscLogEventRegister("ODAmatDiagFinest",classid, &Jac1FinestDiagEvent); PetscLogEventRegister("ODAmatMultFinest",classid, &Jac1FinestMultEvent); PetscLogEventRegister("OMGmatDiag-2",classid, &Jac2DiagEvent); PetscLogEventRegister("OMGmatMult-2",classid, &Jac2MultEvent); PetscLogEventRegister("OMGmatDiagFinest-2",classid, &Jac2FinestDiagEvent); PetscLogEventRegister("OMGmatMultFinest-2",classid, &Jac2FinestMultEvent); PetscLogEventRegister("OMGmatDiag-3",classid, &Jac3DiagEvent); PetscLogEventRegister("OMGmatMult-3",classid, &Jac3MultEvent); PetscLogEventRegister("OMGmatDiagFinest-3",classid, &Jac3FinestDiagEvent); PetscLogEventRegister("OMGmatMultFinest-3",classid, &Jac3FinestMultEvent); int stages[3]; PetscLogStageRegister("P2O.",&stages[0]); PetscLogStageRegister("Bal",&stages[1]); PetscLogStageRegister("Solve",&stages[2]); #endif MPI_Comm_size(MPI_COMM_WORLD,&size); MPI_Comm_rank(MPI_COMM_WORLD,&rank); if(argc < 3) { std::cerr << "Usage: " << argv[0] << "inpfile maxDepth[30] solveU[0]\ writeB[0] dim[3] maxNumPtsPerOctant[1] incCorner[1] compressLut[1] mgLoadFac[2.0] " << std::endl; return -1; }
static PetscErrorCode TestCone(DM dm, AppCtx *user) { PetscInt numRuns, cStart, cEnd, c, i; PetscReal maxTimePerRun = user->maxConeTime; PetscLogStage stage; PetscLogEvent event; PetscEventPerfInfo eventInfo; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLogStageRegister("DMPlex Cone Test", &stage);CHKERRQ(ierr); ierr = PetscLogEventRegister("Cone", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr); ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr); for (i = 0; i < user->iterations; ++i) { for (c = cStart; c < cEnd; ++c) { const PetscInt *cone; ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr); } } ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr); numRuns = (cEnd-cStart) * user->iterations; if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1); if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0); if (eventInfo.time > maxTimePerRun * numRuns) { ierr = PetscPrintf(PETSC_COMM_SELF, "Cones: %d Average time per cone: %gs standard: %gs\n", numRuns, eventInfo.time/numRuns, maxTimePerRun); if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for cone %g > standard %g", eventInfo.time/numRuns, maxTimePerRun); } PetscFunctionReturn(0); }
PetscErrorCode TestVecClosure(DM dm, AppCtx *user) { PetscSection s; Vec v; PetscInt numRuns, cStart, cEnd, c, i; PetscScalar tmpArray[64]; PetscScalar *userArray = user->reuseArray ? tmpArray : NULL; PetscReal maxTimePerRun = user->maxVecClosureTime; PetscStageLog stageLog; PetscEventPerfLog eventLog; PetscInt stage; PetscLogEvent event; PetscEventPerfInfo eventInfo; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLogStageRegister("DMPlex Vector Closure Test", &stage);CHKERRQ(ierr); ierr = PetscLogEventRegister("VecClosure", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = DMPlexCreateSection(dm, user->dim, user->numFields, user->numComponents, user->numDof, 0, NULL, NULL, &s);CHKERRQ(ierr); ierr = DMSetDefaultSection(dm, s);CHKERRQ(ierr); ierr = PetscSectionDestroy(&s);CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr); ierr = DMGetLocalVector(dm, &v);CHKERRQ(ierr); ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr); for (i = 0; i < user->iterations; ++i) { for (c = cStart; c < cEnd; ++c) { PetscScalar *closure = userArray; PetscInt closureSize = 64;; ierr = DMPlexVecGetClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr); if (!user->reuseArray) {ierr = DMPlexVecRestoreClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr);} } } ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr); ierr = DMRestoreLocalVector(dm, &v);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogGetStageLog(&stageLog); ierr = PetscStageLogGetEventPerfLog(stageLog, stage, &eventLog); numRuns = (cEnd-cStart) * user->iterations; eventInfo = eventLog->eventInfo[event]; if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1); if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0); if (eventInfo.time > maxTimePerRun * numRuns) { ierr = PetscPrintf(PETSC_COMM_SELF, "VecClosures: %d Average time per cone: %gs standard: %gs\n", numRuns, eventInfo.time/numRuns, maxTimePerRun); if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for vector closure %g > standard %g", eventInfo.time/numRuns, maxTimePerRun); } PetscFunctionReturn(0); }
// initialize the decoupled IBPM solver for moving rigid bodies PetscErrorCode RigidKinematicsSolver::init(const MPI_Comm &world, const YAML::Node &node) { PetscErrorCode ierr; PetscFunctionBeginUser; ierr = DecoupledIBPMSolver::init(world, node); CHKERRQ(ierr); ierr = PetscLogStagePush(stageInitialize); CHKERRQ(ierr); ierr = PetscLogStageRegister("moveIB", &stageMoveIB); CHKERRQ(ierr); ierr = VecDuplicate(f, &UB); CHKERRQ(ierr); ierr = VecSet(UB, 0.0); CHKERRQ(ierr); ierr = PetscLogStagePop(); CHKERRQ(ierr); // end of stageInitialize PetscFunctionReturn(0); } // init
PetscErrorCode TestTransitiveClosure(DM dm, AppCtx *user) { PetscInt numRuns, cStart, cEnd, c, i; PetscReal maxTimePerRun = user->maxClosureTime; PetscStageLog stageLog; PetscEventPerfLog eventLog; PetscInt stage; PetscLogEvent event; PetscEventPerfInfo eventInfo; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLogStageRegister("DMPlex Transitive Closure Test", &stage);CHKERRQ(ierr); ierr = PetscLogEventRegister("TransitiveClosure", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr); ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr); for (i = 0; i < user->iterations; ++i) { for (c = cStart; c < cEnd; ++c) { PetscInt *closure = NULL; PetscInt closureSize; ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr); ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr); } } ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogGetStageLog(&stageLog); ierr = PetscStageLogGetEventPerfLog(stageLog, stage, &eventLog); numRuns = (cEnd-cStart) * user->iterations; eventInfo = eventLog->eventInfo[event]; if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1); if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0); if (eventInfo.time > maxTimePerRun * numRuns) { ierr = PetscPrintf(PETSC_COMM_SELF, "Closures: %d Average time per cone: %gs standard: %gs\n", numRuns, eventInfo.time/numRuns, maxTimePerRun); if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for closure %g > standard %g", eventInfo.time/numRuns, maxTimePerRun); } PetscFunctionReturn(0); }
int main(int argc,char **args) { Vec x1,b1,x2,b2; /* solution and RHS vectors for systems #1 and #2 */ Vec u; /* exact solution vector */ Mat C1,C2; /* matrices for systems #1 and #2 */ KSP ksp1,ksp2; /* KSP contexts for systems #1 and #2 */ PetscInt ntimes = 3; /* number of times to solve the linear systems */ PetscLogEvent CHECK_ERROR; /* event number for error checking */ PetscInt ldim,low,high,iglobal,Istart,Iend,Istart2,Iend2; PetscInt Ii,J,i,j,m = 3,n = 2,its,t; PetscErrorCode ierr; PetscBool flg = PETSC_FALSE; PetscScalar v; PetscMPIInt rank,size; #if defined (PETSC_USE_LOG) PetscLogStage stages[3]; #endif PetscInitialize(&argc,&args,(char *)0,help); ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-t",&ntimes,PETSC_NULL);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); n = 2*size; /* Register various stages for profiling */ ierr = PetscLogStageRegister("Prelim setup",&stages[0]);CHKERRQ(ierr); ierr = PetscLogStageRegister("Linear System 1",&stages[1]);CHKERRQ(ierr); ierr = PetscLogStageRegister("Linear System 2",&stages[2]);CHKERRQ(ierr); /* Register a user-defined event for profiling (error checking). */ CHECK_ERROR = 0; ierr = PetscLogEventRegister("Check Error",KSP_CLASSID,&CHECK_ERROR);CHKERRQ(ierr); /* - - - - - - - - - - - - Stage 0: - - - - - - - - - - - - - - Preliminary Setup - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr); /* Create data structures for first linear system. - Create parallel matrix, specifying only its global dimensions. When using MatCreate(), the matrix format can be specified at runtime. Also, the parallel partitioning of the matrix is determined by PETSc at runtime. - Create parallel vectors. - When using VecSetSizes(), we specify only the vector's global dimension; the parallel partitioning is determined at runtime. - Note: We form 1 vector from scratch and then duplicate as needed. */ ierr = MatCreate(PETSC_COMM_WORLD,&C1);CHKERRQ(ierr); ierr = MatSetSizes(C1,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr); ierr = MatSetFromOptions(C1);CHKERRQ(ierr); ierr = MatSetUp(C1);CHKERRQ(ierr); ierr = MatGetOwnershipRange(C1,&Istart,&Iend);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr); ierr = VecSetSizes(u,PETSC_DECIDE,m*n);CHKERRQ(ierr); ierr = VecSetFromOptions(u);CHKERRQ(ierr); ierr = VecDuplicate(u,&b1);CHKERRQ(ierr); ierr = VecDuplicate(u,&x1);CHKERRQ(ierr); /* Create first linear solver context. Set runtime options (e.g., -pc_type <type>). Note that the first linear system uses the default option names, while the second linear systme uses a different options prefix. */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp1);CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp1);CHKERRQ(ierr); /* Set user-defined monitoring routine for first linear system. */ ierr = PetscOptionsGetBool(PETSC_NULL,"-my_ksp_monitor",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) {ierr = KSPMonitorSet(ksp1,MyKSPMonitor,PETSC_NULL,0);CHKERRQ(ierr);} /* Create data structures for second linear system. */ ierr = MatCreate(PETSC_COMM_WORLD,&C2);CHKERRQ(ierr); ierr = MatSetSizes(C2,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr); ierr = MatSetFromOptions(C2);CHKERRQ(ierr); ierr = MatSetUp(C2);CHKERRQ(ierr); ierr = MatGetOwnershipRange(C2,&Istart2,&Iend2);CHKERRQ(ierr); ierr = VecDuplicate(u,&b2);CHKERRQ(ierr); ierr = VecDuplicate(u,&x2);CHKERRQ(ierr); /* Create second linear solver context */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp2);CHKERRQ(ierr); /* Set different options prefix for second linear system. Set runtime options (e.g., -s2_pc_type <type>) */ ierr = KSPAppendOptionsPrefix(ksp2,"s2_");CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp2);CHKERRQ(ierr); /* Assemble exact solution vector in parallel. Note that each processor needs to set only its local part of the vector. */ ierr = VecGetLocalSize(u,&ldim);CHKERRQ(ierr); ierr = VecGetOwnershipRange(u,&low,&high);CHKERRQ(ierr); for (i=0; i<ldim; i++) { iglobal = i + low; v = (PetscScalar)(i + 100*rank); ierr = VecSetValues(u,1,&iglobal,&v,ADD_VALUES);CHKERRQ(ierr); } ierr = VecAssemblyBegin(u);CHKERRQ(ierr); ierr = VecAssemblyEnd(u);CHKERRQ(ierr); /* Log the number of flops for computing vector entries */ ierr = PetscLogFlops(2.0*ldim);CHKERRQ(ierr); /* End curent profiling stage */ ierr = PetscLogStagePop();CHKERRQ(ierr); /* -------------------------------------------------------------- Linear solver loop: Solve 2 different linear systems several times in succession -------------------------------------------------------------- */ for (t=0; t<ntimes; t++) { /* - - - - - - - - - - - - Stage 1: - - - - - - - - - - - - - - Assemble and solve first linear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Begin profiling stage #1 */ ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr); /* Initialize all matrix entries to zero. MatZeroEntries() retains the nonzero structure of the matrix for sparse formats. */ if (t > 0) {ierr = MatZeroEntries(C1);CHKERRQ(ierr);} /* Set matrix entries in parallel. Also, log the number of flops for computing matrix entries. - Each processor needs to insert only elements that it owns locally (but any non-local elements will be sent to the appropriate processor during matrix assembly). - Always specify global row and columns of matrix entries. */ for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; i = Ii/n; j = Ii - i*n; if (i>0) {J = Ii - n; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = Ii + n; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} if (j>0) {J = Ii - 1; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} v = 4.0; ierr = MatSetValues(C1,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr); } for (Ii=Istart; Ii<Iend; Ii++) { /* Make matrix nonsymmetric */ v = -1.0*(t+0.5); i = Ii/n; if (i>0) {J = Ii - n; ierr = MatSetValues(C1,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} } ierr = PetscLogFlops(2.0*(Iend-Istart));CHKERRQ(ierr); /* Assemble matrix, using the 2-step process: MatAssemblyBegin(), MatAssemblyEnd() Computations can be done while messages are in transition by placing code between these two statements. */ ierr = MatAssemblyBegin(C1,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(C1,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* Indicate same nonzero structure of successive linear system matrices */ ierr = MatSetOption(C1,MAT_NEW_NONZERO_LOCATIONS,PETSC_TRUE);CHKERRQ(ierr); /* Compute right-hand-side vector */ ierr = MatMult(C1,u,b1);CHKERRQ(ierr); /* Set operators. Here the matrix that defines the linear system also serves as the preconditioning matrix. - The flag SAME_NONZERO_PATTERN indicates that the preconditioning matrix has identical nonzero structure as during the last linear solve (although the values of the entries have changed). Thus, we can save some work in setting up the preconditioner (e.g., no need to redo symbolic factorization for ILU/ICC preconditioners). - If the nonzero structure of the matrix is different during the second linear solve, then the flag DIFFERENT_NONZERO_PATTERN must be used instead. If you are unsure whether the matrix structure has changed or not, use the flag DIFFERENT_NONZERO_PATTERN. - Caution: If you specify SAME_NONZERO_PATTERN, PETSc believes your assertion and does not check the structure of the matrix. If you erroneously claim that the structure is the same when it actually is not, the new preconditioner will not function correctly. Thus, use this optimization feature with caution! */ ierr = KSPSetOperators(ksp1,C1,C1,SAME_NONZERO_PATTERN);CHKERRQ(ierr); /* Use the previous solution of linear system #1 as the initial guess for the next solve of linear system #1. The user MUST call KSPSetInitialGuessNonzero() in indicate use of an initial guess vector; otherwise, an initial guess of zero is used. */ if (t>0) { ierr = KSPSetInitialGuessNonzero(ksp1,PETSC_TRUE);CHKERRQ(ierr); } /* Solve the first linear system. Here we explicitly call KSPSetUp() for more detailed performance monitoring of certain preconditioners, such as ICC and ILU. This call is optional, ase KSPSetUp() will automatically be called within KSPSolve() if it hasn't been called already. */ ierr = KSPSetUp(ksp1);CHKERRQ(ierr); ierr = KSPSolve(ksp1,b1,x1);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp1,&its);CHKERRQ(ierr); /* Check error of solution to first linear system */ ierr = CheckError(u,x1,b1,its,1.e-4,CHECK_ERROR);CHKERRQ(ierr); /* - - - - - - - - - - - - Stage 2: - - - - - - - - - - - - - - Assemble and solve second linear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Conclude profiling stage #1; begin profiling stage #2 */ ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStagePush(stages[2]);CHKERRQ(ierr); /* Initialize all matrix entries to zero */ if (t > 0) {ierr = MatZeroEntries(C2);CHKERRQ(ierr);} /* Assemble matrix in parallel. Also, log the number of flops for computing matrix entries. - To illustrate the features of parallel matrix assembly, we intentionally set the values differently from the way in which the matrix is distributed across the processors. Each entry that is not owned locally will be sent to the appropriate processor during MatAssemblyBegin() and MatAssemblyEnd(). - For best efficiency the user should strive to set as many entries locally as possible. */ for (i=0; i<m; i++) { for (j=2*rank; j<2*rank+2; j++) { v = -1.0; Ii = j + n*i; if (i>0) {J = Ii - n; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = Ii + n; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} if (j>0) {J = Ii - 1; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} v = 6.0 + t*0.5; ierr = MatSetValues(C2,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr); } } for (Ii=Istart2; Ii<Iend2; Ii++) { /* Make matrix nonsymmetric */ v = -1.0*(t+0.5); i = Ii/n; if (i>0) {J = Ii - n; ierr = MatSetValues(C2,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);} } ierr = MatAssemblyBegin(C2,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(C2,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = PetscLogFlops(2.0*(Iend-Istart));CHKERRQ(ierr); /* Indicate same nonzero structure of successive linear system matrices */ ierr = MatSetOption(C2,MAT_NEW_NONZERO_LOCATIONS,PETSC_FALSE);CHKERRQ(ierr); /* Compute right-hand-side vector */ ierr = MatMult(C2,u,b2);CHKERRQ(ierr); /* Set operators. Here the matrix that defines the linear system also serves as the preconditioning matrix. Indicate same nonzero structure of successive preconditioner matrices by setting flag SAME_NONZERO_PATTERN. */ ierr = KSPSetOperators(ksp2,C2,C2,SAME_NONZERO_PATTERN);CHKERRQ(ierr); /* Solve the second linear system */ ierr = KSPSetUp(ksp2);CHKERRQ(ierr); ierr = KSPSolve(ksp2,b2,x2);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp2,&its);CHKERRQ(ierr); /* Check error of solution to second linear system */ ierr = CheckError(u,x2,b2,its,1.e-4,CHECK_ERROR);CHKERRQ(ierr); /* Conclude profiling stage #2 */ ierr = PetscLogStagePop();CHKERRQ(ierr); } /* -------------------------------------------------------------- End of linear solver loop -------------------------------------------------------------- */ /* Free work space. All PETSc objects should be destroyed when they are no longer needed. */ ierr = KSPDestroy(&ksp1);CHKERRQ(ierr); ierr = KSPDestroy(&ksp2);CHKERRQ(ierr); ierr = VecDestroy(&x1);CHKERRQ(ierr); ierr = VecDestroy(&x2);CHKERRQ(ierr); ierr = VecDestroy(&b1);CHKERRQ(ierr); ierr = VecDestroy(&b2);CHKERRQ(ierr); ierr = MatDestroy(&C1);CHKERRQ(ierr); ierr = MatDestroy(&C2);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
int main(int argc, char ** argv ) { int size, rank; double startTime, endTime, minTime; unsigned int incInt = 1; bool incCorner = 1; unsigned int numPts; unsigned int writePOut = 0; unsigned int writeBOut = 0; unsigned int readPtsFile = 0; unsigned int readOctFile =0; unsigned int runOpt =2;//2 for both, 1 for pt alone and 0 for bal alone. char Kstr[20]; char inpFileName[50], p2nIn[50], p2nOut[50], n2oOut[50], balOut[50]; double gSize[3]; PetscInitialize(&argc,&argv,"options",NULL); ot::RegisterEvents(); #ifdef PETSC_USE_LOG int stages[4]; PetscLogStageRegister("Prepare Input.",&stages[0]); PetscLogStageRegister("P2O",&stages[1]); PetscLogStageRegister("N2O",&stages[2]); PetscLogStageRegister("Bal",&stages[3]); #endif MPI_Comm_size(MPI_COMM_WORLD,&size); MPI_Comm_rank(MPI_COMM_WORLD,&rank); if(argc < 3) { std::cerr << "Usage: " << argv[0] << " inpfile complx incInt[1] runOpt[2]"<< "readPtsFile[0] readOctFile[0] maxDepth[30] writePOut[0] writeBOut[0]"<< " dim[3] maxNumPtsPerOctant[1] incCorner[0] " << std::endl; return -1; } unsigned int maxNumPts= 1; int complx = atoi(argv[2]); unsigned int dim=3; unsigned int maxDepth=30; if(argc > 3) { incInt = atoi(argv[3]); } if(argc > 4) { runOpt = atoi(argv[4]); } if(argc > 5) { readPtsFile = atoi(argv[5]); } if(argc > 6) { readOctFile = atoi(argv[6]); } if(argc > 7) { maxDepth = atoi(argv[7]); } if(argc > 8) { writePOut = atoi(argv[8]); } if(argc > 9) { writeBOut = atoi(argv[9]); } if(argc > 10) { dim = atoi(argv[10]); } if(argc > 11) { maxNumPts = atoi(argv[11]); } if(argc > 12) { incCorner = (bool)(atoi(argv[12]));} strcpy(inpFileName, argv[1]); strcpy(balOut,inpFileName); ot::int2str(rank,Kstr); strcat(balOut,Kstr); strcat(balOut,"_\0"); ot::int2str(size,Kstr); strcat(balOut,Kstr); strcpy(n2oOut,balOut); strcpy(p2nIn,balOut); strcpy(p2nOut,balOut); strcat(balOut,"_Bal.ot\0"); strcat(n2oOut,"_Con.ot\0"); strcat(p2nIn,".pts\0"); strcat(p2nOut,"_Out.pts\0"); strcat(inpFileName,".inp\0"); std::vector<ot::TreeNode> nodes; std::vector<double> pts; unsigned int ptsLen; // print out the format ... /* if (!rank) { cout << "============================================================================================" << endl; cout << "#Points p2n_time p2n_imbal #cellsConstr #cellsBal bal_time bal_imbal maxLevBef minLevBef avgLevBef maxLevAft minLevAft avgLevAft" << endl; cout << "============================================================================================" << endl << endl; } */ #ifdef PETSC_USE_LOG PetscLogStagePush(stages[0]); #endif if(runOpt == 1 || runOpt == 2) { if(readPtsFile) { if(!rank){ std::cout << " reading "<<p2nIn<<std::endl; // Point size } ot::readPtsFromFile(p2nIn, pts); if(!rank){ std::cout << " finished reading "<<p2nIn<<std::endl; // Point size } ptsLen = pts.size(); std::vector<ot::TreeNode> tmpNodes; for(int i = 0; i < ptsLen; i += 3) { if( (pts[i] > 0.0) && (pts[i+1] > 0.0) && (pts[i+2] > 0.0) && ( ((unsigned int)(pts[i]*((double)(1u<<maxDepth)))) < (1u<<maxDepth)) && ( ((unsigned int)(pts[i+1]*((double)(1u<<maxDepth)))) < (1u<<maxDepth)) && ( ((unsigned int)(pts[i+2]*((double)(1u<<maxDepth)))) < (1u<<maxDepth)) ) { tmpNodes.push_back( ot::TreeNode((unsigned int)(pts[i]*(double)(1u<<maxDepth)), (unsigned int)(pts[i+1]*(double)(1u<<maxDepth)), (unsigned int)(pts[i+2]*(double)(1u<<maxDepth)), maxDepth,dim,maxDepth) ); } }//end for i pts.clear(); // std::cout<<"Before Removing Duplicates, the number of pts: "<<tmpNodes.size()<<std::endl; par::removeDuplicates<ot::TreeNode>(tmpNodes,false,MPI_COMM_WORLD); // std::cout<<"After Removing Duplicates, the number of pts: "<<tmpNodes.size()<<std::endl; nodes = tmpNodes; tmpNodes.clear(); par::partitionW<ot::TreeNode>(nodes, NULL,MPI_COMM_WORLD); // reduce and only print the total ... DendroIntL locSz, totalSz; locSz = nodes.size(); par::Mpi_Reduce<DendroIntL>(&locSz, &totalSz, 1, MPI_SUM, 0, MPI_COMM_WORLD); if(rank==0) { std::cout<<"rank= "<<rank<<" size= " << size << " total pts= " << totalSz<<std::endl; } pts.resize(3*(nodes.size())); ptsLen = (3*(nodes.size())); for(int i=0;i<nodes.size();i++) { pts[3*i] = (((double)(nodes[i].getX())) + 0.5)/((double)(1u<<maxDepth)); pts[(3*i)+1] = (((double)(nodes[i].getY())) +0.5)/((double)(1u<<maxDepth)); pts[(3*i)+2] = (((double)(nodes[i].getZ())) +0.5)/((double)(1u<<maxDepth)); }//end for i nodes.clear(); }//end if readPts gSize[0] = 1.0; gSize[1] = 1.0; gSize[2] = 1.0; if(writePOut) { if(!rank) { std::cout<<"Writing pts to: "<<p2nOut<<std::endl; } ot::writePtsToFile(p2nOut,pts); } #ifdef PETSC_USE_LOG PetscLogStagePop(); #endif MPI_Barrier(MPI_COMM_WORLD); #ifdef PETSC_USE_LOG PetscLogStagePush(stages[1]); #endif startTime = MPI_Wtime(); ot::points2Octree(pts, gSize, nodes, dim, maxDepth, maxNumPts, MPI_COMM_WORLD); endTime = MPI_Wtime(); #ifdef PETSC_USE_LOG PetscLogStagePop(); #endif double locTime, totalTime, minTime; locTime = endTime - startTime; par::Mpi_Reduce<double>(&locTime, &totalTime, 1, MPI_MAX, 0, MPI_COMM_WORLD); par::Mpi_Reduce<double>(&locTime, &minTime, 1, MPI_MIN, 0, MPI_COMM_WORLD); if(!rank){ std::cout <<"P2n Time: "<<totalTime << " " << "secs imbalance: "<< totalTime/minTime << std::endl; } pts.clear(); } //TestBuildOCt from here on: if(runOpt == 0 || runOpt == 2) { if(readOctFile) { ot::readNodesFromFile(n2oOut,nodes); } std::vector<ot::TreeNode > linOct; #ifdef PETSC_USE_LOG PetscLogStagePush(stages[2]); #endif DendroIntL inputNodes = nodes.size(); DendroIntL totInp; par::Mpi_Reduce<DendroIntL>(&inputNodes, &totInp, 1, MPI_SUM, 0, MPI_COMM_WORLD); if(!rank) { std::cout << "Nodes after p2o: "<< totInp << std::endl; } ot::completeOctree(nodes,linOct,dim,maxDepth,false, false, false, MPI_COMM_WORLD); #ifdef PETSC_USE_LOG PetscLogStagePop(); #endif assert(!linOct.empty()); if(writeBOut) { if(!rank) { std::cout<<"Writing octree after p2o to: "<<n2oOut<<std::endl; } ot::writeNodesToFile(n2oOut,linOct); } inputNodes = linOct.size(); nodes.clear(); std::vector<ot::TreeNode > balOct; par::Mpi_Reduce<DendroIntL>(&inputNodes, &totInp, 1, MPI_SUM, 0, MPI_COMM_WORLD); if(!rank) { std::cout << "Nodes before balancing: "<< totInp << std::endl; } MPI_Barrier(MPI_COMM_WORLD); #ifdef PETSC_USE_LOG PetscLogStagePush(stages[3]); #endif startTime = MPI_Wtime(); ot::balanceOctree (linOct, balOct, dim, maxDepth, incCorner, MPI_COMM_WORLD, NULL, NULL); endTime = MPI_Wtime(); #ifdef PETSC_USE_LOG PetscLogStagePop(); #endif assert(!balOct.empty()); if(writeBOut) { if(!rank) { std::cout<<"Writing octree after balancing to: "<<balOut<<std::endl; } ot::writeNodesToFile(balOut,balOct); } // compute total inp size and output size DendroIntL locBalNodes = balOct.size(); DendroIntL totBal; double balTime, totTime; balTime = endTime - startTime; par::Mpi_Reduce<DendroIntL>(&locBalNodes, &totBal, 1, MPI_SUM, 0, MPI_COMM_WORLD); par::Mpi_Reduce<double>(&balTime, &totTime, 1, MPI_MAX, 0, MPI_COMM_WORLD); par::Mpi_Reduce<double>(&balTime, &minTime, 1, MPI_MIN, 0, MPI_COMM_WORLD); if(!rank) { std::cout << "Nodes after balancing: "<< totBal << std::endl; std::cout << "bal Time: "<<totTime << " " << "secs imbalance: "<< totTime/minTime << std::endl; } linOct.clear(); balOct.clear(); } PetscFinalize(); }
int main(int argc,char **args) { Mat Amat; PetscErrorCode ierr; SNES snes; KSP ksp; MPI_Comm comm; PetscMPIInt npe,rank; PetscLogStage stage[7]; PetscBool test_nonzero_cols=PETSC_FALSE,use_nearnullspace=PETSC_TRUE; Vec xx,bb; PetscInt iter,i,N,dim=3,cells[3]={1,1,1},max_conv_its,local_sizes[7],run_type=1; DM dm,distdm,basedm; PetscBool flg; char convType[256]; PetscReal Lx,mdisp[10],err[10]; const char * const options[10] = {"-ex56_dm_refine 0", "-ex56_dm_refine 1", "-ex56_dm_refine 2", "-ex56_dm_refine 3", "-ex56_dm_refine 4", "-ex56_dm_refine 5", "-ex56_dm_refine 6", "-ex56_dm_refine 7", "-ex56_dm_refine 8", "-ex56_dm_refine 9"}; PetscFunctionBeginUser; ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr; comm = PETSC_COMM_WORLD; ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr); ierr = MPI_Comm_size(comm, &npe);CHKERRQ(ierr); /* options */ ierr = PetscOptionsBegin(comm,NULL,"3D bilinear Q1 elasticity options","");CHKERRQ(ierr); { i = 3; ierr = PetscOptionsIntArray("-cells", "Number of (flux tube) processor in each dimension", "ex56.c", cells, &i, NULL);CHKERRQ(ierr); Lx = 1.; /* or ne for rod */ max_conv_its = 3; ierr = PetscOptionsInt("-max_conv_its","Number of iterations in convergence study","",max_conv_its,&max_conv_its,NULL);CHKERRQ(ierr); if (max_conv_its<=0 || max_conv_its>7) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_USER, "Bad number of iterations for convergence test (%D)",max_conv_its); ierr = PetscOptionsReal("-lx","Length of domain","",Lx,&Lx,NULL);CHKERRQ(ierr); ierr = PetscOptionsReal("-alpha","material coefficient inside circle","",s_soft_alpha,&s_soft_alpha,NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-test_nonzero_cols","nonzero test","",test_nonzero_cols,&test_nonzero_cols,NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-use_mat_nearnullspace","MatNearNullSpace API test","",use_nearnullspace,&use_nearnullspace,NULL);CHKERRQ(ierr); ierr = PetscOptionsInt("-run_type","0: twisting load on cantalever, 1: 3rd order accurate convergence test","",run_type,&run_type,NULL);CHKERRQ(ierr); i = 3; ierr = PetscOptionsInt("-mat_block_size","","",i,&i,&flg);CHKERRQ(ierr); if (!flg || i!=3) SETERRQ2(PETSC_COMM_WORLD, PETSC_ERR_USER, "'-mat_block_size 3' must be set (%D) and = 3 (%D)",flg,flg? i : 3); } ierr = PetscOptionsEnd();CHKERRQ(ierr); ierr = PetscLogStageRegister("Mesh Setup", &stage[6]);CHKERRQ(ierr); ierr = PetscLogStageRegister("1st Setup", &stage[0]);CHKERRQ(ierr); ierr = PetscLogStageRegister("1st Solve", &stage[1]);CHKERRQ(ierr); /* create DM, Plex calls DMSetup */ ierr = PetscLogStagePush(stage[6]);CHKERRQ(ierr); ierr = DMPlexCreateHexBoxMesh(comm, dim, cells, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, &dm);CHKERRQ(ierr); { DMLabel label; IS is; ierr = DMCreateLabel(dm, "boundary");CHKERRQ(ierr); ierr = DMGetLabel(dm, "boundary", &label);CHKERRQ(ierr); ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr); if (run_type==0) { ierr = DMGetStratumIS(dm, "boundary", 1, &is);CHKERRQ(ierr); ierr = DMCreateLabel(dm,"Faces");CHKERRQ(ierr); if (is) { PetscInt d, f, Nf; const PetscInt *faces; PetscInt csize; PetscSection cs; Vec coordinates ; DM cdm; ierr = ISGetLocalSize(is, &Nf);CHKERRQ(ierr); ierr = ISGetIndices(is, &faces);CHKERRQ(ierr); ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr); ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr); ierr = DMGetDefaultSection(cdm, &cs);CHKERRQ(ierr); /* Check for each boundary face if any component of its centroid is either 0.0 or 1.0 */ for (f = 0; f < Nf; ++f) { PetscReal faceCoord; PetscInt b,v; PetscScalar *coords = NULL; PetscInt Nv; ierr = DMPlexVecGetClosure(cdm, cs, coordinates, faces[f], &csize, &coords);CHKERRQ(ierr); Nv = csize/dim; /* Calculate mean coordinate vector */ for (d = 0; d < dim; ++d) { faceCoord = 0.0; for (v = 0; v < Nv; ++v) faceCoord += PetscRealPart(coords[v*dim+d]); faceCoord /= Nv; for (b = 0; b < 2; ++b) { if (PetscAbs(faceCoord - b) < PETSC_SMALL) { /* domain have not been set yet, still [0,1]^3 */ ierr = DMSetLabelValue(dm, "Faces", faces[f], d*2+b+1);CHKERRQ(ierr); } } } ierr = DMPlexVecRestoreClosure(cdm, cs, coordinates, faces[f], &csize, &coords);CHKERRQ(ierr); } ierr = ISRestoreIndices(is, &faces);CHKERRQ(ierr); } ierr = ISDestroy(&is);CHKERRQ(ierr); ierr = DMGetLabel(dm, "Faces", &label);CHKERRQ(ierr); ierr = DMPlexLabelComplete(dm, label);CHKERRQ(ierr); } } { PetscInt dimEmbed, i; PetscInt nCoords; PetscScalar *coords,bounds[] = {0,Lx,-.5,.5,-.5,.5,}; /* x_min,x_max,y_min,y_max */ Vec coordinates; if (run_type==1) { for (i = 0; i < 2*dim; i++) bounds[i] = (i%2) ? 1 : 0; } ierr = DMGetCoordinatesLocal(dm,&coordinates);CHKERRQ(ierr); ierr = DMGetCoordinateDim(dm,&dimEmbed);CHKERRQ(ierr); if (dimEmbed != dim) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"dimEmbed != dim %D",dimEmbed);CHKERRQ(ierr); ierr = VecGetLocalSize(coordinates,&nCoords);CHKERRQ(ierr); if (nCoords % dimEmbed) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Coordinate vector the wrong size");CHKERRQ(ierr); ierr = VecGetArray(coordinates,&coords);CHKERRQ(ierr); for (i = 0; i < nCoords; i += dimEmbed) { PetscInt j; PetscScalar *coord = &coords[i]; for (j = 0; j < dimEmbed; j++) { coord[j] = bounds[2 * j] + coord[j] * (bounds[2 * j + 1] - bounds[2 * j]); } } ierr = VecRestoreArray(coordinates,&coords);CHKERRQ(ierr); ierr = DMSetCoordinatesLocal(dm,coordinates);CHKERRQ(ierr); } /* convert to p4est, and distribute */ ierr = PetscOptionsBegin(comm, "", "Mesh conversion options", "DMPLEX");CHKERRQ(ierr); ierr = PetscOptionsFList("-dm_type","Convert DMPlex to another format (should not be Plex!)","ex56.c",DMList,DMPLEX,convType,256,&flg);CHKERRQ(ierr); ierr = PetscOptionsEnd(); if (flg) { DM newdm; ierr = DMConvert(dm,convType,&newdm);CHKERRQ(ierr); if (newdm) { const char *prefix; PetscBool isForest; ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr); ierr = PetscObjectSetOptionsPrefix((PetscObject)newdm,prefix);CHKERRQ(ierr); ierr = DMIsForest(newdm,&isForest);CHKERRQ(ierr); if (isForest) { } else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Converted to non Forest?"); ierr = DMDestroy(&dm);CHKERRQ(ierr); dm = newdm; } else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Convert failed?"); } else { /* Plex Distribute mesh over processes */ ierr = DMPlexDistribute(dm, 0, NULL, &distdm);CHKERRQ(ierr); if (distdm) { const char *prefix; ierr = PetscObjectGetOptionsPrefix((PetscObject)dm,&prefix);CHKERRQ(ierr); ierr = PetscObjectSetOptionsPrefix((PetscObject)distdm,prefix);CHKERRQ(ierr); ierr = DMDestroy(&dm);CHKERRQ(ierr); dm = distdm; } } ierr = PetscLogStagePop();CHKERRQ(ierr); basedm = dm; dm = NULL; for (iter=0 ; iter<max_conv_its ; iter++) { ierr = PetscLogStagePush(stage[6]);CHKERRQ(ierr); /* make new DM */ ierr = DMClone(basedm, &dm);CHKERRQ(ierr); ierr = PetscObjectSetOptionsPrefix((PetscObject) dm, "ex56_");CHKERRQ(ierr); ierr = PetscObjectSetName( (PetscObject)dm,"Mesh");CHKERRQ(ierr); ierr = PetscOptionsClearValue(NULL,"-ex56_dm_refine");CHKERRQ(ierr); ierr = PetscOptionsInsertString(NULL,options[iter]);CHKERRQ(ierr); ierr = DMSetFromOptions(dm);CHKERRQ(ierr); /* refinement done here in Plex, p4est */ /* snes */ ierr = SNESCreate(comm, &snes);CHKERRQ(ierr); ierr = SNESSetDM(snes, dm);CHKERRQ(ierr); /* fem */ { const PetscInt Ncomp = dim; const PetscInt components[] = {0,1,2}; const PetscInt Nfid = 1, Npid = 1; const PetscInt fid[] = {1}; /* The fixed faces (x=0) */ const PetscInt pid[] = {2}; /* The faces with loading (x=L_x) */ PetscFE fe; PetscDS prob; DM cdm = dm; ierr = PetscFECreateDefault(dm, dim, dim, PETSC_FALSE, NULL, PETSC_DECIDE, &fe);CHKERRQ(ierr); /* elasticity */ ierr = PetscObjectSetName((PetscObject) fe, "deformation");CHKERRQ(ierr); /* FEM prob */ ierr = DMGetDS(dm, &prob);CHKERRQ(ierr); ierr = PetscDSSetDiscretization(prob, 0, (PetscObject) fe);CHKERRQ(ierr); /* setup problem */ if (run_type==1) { ierr = PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d);CHKERRQ(ierr); ierr = PetscDSSetResidual(prob, 0, f0_u_x4, f1_u_3d);CHKERRQ(ierr); } else { ierr = PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu_3d_alpha);CHKERRQ(ierr); ierr = PetscDSSetResidual(prob, 0, f0_u, f1_u_3d_alpha);CHKERRQ(ierr); ierr = PetscDSSetBdResidual(prob, 0, f0_bd_u_3d, f1_bd_u);CHKERRQ(ierr); } /* bcs */ if (run_type==1) { PetscInt id = 1; ierr = DMAddBoundary(dm, DM_BC_ESSENTIAL, "wall", "boundary", 0, 0, NULL, (void (*)()) zero, 1, &id, NULL);CHKERRQ(ierr); } else { ierr = PetscDSAddBoundary(prob, DM_BC_ESSENTIAL, "fixed", "Faces", 0, Ncomp, components, (void (*)()) zero, Nfid, fid, NULL);CHKERRQ(ierr); ierr = PetscDSAddBoundary(prob, DM_BC_NATURAL, "traction", "Faces", 0, Ncomp, components, NULL, Npid, pid, NULL);CHKERRQ(ierr); } while (cdm) { ierr = DMSetDS(cdm,prob);CHKERRQ(ierr); ierr = DMGetCoarseDM(cdm, &cdm);CHKERRQ(ierr); } ierr = PetscFEDestroy(&fe);CHKERRQ(ierr); } /* vecs & mat */ ierr = DMCreateGlobalVector(dm,&xx);CHKERRQ(ierr); ierr = VecDuplicate(xx, &bb);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject) bb, "b");CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject) xx, "u");CHKERRQ(ierr); ierr = DMCreateMatrix(dm, &Amat);CHKERRQ(ierr); ierr = VecGetSize(bb,&N);CHKERRQ(ierr); local_sizes[iter] = N; ierr = PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %d global equations, %d vertices\n",rank,PETSC_FUNCTION_NAME,N,N/dim);CHKERRQ(ierr); if (use_nearnullspace && N/dim > 1) { /* Set up the near null space (a.k.a. rigid body modes) that will be used by the multigrid preconditioner */ DM subdm; MatNullSpace nearNullSpace; PetscInt fields = 0; PetscObject deformation; ierr = DMCreateSubDM(dm, 1, &fields, NULL, &subdm);CHKERRQ(ierr); ierr = DMPlexCreateRigidBody(subdm, &nearNullSpace);CHKERRQ(ierr); ierr = DMGetField(dm, 0, &deformation);CHKERRQ(ierr); ierr = PetscObjectCompose(deformation, "nearnullspace", (PetscObject) nearNullSpace);CHKERRQ(ierr); ierr = DMDestroy(&subdm);CHKERRQ(ierr); ierr = MatNullSpaceDestroy(&nearNullSpace);CHKERRQ(ierr); /* created by DM and destroyed by Mat */ } ierr = DMPlexSetSNESLocalFEM(dm,NULL,NULL,NULL);CHKERRQ(ierr); ierr = SNESSetJacobian(snes, Amat, Amat, NULL, NULL);CHKERRQ(ierr); ierr = SNESSetFromOptions(snes);CHKERRQ(ierr); ierr = DMSetUp(dm);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr); /* ksp */ ierr = SNESGetKSP(snes, &ksp);CHKERRQ(ierr); ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr); /* test BCs */ ierr = VecZeroEntries(xx);CHKERRQ(ierr); if (test_nonzero_cols) { if (rank==0) ierr = VecSetValue(xx,0,1.0,INSERT_VALUES);CHKERRQ(ierr); ierr = VecAssemblyBegin(xx);CHKERRQ(ierr); ierr = VecAssemblyEnd(xx);CHKERRQ(ierr); } ierr = VecZeroEntries(bb);CHKERRQ(ierr); ierr = VecGetSize(bb,&i);CHKERRQ(ierr); local_sizes[iter] = i; ierr = PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %d equations in vector, %d vertices\n",rank,PETSC_FUNCTION_NAME,i,i/dim);CHKERRQ(ierr); /* setup solver, dummy solve to really setup */ if (0) { ierr = KSPSetTolerances(ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1);CHKERRQ(ierr); ierr = SNESSolve(snes, bb, xx);CHKERRQ(ierr); ierr = KSPSetTolerances(ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,50);CHKERRQ(ierr); ierr = VecZeroEntries(xx);CHKERRQ(ierr); } ierr = PetscLogStagePop();CHKERRQ(ierr); /* solve */ ierr = PetscLogStagePush(stage[1]);CHKERRQ(ierr); ierr = SNESSolve(snes, bb, xx);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = VecNorm(xx,NORM_INFINITY,&mdisp[iter]);CHKERRQ(ierr); ierr = DMViewFromOptions(dm, NULL, "-dm_view");CHKERRQ(ierr); { PetscViewer viewer = NULL; PetscViewerFormat fmt; ierr = PetscOptionsGetViewer(comm,"ex56_","-vec_view",&viewer,&fmt,&flg);CHKERRQ(ierr); if (flg) { ierr = PetscViewerPushFormat(viewer,fmt);CHKERRQ(ierr); ierr = VecView(xx,viewer);CHKERRQ(ierr); ierr = VecView(bb,viewer);CHKERRQ(ierr); ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr); } ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); } /* Free work space */ ierr = DMDestroy(&dm);CHKERRQ(ierr); ierr = SNESDestroy(&snes);CHKERRQ(ierr); ierr = VecDestroy(&xx);CHKERRQ(ierr); ierr = VecDestroy(&bb);CHKERRQ(ierr); ierr = MatDestroy(&Amat);CHKERRQ(ierr); } ierr = DMDestroy(&basedm);CHKERRQ(ierr); if (run_type==1) { err[0] = 59.975208 - mdisp[0]; /* error with what I think is the exact solution */ } else { err[0] = 171.038 - mdisp[0]; } for (iter=1 ; iter<max_conv_its ; iter++) { if (run_type==1) { err[iter] = 59.975208 - mdisp[iter]; } else { err[iter] = 171.038 - mdisp[iter]; } PetscPrintf(PETSC_COMM_WORLD,"[%d]%s %D) N=%12D, max displ=%9.7e, disp diff=%9.2e, error=%4.3e, rate=%3.2g\n", rank,PETSC_FUNCTION_NAME,iter,local_sizes[iter],mdisp[iter], mdisp[iter]-mdisp[iter-1],err[iter],log(err[iter-1]/err[iter])/log(2.)); } ierr = PetscFinalize(); return ierr; }
int main(int argc,char ** argv) { PetscErrorCode ierr; char pfdata_file[PETSC_MAX_PATH_LEN]="datafiles/case9.m"; PFDATA *pfdata; PetscInt numEdges=0,numVertices=0; int *edges = NULL; PetscInt i; DM networkdm; PetscInt componentkey[4]; UserCtx User; PetscLogStage stage1,stage2; PetscMPIInt size,rank; PetscInt eStart, eEnd, vStart, vEnd,j; PetscInt genj,loadj; Vec X,F; Mat J; SNES snes; ierr = PetscInitialize(&argc,&argv,"pfoptions",help);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); { /* introduce the const crank so the clang static analyzer realizes that if it enters any of the if (crank) then it must have entered the first */ /* this is an experiment to see how the analyzer reacts */ const PetscMPIInt crank = rank; /* Create an empty network object */ ierr = DMNetworkCreate(PETSC_COMM_WORLD,&networkdm);CHKERRQ(ierr); /* Register the components in the network */ ierr = DMNetworkRegisterComponent(networkdm,"branchstruct",sizeof(struct _p_EDGEDATA),&componentkey[0]);CHKERRQ(ierr); ierr = DMNetworkRegisterComponent(networkdm,"busstruct",sizeof(struct _p_VERTEXDATA),&componentkey[1]);CHKERRQ(ierr); ierr = DMNetworkRegisterComponent(networkdm,"genstruct",sizeof(struct _p_GEN),&componentkey[2]);CHKERRQ(ierr); ierr = DMNetworkRegisterComponent(networkdm,"loadstruct",sizeof(struct _p_LOAD),&componentkey[3]);CHKERRQ(ierr); ierr = PetscLogStageRegister("Read Data",&stage1);CHKERRQ(ierr); PetscLogStagePush(stage1); /* READ THE DATA */ if (!crank) { /* READ DATA */ /* Only rank 0 reads the data */ ierr = PetscOptionsGetString(NULL,NULL,"-pfdata",pfdata_file,PETSC_MAX_PATH_LEN-1,NULL);CHKERRQ(ierr); ierr = PetscNew(&pfdata);CHKERRQ(ierr); ierr = PFReadMatPowerData(pfdata,pfdata_file);CHKERRQ(ierr); User.Sbase = pfdata->sbase; numEdges = pfdata->nbranch; numVertices = pfdata->nbus; ierr = PetscMalloc(2*numEdges*sizeof(int),&edges);CHKERRQ(ierr); ierr = GetListofEdges(pfdata->nbranch,pfdata->branch,edges);CHKERRQ(ierr); } PetscLogStagePop(); ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); ierr = PetscLogStageRegister("Create network",&stage2);CHKERRQ(ierr); PetscLogStagePush(stage2); /* Set number of nodes/edges */ ierr = DMNetworkSetSizes(networkdm,numVertices,numEdges,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr); /* Add edge connectivity */ ierr = DMNetworkSetEdgeList(networkdm,edges);CHKERRQ(ierr); /* Set up the network layout */ ierr = DMNetworkLayoutSetUp(networkdm);CHKERRQ(ierr); if (!crank) { ierr = PetscFree(edges);CHKERRQ(ierr); } /* Add network components only process 0 has any data to add*/ if (!crank) { genj=0; loadj=0; ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr); for (i = eStart; i < eEnd; i++) { ierr = DMNetworkAddComponent(networkdm,i,componentkey[0],&pfdata->branch[i-eStart]);CHKERRQ(ierr); } ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr); for (i = vStart; i < vEnd; i++) { ierr = DMNetworkAddComponent(networkdm,i,componentkey[1],&pfdata->bus[i-vStart]);CHKERRQ(ierr); if (pfdata->bus[i-vStart].ngen) { for (j = 0; j < pfdata->bus[i-vStart].ngen; j++) { ierr = DMNetworkAddComponent(networkdm,i,componentkey[2],&pfdata->gen[genj++]);CHKERRQ(ierr); } } if (pfdata->bus[i-vStart].nload) { for (j=0; j < pfdata->bus[i-vStart].nload; j++) { ierr = DMNetworkAddComponent(networkdm,i,componentkey[3],&pfdata->load[loadj++]);CHKERRQ(ierr); } } /* Add number of variables */ ierr = DMNetworkAddNumVariables(networkdm,i,2);CHKERRQ(ierr); } } /* Set up DM for use */ ierr = DMSetUp(networkdm);CHKERRQ(ierr); if (!crank) { ierr = PetscFree(pfdata->bus);CHKERRQ(ierr); ierr = PetscFree(pfdata->gen);CHKERRQ(ierr); ierr = PetscFree(pfdata->branch);CHKERRQ(ierr); ierr = PetscFree(pfdata->load);CHKERRQ(ierr); ierr = PetscFree(pfdata);CHKERRQ(ierr); } ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); if (size > 1) { DM distnetworkdm; /* Network partitioning and distribution of data */ ierr = DMNetworkDistribute(networkdm,0,&distnetworkdm);CHKERRQ(ierr); ierr = DMDestroy(&networkdm);CHKERRQ(ierr); networkdm = distnetworkdm; } PetscLogStagePop(); ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr); ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr); #if 0 PetscInt numComponents; EDGEDATA edge; PetscInt offset,key,kk; DMNetworkComponentGenericDataType *arr; VERTEXDATA bus; GEN gen; LOAD load; for (i = eStart; i < eEnd; i++) { ierr = DMNetworkGetComponentDataArray(networkdm,&arr);CHKERRQ(ierr); ierr = DMNetworkGetComponentTypeOffset(networkdm,i,0,&key,&offset);CHKERRQ(ierr); edge = (EDGEDATA)(arr+offset); ierr = DMNetworkGetNumComponents(networkdm,i,&numComponents);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d ncomps = %d Line %d ---- %d\n",crank,numComponents,edge->internal_i,edge->internal_j);CHKERRQ(ierr); } for (i = vStart; i < vEnd; i++) { ierr = DMNetworkGetComponentDataArray(networkdm,&arr);CHKERRQ(ierr); ierr = DMNetworkGetNumComponents(networkdm,i,&numComponents);CHKERRQ(ierr); for (kk=0; kk < numComponents; kk++) { ierr = DMNetworkGetComponentTypeOffset(networkdm,i,kk,&key,&offset);CHKERRQ(ierr); if (key == 1) { bus = (VERTEXDATA)(arr+offset); ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d ncomps = %d Bus %d\n",crank,numComponents,bus->internal_i);CHKERRQ(ierr); } else if (key == 2) { gen = (GEN)(arr+offset); ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d Gen pg = %f qg = %f\n",crank,gen->pg,gen->qg);CHKERRQ(ierr); } else if (key == 3) { load = (LOAD)(arr+offset); ierr = PetscPrintf(PETSC_COMM_SELF,"Rank %d Load pl = %f ql = %f\n",crank,load->pl,load->ql);CHKERRQ(ierr); } } } #endif /* Broadcast Sbase to all processors */ ierr = MPI_Bcast(&User.Sbase,1,MPIU_SCALAR,0,PETSC_COMM_WORLD);CHKERRQ(ierr); ierr = DMCreateGlobalVector(networkdm,&X);CHKERRQ(ierr); ierr = VecDuplicate(X,&F);CHKERRQ(ierr); ierr = DMCreateMatrix(networkdm,&J);CHKERRQ(ierr); ierr = MatSetOption(J,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr); ierr = SetInitialValues(networkdm,X,&User);CHKERRQ(ierr); /* HOOK UP SOLVER */ ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr); ierr = SNESSetDM(snes,networkdm);CHKERRQ(ierr); ierr = SNESSetFunction(snes,F,FormFunction,&User);CHKERRQ(ierr); ierr = SNESSetJacobian(snes,J,J,FormJacobian,&User);CHKERRQ(ierr); ierr = SNESSetFromOptions(snes);CHKERRQ(ierr); ierr = SNESSolve(snes,NULL,X);CHKERRQ(ierr); ierr = VecDestroy(&X);CHKERRQ(ierr); ierr = VecDestroy(&F);CHKERRQ(ierr); ierr = MatDestroy(&J);CHKERRQ(ierr); ierr = SNESDestroy(&snes);CHKERRQ(ierr); ierr = DMDestroy(&networkdm);CHKERRQ(ierr); } ierr = PetscFinalize(); return ierr; }
int main(int argc,char **args) { Mat A,Atrans,sA,*submatA,*submatsA; PetscErrorCode ierr; PetscMPIInt size,rank; PetscInt bs=1,mbs=10,ov=1,i,j,k,*rows,*cols,nd=2,*idx,rstart,rend,sz,M,N,Mbs; PetscScalar *vals,rval,one=1.0; IS *is1,*is2; PetscRandom rand; PetscBool flg,TestOverlap,TestSubMat,TestAllcols,test_sorted=PETSC_FALSE; PetscInt vid = -1; #if defined(PETSC_USE_LOG) PetscLogStage stages[2]; #endif ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr; ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-mat_mbs",&mbs,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-view_id",&vid,NULL);CHKERRQ(ierr); ierr = PetscOptionsHasName(NULL,NULL, "-test_overlap", &TestOverlap);CHKERRQ(ierr); ierr = PetscOptionsHasName(NULL,NULL, "-test_submat", &TestSubMat);CHKERRQ(ierr); ierr = PetscOptionsHasName(NULL,NULL, "-test_allcols", &TestAllcols);CHKERRQ(ierr); ierr = PetscOptionsGetBool(NULL,NULL,"-test_sorted",&test_sorted,NULL);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,mbs*bs,mbs*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr); ierr = MatSetType(A,MATBAIJ);CHKERRQ(ierr); ierr = MatSeqBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr); ierr = MatMPIBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr); ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr); ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr); Mbs = M/bs; ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr); ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr); ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr); ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr); /* Now set blocks of values */ for (j=0; j<bs*bs; j++) vals[j] = 0.0; for (i=0; i<Mbs; i++) { cols[0] = i*bs; rows[0] = i*bs; for (j=1; j<bs; j++) { rows[j] = rows[j-1]+1; cols[j] = cols[j-1]+1; } ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr); } /* second, add random blocks */ for (i=0; i<20*bs; i++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); cols[0] = bs*(PetscInt)(PetscRealPart(rval)*Mbs); ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); rows[0] = rstart + bs*(PetscInt)(PetscRealPart(rval)*mbs); for (j=1; j<bs; j++) { rows[j] = rows[j-1]+1; cols[j] = cols[j-1]+1; } for (j=0; j<bs*bs; j++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); vals[j] = rval; } ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr); } ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* make A a symmetric matrix: A <- A^T + A */ ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr); ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); ierr = MatDestroy(&Atrans);CHKERRQ(ierr); ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr); ierr = MatEqual(A, Atrans, &flg);CHKERRQ(ierr); if (flg) { ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr); } else SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric"); ierr = MatDestroy(&Atrans);CHKERRQ(ierr); /* create a SeqSBAIJ matrix sA (= A) */ ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr); if (vid >= 0 && vid < size) { if (!rank) printf("A: \n"); ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); if (!rank) printf("sA: \n"); ierr = MatView(sA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Test sA==A through MatMult() */ ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Error in MatConvert(): A != sA"); /* Test MatIncreaseOverlap() */ ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr); ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr); for (i=0; i<nd; i++) { if (!TestAllcols) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); sz = (PetscInt)((0.5+0.2*PetscRealPart(rval))*mbs); /* 0.5*mbs < sz < 0.7*mbs */ for (j=0; j<sz; j++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); idx[j*bs] = bs*(PetscInt)(PetscRealPart(rval)*Mbs); for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k; } ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr); ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr); if (rank == vid) { ierr = PetscPrintf(PETSC_COMM_SELF," [%d] IS sz[%d]: %d\n",rank,i,sz);CHKERRQ(ierr); ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } } else { /* Test all rows and colums */ sz = M; ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is1+i);CHKERRQ(ierr); ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is2+i);CHKERRQ(ierr); if (rank == vid) { PetscBool colflag; ierr = ISIdentity(is2[i],&colflag);CHKERRQ(ierr); printf("[%d] is2[%d], colflag %d\n",rank,(int)i,(int)colflag); ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } } } ierr = PetscLogStageRegister("MatOv_SBAIJ",&stages[0]);CHKERRQ(ierr); ierr = PetscLogStageRegister("MatOv_BAIJ",&stages[1]);CHKERRQ(ierr); /* Test MatIncreaseOverlap */ if (TestOverlap) { ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr); ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr); ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); if (rank == vid) { printf("\n[%d] IS from BAIJ:\n",rank); ierr = ISView(is1[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); printf("\n[%d] IS from SBAIJ:\n",rank); ierr = ISView(is2[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } for (i=0; i<nd; ++i) { ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr); if (!flg) { if (!rank) { ierr = ISSort(is1[i]);CHKERRQ(ierr); /* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */ ierr = ISSort(is2[i]);CHKERRQ(ierr); /* ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */ } SETERRQ1(PETSC_COMM_SELF,1,"i=%D, is1 != is2",i); } } } /* Test MatCreateSubmatrices */ if (TestSubMat) { if (test_sorted) { for (i = 0; i < nd; ++i) { ierr = ISSort(is1[i]);CHKERRQ(ierr); } } ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr); ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr); ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"A != sA"); /* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */ ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr); ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr); ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"MatCreateSubmatrices(): A != sA"); ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr); ierr = MatDestroySubMatrices(nd,&submatsA);CHKERRQ(ierr); } /* Free allocated memory */ for (i=0; i<nd; ++i) { ierr = ISDestroy(&is1[i]);CHKERRQ(ierr); ierr = ISDestroy(&is2[i]);CHKERRQ(ierr); } ierr = PetscFree(is1);CHKERRQ(ierr); ierr = PetscFree(is2);CHKERRQ(ierr); ierr = PetscFree(idx);CHKERRQ(ierr); ierr = PetscFree(rows);CHKERRQ(ierr); ierr = PetscFree(cols);CHKERRQ(ierr); ierr = PetscFree(vals);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = MatDestroy(&sA);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options) { const char *dShapes[2] = {"box", "cylinder"}; PetscInt shape, bd, n; static PetscInt domainBoxSizes[3] = {1,1,1}; static PetscReal domainBoxL[3] = {0.,0.,0.}; static PetscReal domainBoxU[3] = {1.,1.,1.}; PetscBool flg; PetscErrorCode ierr; PetscFunctionBegin; options->debug = 0; options->dim = 2; options->interpolate = PETSC_FALSE; options->refinementLimit = 0.0; options->cellSimplex = PETSC_TRUE; options->cellWedge = PETSC_FALSE; options->domainShape = BOX; options->domainBoxSizes = NULL; options->domainBoxL = NULL; options->domainBoxU = NULL; options->periodicity[0] = DM_BOUNDARY_NONE; options->periodicity[1] = DM_BOUNDARY_NONE; options->periodicity[2] = DM_BOUNDARY_NONE; options->filename[0] = '\0'; options->bdfilename[0] = '\0'; options->extfilename[0] = '\0'; options->testPartition = PETSC_FALSE; options->overlap = PETSC_FALSE; options->testShape = PETSC_FALSE; options->simplex2tensor = PETSC_FALSE; options->extrude_layers = 2; options->extrude_thickness = 0.1; options->testp4est[0] = PETSC_FALSE; options->testp4est[1] = PETSC_FALSE; ierr = PetscOptionsBegin(comm, "", "Meshing Problem Options", "DMPLEX");CHKERRQ(ierr); ierr = PetscOptionsInt("-debug", "The debugging level", "ex1.c", options->debug, &options->debug, NULL);CHKERRQ(ierr); ierr = PetscOptionsInt("-dim", "The topological mesh dimension", "ex1.c", options->dim, &options->dim, NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-interpolate", "Generate intermediate mesh elements", "ex1.c", options->interpolate, &options->interpolate, NULL);CHKERRQ(ierr); ierr = PetscOptionsReal("-refinement_limit", "The largest allowable cell volume", "ex1.c", options->refinementLimit, &options->refinementLimit, NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-cell_simplex", "Use simplices if true, otherwise hexes", "ex1.c", options->cellSimplex, &options->cellSimplex, NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-cell_wedge", "Use wedges if true", "ex1.c", options->cellWedge, &options->cellWedge, NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-simplex2tensor", "Refine simplicial cells in tensor product cells", "ex1.c", options->simplex2tensor, &options->simplex2tensor, NULL);CHKERRQ(ierr); if (options->simplex2tensor) options->interpolate = PETSC_TRUE; shape = options->domainShape; ierr = PetscOptionsEList("-domain_shape","The shape of the domain","ex1.c", dShapes, 2, dShapes[options->domainShape], &shape, NULL);CHKERRQ(ierr); options->domainShape = (DomainShape) shape; ierr = PetscOptionsIntArray("-domain_box_sizes","The sizes of the box domain","ex1.c", domainBoxSizes, (n=3,&n), &flg);CHKERRQ(ierr); if (flg) { options->domainShape = BOX; options->domainBoxSizes = domainBoxSizes;} ierr = PetscOptionsRealArray("-domain_box_ll","Coordinates of the lower left corner of the box domain","ex1.c", domainBoxL, (n=3,&n), &flg);CHKERRQ(ierr); if (flg) { options->domainBoxL = domainBoxL;} ierr = PetscOptionsRealArray("-domain_box_ur","Coordinates of the upper right corner of the box domain","ex1.c", domainBoxU, (n=3,&n), &flg);CHKERRQ(ierr); if (flg) { options->domainBoxU = domainBoxU;} bd = options->periodicity[0]; ierr = PetscOptionsEList("-x_periodicity", "The x-boundary periodicity", "ex1.c", DMBoundaryTypes, 5, DMBoundaryTypes[options->periodicity[0]], &bd, NULL);CHKERRQ(ierr); options->periodicity[0] = (DMBoundaryType) bd; bd = options->periodicity[1]; ierr = PetscOptionsEList("-y_periodicity", "The y-boundary periodicity", "ex1.c", DMBoundaryTypes, 5, DMBoundaryTypes[options->periodicity[1]], &bd, NULL);CHKERRQ(ierr); options->periodicity[1] = (DMBoundaryType) bd; bd = options->periodicity[2]; ierr = PetscOptionsEList("-z_periodicity", "The z-boundary periodicity", "ex1.c", DMBoundaryTypes, 5, DMBoundaryTypes[options->periodicity[2]], &bd, NULL);CHKERRQ(ierr); options->periodicity[2] = (DMBoundaryType) bd; ierr = PetscOptionsString("-filename", "The mesh file", "ex1.c", options->filename, options->filename, PETSC_MAX_PATH_LEN, NULL);CHKERRQ(ierr); ierr = PetscOptionsString("-bd_filename", "The mesh boundary file", "ex1.c", options->bdfilename, options->bdfilename, PETSC_MAX_PATH_LEN, NULL);CHKERRQ(ierr); ierr = PetscOptionsString("-ext_filename", "The 2D mesh file to be extruded", "ex1.c", options->extfilename, options->extfilename, PETSC_MAX_PATH_LEN, NULL);CHKERRQ(ierr); ierr = PetscOptionsInt("-ext_layers", "The number of layers to extrude", "ex1.c", options->extrude_layers, &options->extrude_layers, NULL);CHKERRQ(ierr); ierr = PetscOptionsReal("-ext_thickness", "The thickness of the layer to be extruded", "ex1.c", options->extrude_thickness, &options->extrude_thickness, NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-test_partition", "Use a fixed partition for testing", "ex1.c", options->testPartition, &options->testPartition, NULL);CHKERRQ(ierr); ierr = PetscOptionsInt("-overlap", "The cell overlap for partitioning", "ex1.c", options->overlap, &options->overlap, NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-test_shape", "Report cell shape qualities (Jacobian condition numbers)", "ex1.c", options->testShape, &options->testShape, NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-test_p4est_seq", "Test p4est with sequential base DM", "ex1.c", options->testp4est[0], &options->testp4est[0], NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-test_p4est_par", "Test p4est with parallel base DM", "ex1.c", options->testp4est[1], &options->testp4est[1], NULL);CHKERRQ(ierr); ierr = PetscOptionsEnd(); ierr = PetscLogEventRegister("CreateMesh", DM_CLASSID, &options->createMeshEvent);CHKERRQ(ierr); ierr = PetscLogStageRegister("MeshLoad", &options->stages[STAGE_LOAD]);CHKERRQ(ierr); ierr = PetscLogStageRegister("MeshDistribute", &options->stages[STAGE_DISTRIBUTE]);CHKERRQ(ierr); ierr = PetscLogStageRegister("MeshRefine", &options->stages[STAGE_REFINE]);CHKERRQ(ierr); ierr = PetscLogStageRegister("MeshOverlap", &options->stages[STAGE_OVERLAP]);CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc, char ** argv ) { int size, rank; bool incCorner = 1; unsigned int dim=3; unsigned int maxDepth=29; bool compressLut=true; std::vector<ot::TreeNode> balOct; double mgLoadFac = 2.0; unsigned int regLev = 2; PetscInitialize(&argc, &argv, "options", help); ot::RegisterEvents(); ot::DAMG_Initialize(MPI_COMM_WORLD); #ifdef PETSC_USE_LOG PetscClassId classid; PetscClassIdRegister("Dendro",&classid); PetscLogEventRegister("matProp",classid, &matPropEvent); PetscLogEventRegister("ODAmatDiag",classid, &Jac1DiagEvent); PetscLogEventRegister("ODAmatMult",classid, &Jac1MultEvent); PetscLogEventRegister("ODAmatDiagFinest",classid, &Jac1FinestDiagEvent); PetscLogEventRegister("ODAmatMultFinest",classid, &Jac1FinestMultEvent); PetscLogEventRegister("OMGmatDiag-2",classid, &Jac2DiagEvent); PetscLogEventRegister("OMGmatMult-2",classid, &Jac2MultEvent); PetscLogEventRegister("OMGmatDiagFinest-2",classid, &Jac2FinestDiagEvent); PetscLogEventRegister("OMGmatMultFinest-2",classid, &Jac2FinestMultEvent); PetscLogEventRegister("OMGmatDiag-3",classid, &Jac3DiagEvent); PetscLogEventRegister("OMGmatMult-3",classid, &Jac3MultEvent); PetscLogEventRegister("OMGmatDiagFinest-3",classid, &Jac3FinestDiagEvent); PetscLogEventRegister("OMGmatMultFinest-3",classid, &Jac3FinestMultEvent); int stages[1]; PetscLogStageRegister("Solve",&stages[0]); #endif MPI_Comm_size(MPI_COMM_WORLD,&size); MPI_Comm_rank(MPI_COMM_WORLD,&rank); if(argc > 1) { regLev = atoi(argv[1]); } if(argc > 2) { maxDepth = atoi(argv[2]); } if(argc > 3) { dim = atoi(argv[3]); } if(argc > 4) { incCorner = (bool)(atoi(argv[4]));} if(argc > 5) { compressLut = (bool)(atoi(argv[5]));} if(argc > 6) { mgLoadFac = atof(argv[6]); } #ifdef PETSC_USE_LOG PetscLogStagePush(stages[0]); #endif MPI_Barrier(MPI_COMM_WORLD); ot::DAMG *damg; int nlevels = 1; //number of multigrid levels unsigned int dof = 1; // degrees of freedom per node createRegularOctree(balOct, regLev, dim, maxDepth, MPI_COMM_WORLD); PetscInt nlevelsPetscInt = nlevels; //To keep the compilers happy when using 64-bit indices PetscOptionsGetInt(0, "-nlevels", &nlevelsPetscInt, 0); nlevels = nlevelsPetscInt; // Note: The user context for all levels will be set separately later. MPI_Barrier(MPI_COMM_WORLD); if(!rank) { std::cout<<" nlevels initial: "<<nlevels<<std::endl; } ot::DAMGCreateAndSetDA(PETSC_COMM_WORLD, nlevels, NULL, &damg, balOct, dof, mgLoadFac, compressLut, incCorner); if(!rank) { std::cout<<" nlevels final: "<<nlevels<<std::endl; } MPI_Barrier(MPI_COMM_WORLD); if(!rank) { std::cout << "Created DA for all levels."<< std::endl; } MPI_Barrier(MPI_COMM_WORLD); ot::PrintDAMG(damg); MPI_Barrier(MPI_COMM_WORLD); for(int i=0;i<nlevels;i++) { bool isRegOct = isRegularGrid(damg[i]->da); if(!rank) { std::cout<<"Level "<<i<<" is regular? "<<isRegOct<<std::endl; } }//end for i SetUserContexts(damg); if(!rank) { std::cout << "Set User Contexts all levels."<< std::endl; } MPI_Barrier(MPI_COMM_WORLD); PetscInt jacType = 1; PetscOptionsGetInt(0,"-jacType",&jacType,0); PetscInt rhsType = 1; PetscOptionsGetInt(0,"-rhsType",&rhsType,0); createLmatType2(LaplacianType2Stencil); createMmatType2(MassType2Stencil); if(jacType == 3) { createLmatType1(LaplacianType1Stencil); createMmatType1(MassType1Stencil); } createShFnMat(ShapeFnStencil); if(!rank) { std::cout << "Created Stencils."<< std::endl; } //Function handles PetscErrorCode (*ComputeRHSHandle)(ot::DAMG damg,Vec rhs) = NULL; PetscErrorCode (*CreateJacobianHandle)(ot::DAMG damg,Mat *B) = NULL; PetscErrorCode (*ComputeJacobianHandle)(ot::DAMG damg,Mat J, Mat B) = NULL; if(rhsType == 0) { ComputeRHSHandle = ComputeRHS0; } else if (rhsType == 1) { ComputeRHSHandle = ComputeRHS1; } else if (rhsType == 2) { ComputeRHSHandle = ComputeRHS2; } else if (rhsType == 3) { ComputeRHSHandle = ComputeRHS3; } else if (rhsType == 4) { ComputeRHSHandle = ComputeRHS4; } else if (rhsType == 5) { ComputeRHSHandle = ComputeRHS5; } else if (rhsType == 6) { ComputeRHSHandle = ComputeRHS6; } else if (rhsType == 7) { ComputeRHSHandle = ComputeRHS7; } else if (rhsType == 8) { ComputeRHSHandle = ComputeRHS8; } else { assert(false); } if(jacType == 1) { CreateJacobianHandle = CreateJacobian1; ComputeJacobianHandle = ComputeJacobian1; } else if (jacType == 2) { CreateJacobianHandle = CreateJacobian2; ComputeJacobianHandle = ComputeJacobian2; } else if (jacType == 3) { CreateJacobianHandle = CreateJacobian3; ComputeJacobianHandle = ComputeJacobian3; //Skip the finest and the coarsest levels. For the other levels, J and B //must be different for(int i = 1; i < (nlevels-1); i++) { ot::DAMGCreateJMatrix(damg[i], CreateJacobianHandle); } } else { assert(false); } //Global Function Handles for using KSP_Shell (will be used @ the coarsest grid if not all //processors are active on the coarsest grid) if (jacType == 1) { ot::getPrivateMatricesForKSP_Shell = getPrivateMatricesForKSP_Shell_Jac1; } else if (jacType == 2) { ot::getPrivateMatricesForKSP_Shell = getPrivateMatricesForKSP_Shell_Jac2; } else if (jacType == 3) { ot::getPrivateMatricesForKSP_Shell = getPrivateMatricesForKSP_Shell_Jac3; } else { assert(false); } ot::DAMGSetKSP(damg, CreateJacobianHandle, ComputeJacobianHandle, ComputeRHSHandle); if(!rank) { std::cout<<"Solving u-Lu=f"<<std::endl; } iC(ot::DAMGSolve(damg)); destroyLmatType2(LaplacianType2Stencil); destroyMmatType2(MassType2Stencil); if(jacType == 3) { destroyLmatType1(LaplacianType1Stencil); destroyMmatType1(MassType1Stencil); } destroyShFnMat(ShapeFnStencil); MPI_Barrier(MPI_COMM_WORLD); DestroyUserContexts(damg); if (!rank) { std::cout << GRN << "Destroyed User Contexts." << NRM << std::endl; } MPI_Barrier(MPI_COMM_WORLD); iC(DAMGDestroy(damg)); if (!rank) { std::cout << GRN << "Destroyed DAMG" << NRM << std::endl; } #ifdef PETSC_USE_LOG PetscLogStagePop(); #endif balOct.clear(); if (!rank) { std::cout << GRN << "Finalizing ..." << NRM << std::endl; } ot::DAMG_Finalize(); PetscFinalize(); }//end function
int main(int argc, char **argv){ PetscErrorCode ierr; int nx = 63, ny = 63; DM dm; PetscBool flg; Mat A; Vec u, b; KSP solver; PC pc; double norm; PetscInt stage; ierr = PetscInitialize(&argc, &argv, NULL, NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL, "-nx", &nx, PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL, "-ny", &ny, PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsHasName(PETSC_NULL, "-assemble", &flg);CHKERRQ(ierr); ierr = PetscLogStageRegister("preparing",&stage); CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = PetscLogStageRegister("Domain creation",&stage); CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = createDomain(&dm, nx, ny);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStageRegister("matrix creation",&stage); CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = createMat(dm, &A, flg);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStageRegister("Vector creation",&stage); CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = DMCreateGlobalVector(dm, &b);CHKERRQ(ierr); ierr = VecDuplicate(b, &u);CHKERRQ(ierr); ierr = PetscLogStageRegister("Domain initialisation",&stage); CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = init2d(dm, b);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStageRegister("solver creation",&stage); CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = KSPCreate(PETSC_COMM_WORLD, &solver);CHKERRQ(ierr); ierr = KSPSetOptionsPrefix(solver, "poisson_");CHKERRQ(ierr); ierr = KSPSetOperators(solver, A, A, DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); ierr = KSPSetType(solver, KSPCG); ierr = KSPGetPC(solver, &pc);CHKERRQ(ierr); ierr = PCSetType(pc, PCNONE);CHKERRQ(ierr); ierr = KSPSetFromOptions(solver);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStageRegister("Solving",&stage); CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = KSPSolve(solver, b, u);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); //ierr = VecView(u, PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr); //sleep(10); VecDestroy(&u); VecDestroy(&b); MatDestroy(&A); DMDestroy(&dm); KSPDestroy(&solver); ierr = PetscFinalize(); return 0; }
int main(int argc,char **argv) { Mat A,B,C,PtAP,PtAP_copy,PtAP_squared; PetscInt i,M,N,Istart,Iend,n=7,j,J,Ii,m=8,k,o=1; PetscScalar v; PetscErrorCode ierr; PetscBool equal=PETSC_FALSE,mat_view=PETSC_FALSE; char stencil[PETSC_MAX_PATH_LEN]; #if defined(PETSC_USE_LOG) PetscLogStage fullMatMatMultStage; #endif ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-o",&o,NULL);CHKERRQ(ierr); ierr = PetscOptionsHasName(NULL,NULL,"-result_view",&mat_view);CHKERRQ(ierr); ierr = PetscOptionsGetString(NULL,NULL,"-stencil",stencil,PETSC_MAX_PATH_LEN,NULL);CHKERRQ(ierr); /* Create a aij matrix A */ M = N = m*n*o; ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,M,N);CHKERRQ(ierr); ierr = MatSetType(A,MATAIJ);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); /* Consistency checks */ if (o < 1 || m < 1 || n < 1) SETERRQ(PETSC_COMM_WORLD,1,"Dimensions need to be larger than zero!"); /************ 2D stencils ***************/ ierr = PetscStrcmp(stencil, "2d5point", &equal);CHKERRQ(ierr); if (equal) { /* 5-point stencil, 2D */ ierr = MatMPIAIJSetPreallocation(A,5,NULL,5,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,5,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); if (i>0) {J = global_index(i-1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = global_index(i+1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = global_index(i,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = global_index(i,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 4.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = PetscStrcmp(stencil, "2d9point", &equal);CHKERRQ(ierr); if (equal) { /* 9-point stencil, 2D */ ierr = MatMPIAIJSetPreallocation(A,9,NULL,9,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,9,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); if (i>0) {J = global_index(i-1,j, k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>0 && j>0) {J = global_index(i-1,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if ( j>0) {J = global_index(i, j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1 && j>0) {J = global_index(i+1,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = global_index(i+1,j, k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1 && j<n-1) {J = global_index(i+1,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = global_index(i, j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>0 && j<n-1) {J = global_index(i-1,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 8.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = PetscStrcmp(stencil, "2d9point2", &equal);CHKERRQ(ierr); if (equal) { /* 9-point Cartesian stencil (width 2 per coordinate), 2D */ ierr = MatMPIAIJSetPreallocation(A,9,NULL,9,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,9,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); if (i>0) {J = global_index(i-1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>1) {J = global_index(i-2,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = global_index(i+1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-2) {J = global_index(i+2,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = global_index(i,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>1) {J = global_index(i,j-2,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = global_index(i,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-2) {J = global_index(i,j+2,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 8.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = PetscStrcmp(stencil, "2d13point", &equal);CHKERRQ(ierr); if (equal) { /* 13-point Cartesian stencil (width 3 per coordinate), 2D */ ierr = MatMPIAIJSetPreallocation(A,13,NULL,13,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,13,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); if (i>0) {J = global_index(i-1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>1) {J = global_index(i-2,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>2) {J = global_index(i-3,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = global_index(i+1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-2) {J = global_index(i+2,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-3) {J = global_index(i+3,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = global_index(i,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>1) {J = global_index(i,j-2,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>2) {J = global_index(i,j-3,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = global_index(i,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-2) {J = global_index(i,j+2,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-3) {J = global_index(i,j+3,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 12.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } /************ 3D stencils ***************/ ierr = PetscStrcmp(stencil, "3d7point", &equal);CHKERRQ(ierr); if (equal) { /* 7-point stencil, 3D */ ierr = MatMPIAIJSetPreallocation(A,7,NULL,7,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,7,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); if (i>0) {J = global_index(i-1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = global_index(i+1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = global_index(i,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = global_index(i,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k>0) {J = global_index(i,j,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k<o-1) {J = global_index(i,j,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 6.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = PetscStrcmp(stencil, "3d13point", &equal);CHKERRQ(ierr); if (equal) { /* 13-point stencil, 3D */ ierr = MatMPIAIJSetPreallocation(A,13,NULL,13,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,13,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); if (i>0) {J = global_index(i-1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>1) {J = global_index(i-2,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = global_index(i+1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-2) {J = global_index(i+2,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = global_index(i,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>1) {J = global_index(i,j-2,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = global_index(i,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-2) {J = global_index(i,j+2,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k>0) {J = global_index(i,j,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k>1) {J = global_index(i,j,k-2,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k<o-1) {J = global_index(i,j,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k<o-2) {J = global_index(i,j,k+2,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 12.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = PetscStrcmp(stencil, "3d19point", &equal);CHKERRQ(ierr); if (equal) { /* 19-point stencil, 3D */ ierr = MatMPIAIJSetPreallocation(A,19,NULL,19,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,19,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); /* one hop */ if (i>0) {J = global_index(i-1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = global_index(i+1,j,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = global_index(i,j-1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = global_index(i,j+1,k,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k>0) {J = global_index(i,j,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (k<o-1) {J = global_index(i,j,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} /* two hops */ if (i>0 && j>0) {J = global_index(i-1,j-1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>0 && k>0) {J = global_index(i-1,j, k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>0 && j<n-1) {J = global_index(i-1,j+1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i>0 && k<o-1) {J = global_index(i-1,j, k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1 && j>0) {J = global_index(i+1,j-1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1 && k>0) {J = global_index(i+1,j, k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1 && j<n-1) {J = global_index(i+1,j+1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1 && k<o-1) {J = global_index(i+1,j, k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0 && k>0) {J = global_index(i, j-1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0 && k<o-1) {J = global_index(i, j-1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1 && k>0) {J = global_index(i, j+1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1 && k<o-1) {J = global_index(i, j+1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 18.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = PetscStrcmp(stencil, "3d27point", &equal);CHKERRQ(ierr); if (equal) { /* 27-point stencil, 3D */ ierr = MatMPIAIJSetPreallocation(A,27,NULL,27,NULL);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,27,NULL);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; k = Ii / (m*n); j = (Ii - k * m * n) / m; i = (Ii - k * m * n - j * m); if (k>0) { if (j>0) { if (i>0) {J = global_index(i-1,j-1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j-1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j-1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } { if (i>0) {J = global_index(i-1,j, k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j, k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j, k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } if (j<n-1) { if (i>0) {J = global_index(i-1,j+1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j+1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j+1,k-1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } } { if (j>0) { if (i>0) {J = global_index(i-1,j-1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j-1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j-1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } { if (i>0) {J = global_index(i-1,j, k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j, k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j, k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } if (j<n-1) { if (i>0) {J = global_index(i-1,j+1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j+1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j+1,k ,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } } if (k<o-1) { if (j>0) { if (i>0) {J = global_index(i-1,j-1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j-1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j-1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } { if (i>0) {J = global_index(i-1,j, k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j, k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j, k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } if (j<n-1) { if (i>0) {J = global_index(i-1,j+1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} J = global_index(i, j+1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); if (i<m-1) {J = global_index(i+1,j+1,k+1,m,n); ierr = MatSetValues(A,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} } } v = 26.0; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* Copy A into B in order to have a more representative benchmark (A*A has more cache hits than A*B) */ ierr = MatDuplicate(A,MAT_COPY_VALUES,&B);CHKERRQ(ierr); ierr = PetscLogStageRegister("Full MatMatMult",&fullMatMatMultStage);CHKERRQ(ierr); /* Test C = A*B */ ierr = PetscLogStagePush(fullMatMatMultStage);CHKERRQ(ierr); ierr = MatMatMult(A,B,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&C);CHKERRQ(ierr); /* Test PtAP_squared = PtAP(C,C)*PtAP(C,C) */ ierr = MatPtAP(C,C,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&PtAP);CHKERRQ(ierr); ierr = MatDuplicate(PtAP,MAT_COPY_VALUES,&PtAP_copy);CHKERRQ(ierr); ierr = MatMatMult(PtAP,PtAP_copy,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&PtAP_squared);CHKERRQ(ierr); ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = MatView(PtAP_squared,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = MatDestroy(&PtAP_squared);CHKERRQ(ierr); ierr = MatDestroy(&PtAP_copy);CHKERRQ(ierr); ierr = MatDestroy(&PtAP);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = MatDestroy(&B);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
/*@ PetscConvEstGetConvRate - Returns an estimate of the convergence rate for the discretization Not collective Input Parameter: . ce - The PetscConvEst object Output Parameter: . alpha - The convergence rate for each field Note: The convergence rate alpha is defined by $ || u_h - u_exact || < C h^alpha where u_h is the discrete solution, and h is a measure of the discretization size. We solve a series of problems on refined meshes, calculate an error based upon the exact solution in the DS, and then fit the result to our model above using linear regression. Options database keys: . -snes_convergence_estimate : Execute convergence estimation and print out the rate Level: intermediate .keywords: PetscConvEst, convergence .seealso: PetscConvEstSetSolver(), PetscConvEstCreate(), PetscConvEstGetConvRate() @*/ PetscErrorCode PetscConvEstGetConvRate(PetscConvEst ce, PetscReal alpha[]) { DM *dm; PetscObject disc; MPI_Comm comm; const char *uname, *dmname; void *ctx; Vec u; PetscReal t = 0.0, *x, *y, slope, intercept; PetscInt *dof, dim, Nr = ce->Nr, r, f, oldlevel, oldnlev; PetscLogEvent event; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscObjectGetComm((PetscObject) ce, &comm);CHKERRQ(ierr); ierr = DMGetDimension(ce->idm, &dim);CHKERRQ(ierr); ierr = DMGetApplicationContext(ce->idm, &ctx);CHKERRQ(ierr); ierr = DMPlexSetRefinementUniform(ce->idm, PETSC_TRUE);CHKERRQ(ierr); ierr = DMGetRefineLevel(ce->idm, &oldlevel);CHKERRQ(ierr); ierr = PetscMalloc2((Nr+1), &dm, (Nr+1)*ce->Nf, &dof);CHKERRQ(ierr); dm[0] = ce->idm; for (f = 0; f < ce->Nf; ++f) alpha[f] = 0.0; /* Loop over meshes */ ierr = PetscLogEventRegister("ConvEst Error", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); for (r = 0; r <= Nr; ++r) { PetscLogStage stage; char stageName[PETSC_MAX_PATH_LEN]; ierr = PetscSNPrintf(stageName, PETSC_MAX_PATH_LEN-1, "ConvEst Refinement Level %D", r);CHKERRQ(ierr); ierr = PetscLogStageRegister(stageName, &stage);CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); if (r > 0) { ierr = DMRefine(dm[r-1], MPI_COMM_NULL, &dm[r]);CHKERRQ(ierr); ierr = DMSetCoarseDM(dm[r], dm[r-1]);CHKERRQ(ierr); ierr = DMCopyDisc(ce->idm, dm[r]);CHKERRQ(ierr); ierr = DMCopyTransform(ce->idm, dm[r]);CHKERRQ(ierr); ierr = PetscObjectGetName((PetscObject) dm[r-1], &dmname);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject) dm[r], dmname);CHKERRQ(ierr); for (f = 0; f <= ce->Nf; ++f) { PetscErrorCode (*nspconstr)(DM, PetscInt, MatNullSpace *); ierr = DMGetNullSpaceConstructor(dm[r-1], f, &nspconstr);CHKERRQ(ierr); ierr = DMSetNullSpaceConstructor(dm[r], f, nspconstr);CHKERRQ(ierr); } } ierr = DMViewFromOptions(dm[r], NULL, "-conv_dm_view");CHKERRQ(ierr); /* Create solution */ ierr = DMCreateGlobalVector(dm[r], &u);CHKERRQ(ierr); ierr = DMGetField(dm[r], 0, NULL, &disc);CHKERRQ(ierr); ierr = PetscObjectGetName(disc, &uname);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject) u, uname);CHKERRQ(ierr); /* Setup solver */ ierr = SNESReset(ce->snes);CHKERRQ(ierr); ierr = SNESSetDM(ce->snes, dm[r]);CHKERRQ(ierr); ierr = DMPlexSetSNESLocalFEM(dm[r], ctx, ctx, ctx);CHKERRQ(ierr); ierr = SNESSetFromOptions(ce->snes);CHKERRQ(ierr); /* Create initial guess */ ierr = DMProjectFunction(dm[r], t, ce->initGuess, ce->ctxs, INSERT_VALUES, u);CHKERRQ(ierr); ierr = SNESSolve(ce->snes, NULL, u);CHKERRQ(ierr); ierr = PetscLogEventBegin(event, ce, 0, 0, 0);CHKERRQ(ierr); ierr = DMComputeL2FieldDiff(dm[r], t, ce->exactSol, ce->ctxs, u, &ce->errors[r*ce->Nf]);CHKERRQ(ierr); ierr = PetscLogEventEnd(event, ce, 0, 0, 0);CHKERRQ(ierr); for (f = 0; f < ce->Nf; ++f) { PetscSection s, fs; PetscInt lsize; /* Could use DMGetOutputDM() to add in Dirichlet dofs */ ierr = DMGetSection(dm[r], &s);CHKERRQ(ierr); ierr = PetscSectionGetField(s, f, &fs);CHKERRQ(ierr); ierr = PetscSectionGetConstrainedStorageSize(fs, &lsize);CHKERRQ(ierr); ierr = MPI_Allreduce(&lsize, &dof[r*ce->Nf+f], 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) ce->snes));CHKERRQ(ierr); ierr = PetscLogEventSetDof(event, f, dof[r*ce->Nf+f]);CHKERRQ(ierr); ierr = PetscLogEventSetError(event, f, ce->errors[r*ce->Nf+f]);CHKERRQ(ierr); } /* Monitor */ if (ce->monitor) { PetscReal *errors = &ce->errors[r*ce->Nf]; ierr = PetscPrintf(comm, "L_2 Error: ");CHKERRQ(ierr); if (ce->Nf > 1) {ierr = PetscPrintf(comm, "[");CHKERRQ(ierr);} for (f = 0; f < ce->Nf; ++f) { if (f > 0) {ierr = PetscPrintf(comm, ", ");CHKERRQ(ierr);} if (errors[f] < 1.0e-11) {ierr = PetscPrintf(comm, "< 1e-11");CHKERRQ(ierr);} else {ierr = PetscPrintf(comm, "%g", (double)errors[f]);CHKERRQ(ierr);} } if (ce->Nf > 1) {ierr = PetscPrintf(comm, "]");CHKERRQ(ierr);} ierr = PetscPrintf(comm, "\n");CHKERRQ(ierr); } if (!r) { /* PCReset() does not wipe out the level structure */ KSP ksp; PC pc; ierr = SNESGetKSP(ce->snes, &ksp);CHKERRQ(ierr); ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr); ierr = PCMGGetLevels(pc, &oldnlev);CHKERRQ(ierr); } /* Cleanup */ ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); } for (r = 1; r <= Nr; ++r) { ierr = DMDestroy(&dm[r]);CHKERRQ(ierr); } /* Fit convergence rate */ ierr = PetscMalloc2(Nr+1, &x, Nr+1, &y);CHKERRQ(ierr); for (f = 0; f < ce->Nf; ++f) { for (r = 0; r <= Nr; ++r) { x[r] = PetscLog10Real(dof[r*ce->Nf+f]); y[r] = PetscLog10Real(ce->errors[r*ce->Nf+f]); } ierr = PetscLinearRegression(Nr+1, x, y, &slope, &intercept);CHKERRQ(ierr); /* Since h^{-dim} = N, lg err = s lg N + b = -s dim lg h + b */ alpha[f] = -slope * dim; } ierr = PetscFree2(x, y);CHKERRQ(ierr); ierr = PetscFree2(dm, dof);CHKERRQ(ierr); /* Restore solver */ ierr = SNESReset(ce->snes);CHKERRQ(ierr); { /* PCReset() does not wipe out the level structure */ KSP ksp; PC pc; ierr = SNESGetKSP(ce->snes, &ksp);CHKERRQ(ierr); ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr); ierr = PCMGSetLevels(pc, oldnlev, NULL);CHKERRQ(ierr); ierr = DMSetRefineLevel(ce->idm, oldlevel);CHKERRQ(ierr); /* The damn DMCoarsen() calls in PCMG can reset this */ } ierr = SNESSetDM(ce->snes, ce->idm);CHKERRQ(ierr); ierr = DMPlexSetSNESLocalFEM(ce->idm, ctx, ctx, ctx);CHKERRQ(ierr); ierr = SNESSetFromOptions(ce->snes);CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **argv) { PetscErrorCode ierr; PetscBool view = PETSC_FALSE, viewsoln = PETSC_FALSE, noprealloc = PETSC_FALSE; char root[256] = "", nodesname[256], issname[256], solnname[256]; UM mesh; unfemCtx user; SNES snes; KSP ksp; PC pc; Mat A; Vec r, u, uexact; double err, h_max; PetscInitialize(&argc,&argv,NULL,help); ierr = PetscLogStageRegister("Read mesh ", &user.readstage); CHKERRQ(ierr); //STRIP ierr = PetscLogStageRegister("Set-up ", &user.setupstage); CHKERRQ(ierr); //STRIP ierr = PetscLogStageRegister("Solver ", &user.solverstage); CHKERRQ(ierr); //STRIP ierr = PetscLogStageRegister("Residual eval ", &user.resstage); CHKERRQ(ierr); //STRIP ierr = PetscLogStageRegister("Jacobian eval ", &user.jacstage); CHKERRQ(ierr); //STRIP user.quaddeg = 1; user.solncase = 0; ierr = PetscOptionsBegin(PETSC_COMM_WORLD, "un_", "options for unfem", ""); CHKERRQ(ierr); ierr = PetscOptionsInt("-case", "exact solution cases: 0=linear, 1=nonlinear, 2=nonhomoNeumann, 3=chapter3, 4=koch", "unfem.c",user.solncase,&(user.solncase),NULL); CHKERRQ(ierr); ierr = PetscOptionsString("-mesh", "file name root of mesh stored in PETSc binary with .vec,.is extensions", "unfem.c",root,root,sizeof(root),NULL); CHKERRQ(ierr); ierr = PetscOptionsInt("-quaddeg", "quadrature degree (1,2,3)", "unfem.c",user.quaddeg,&(user.quaddeg),NULL); CHKERRQ(ierr); ierr = PetscOptionsBool("-view", "view loaded nodes and elements at stdout", "unfem.c",view,&view,NULL); CHKERRQ(ierr); ierr = PetscOptionsBool("-view_solution", "view solution u(x,y) to binary file; uses root name of mesh plus .soln\nsee petsc2tricontour.py to view graphically", "unfem.c",viewsoln,&viewsoln,NULL); CHKERRQ(ierr); ierr = PetscOptionsBool("-noprealloc", "do not perform preallocation before matrix assembly", "unfem.c",noprealloc,&noprealloc,NULL); CHKERRQ(ierr); ierr = PetscOptionsEnd(); CHKERRQ(ierr); // set parameters and exact solution user.a_fcn = &a_lin; user.f_fcn = &f_lin; user.uexact_fcn = &uexact_lin; user.gD_fcn = &gD_lin; user.gN_fcn = &gN_lin; switch (user.solncase) { case 0 : break; case 1 : user.a_fcn = &a_nonlin; user.f_fcn = &f_nonlin; break; case 2 : user.gN_fcn = &gN_linneu; break; case 3 : user.a_fcn = &a_square; user.f_fcn = &f_square; user.uexact_fcn = &uexact_square; user.gD_fcn = &gD_square; user.gN_fcn = NULL; // seg fault if ever called break; case 4 : user.a_fcn = &a_koch; user.f_fcn = &f_koch; user.uexact_fcn = NULL; user.gD_fcn = &gD_koch; user.gN_fcn = NULL; // seg fault if ever called break; default : SETERRQ(PETSC_COMM_WORLD,1,"other solution cases not implemented"); } // determine filenames strcpy(nodesname, root); strncat(nodesname, ".vec", 4); strcpy(issname, root); strncat(issname, ".is", 3); //STARTMAININITIAL PetscLogStagePush(user.readstage); //STRIP // read mesh object of type UM ierr = UMInitialize(&mesh); CHKERRQ(ierr); ierr = UMReadNodes(&mesh,nodesname); CHKERRQ(ierr); ierr = UMReadISs(&mesh,issname); CHKERRQ(ierr); ierr = UMStats(&mesh, &h_max, NULL, NULL, NULL); CHKERRQ(ierr); if (view) { //STRIP PetscViewer stdoutviewer; //STRIP ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&stdoutviewer); CHKERRQ(ierr); //STRIP ierr = UMViewASCII(&mesh,stdoutviewer); CHKERRQ(ierr); //STRIP } //STRIP user.mesh = &mesh; PetscLogStagePop(); //STRIP // configure Vecs and SNES PetscLogStagePush(user.setupstage); //STRIP ierr = VecCreate(PETSC_COMM_WORLD,&r); CHKERRQ(ierr); ierr = VecSetSizes(r,PETSC_DECIDE,mesh.N); CHKERRQ(ierr); ierr = VecSetFromOptions(r); CHKERRQ(ierr); ierr = VecDuplicate(r,&u); CHKERRQ(ierr); ierr = VecSet(u,0.0); CHKERRQ(ierr); ierr = SNESCreate(PETSC_COMM_WORLD,&snes); CHKERRQ(ierr); ierr = SNESSetFunction(snes,r,FormFunction,&user); CHKERRQ(ierr); // reset default KSP and PC ierr = SNESGetKSP(snes,&ksp); CHKERRQ(ierr); ierr = KSPSetType(ksp,KSPCG); CHKERRQ(ierr); ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr); ierr = PCSetType(pc,PCICC); CHKERRQ(ierr); // setup matrix for Picard iteration, including preallocation ierr = MatCreate(PETSC_COMM_WORLD,&A); CHKERRQ(ierr); ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,mesh.N,mesh.N); CHKERRQ(ierr); ierr = MatSetFromOptions(A); CHKERRQ(ierr); ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE); CHKERRQ(ierr); if (noprealloc) { ierr = MatSetUp(A); CHKERRQ(ierr); } else { ierr = Preallocation(A,&user); CHKERRQ(ierr); } ierr = SNESSetJacobian(snes,A,A,FormPicard,&user); CHKERRQ(ierr); ierr = SNESSetFromOptions(snes); CHKERRQ(ierr); PetscLogStagePop(); //STRIP // solve PetscLogStagePush(user.solverstage); //STRIP ierr = SNESSolve(snes,NULL,u);CHKERRQ(ierr); PetscLogStagePop(); //STRIP //ENDMAININITIAL if (viewsoln) { strcpy(solnname, root); strncat(solnname, ".soln", 5); ierr = UMViewSolutionBinary(&mesh,solnname,u); CHKERRQ(ierr); } if (user.uexact_fcn) { // measure error relative to exact solution ierr = VecDuplicate(r,&uexact); CHKERRQ(ierr); ierr = FillExact(uexact,&user); CHKERRQ(ierr); ierr = VecAXPY(u,-1.0,uexact); CHKERRQ(ierr); // u <- u + (-1.0) uexact ierr = VecNorm(u,NORM_INFINITY,&err); CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD, "case %d result for N=%d nodes with h = %.3e : |u-u_ex|_inf = %g\n", user.solncase,mesh.N,h_max,err); CHKERRQ(ierr); VecDestroy(&uexact); } else { ierr = PetscPrintf(PETSC_COMM_WORLD, "case %d completed for N=%d nodes with h = %.3e (no exact solution)\n", user.solncase,mesh.N,h_max); CHKERRQ(ierr); } // clean-up SNESDestroy(&snes); MatDestroy(&A); VecDestroy(&u); VecDestroy(&r); UMDestroy(&mesh); PetscFinalize(); return 0; }
static PetscErrorCode TestVecClosure(DM dm, PetscBool useIndex, PetscBool useSpectral, AppCtx *user) { PetscSection s; Vec v; PetscInt numRuns, cStart, cEnd, c, i; PetscScalar tmpArray[64]; PetscScalar *userArray = user->reuseArray ? tmpArray : NULL; PetscReal maxTimePerRun = user->maxVecClosureTime; PetscLogStage stage; PetscLogEvent event; PetscEventPerfInfo eventInfo; PetscErrorCode ierr; PetscFunctionBegin; if (useIndex) { if (useSpectral) { ierr = PetscLogStageRegister("DMPlex Vector Closure with Index Test", &stage);CHKERRQ(ierr); ierr = PetscLogEventRegister("VecClosureInd", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); } else { ierr = PetscLogStageRegister("DMPlex Vector Spectral Closure with Index Test", &stage);CHKERRQ(ierr); ierr = PetscLogEventRegister("VecClosureSpecInd", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); } } else { if (useSpectral) { ierr = PetscLogStageRegister("DMPlex Vector Spectral Closure Test", &stage);CHKERRQ(ierr); ierr = PetscLogEventRegister("VecClosureSpec", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); } else { ierr = PetscLogStageRegister("DMPlex Vector Closure Test", &stage);CHKERRQ(ierr); ierr = PetscLogEventRegister("VecClosure", PETSC_OBJECT_CLASSID, &event);CHKERRQ(ierr); } } ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = DMPlexCreateSection(dm, user->dim, user->numFields, user->numComponents, user->numDof, 0, NULL, NULL, NULL, NULL, &s);CHKERRQ(ierr); ierr = DMSetDefaultSection(dm, s);CHKERRQ(ierr); if (useIndex) {ierr = DMPlexCreateClosureIndex(dm, s);CHKERRQ(ierr);} if (useSpectral) {ierr = DMPlexCreateSpectralClosurePermutation(dm, s);CHKERRQ(ierr);} ierr = PetscSectionDestroy(&s);CHKERRQ(ierr); ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr); ierr = DMGetLocalVector(dm, &v);CHKERRQ(ierr); ierr = PetscLogEventBegin(event,0,0,0,0);CHKERRQ(ierr); for (i = 0; i < user->iterations; ++i) { for (c = cStart; c < cEnd; ++c) { PetscScalar *closure = userArray; PetscInt closureSize = 64; ierr = DMPlexVecGetClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr); if (!user->reuseArray) {ierr = DMPlexVecRestoreClosure(dm, s, v, c, &closureSize, &closure);CHKERRQ(ierr);} } } ierr = PetscLogEventEnd(event,0,0,0,0);CHKERRQ(ierr); ierr = DMRestoreLocalVector(dm, &v);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogEventGetPerfInfo(stage, event, &eventInfo);CHKERRQ(ierr); numRuns = (cEnd-cStart) * user->iterations; if (eventInfo.count != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event calls %d should be %d", eventInfo.count, 1); if ((PetscInt) eventInfo.flops != 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of event flops %d should be %d", (PetscInt) eventInfo.flops, 0); if (eventInfo.time > maxTimePerRun * numRuns) { const char *title = "VecClosures"; const char *titleIndex = "VecClosures with Index"; const char *titleSpec = "VecClosures Spectral"; const char *titleSpecIndex = "VecClosures Spectral with Index"; ierr = PetscPrintf(PETSC_COMM_SELF, "%s: %d Average time per vector closure: %gs standard: %gs\n", useIndex ? (useSpectral ? titleSpecIndex : titleIndex) : (useSpectral ? titleSpec : title), numRuns, eventInfo.time/numRuns, maxTimePerRun); if (user->errors) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Average time for vector closure %g > standard %g", eventInfo.time/numRuns, maxTimePerRun); } PetscFunctionReturn(0); }
int main(int argc,char **argv) { DM da; /* distributed array */ Vec x,b,u; /* approx solution, RHS, exact solution */ Mat A; /* linear system matrix */ KSP ksp; /* linear solver context */ PetscRandom rctx; /* random number generator context */ PetscReal norm; /* norm of solution error */ PetscInt i,j,its; PetscErrorCode ierr; PetscBool flg = PETSC_FALSE; PetscLogStage stage; DMDALocalInfo info; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); /* Create distributed array to handle parallel distribution. The problem size will default to 8 by 7, but this can be changed using -da_grid_x M -da_grid_y N */ ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-8,-7,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Compute the matrix and right-hand-side vector that define the linear system, Ax = b. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create parallel matrix preallocated according to the DMDA, format AIJ by default. To use symmetric storage, run with -dm_mat_type sbaij -mat_ignore_lower_triangular */ ierr = DMSetMatType(da,MATAIJ);CHKERRQ(ierr); ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr); /* Set matrix elements for the 2-D, five-point stencil in parallel. - Each processor needs to insert only elements that it owns locally (but any non-local elements will be sent to the appropriate processor during matrix assembly). - Rows and columns are specified by the stencil - Entries are normalized for a domain [0,1]x[0,1] */ ierr = PetscLogStageRegister("Assembly", &stage);CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); for (j=info.ys; j<info.ys+info.ym; j++) { for (i=info.xs; i<info.xs+info.xm; i++) { PetscReal hx = 1./info.mx,hy = 1./info.my; MatStencil row = {0},col[5] = {{0}}; PetscScalar v[5]; PetscInt ncols = 0; row.j = j; row.i = i; col[ncols].j = j; col[ncols].i = i; v[ncols++] = 2*(hx/hy + hy/hx); /* boundaries */ if (i>0) {col[ncols].j = j; col[ncols].i = i-1; v[ncols++] = -hy/hx;} if (i<info.mx-1) {col[ncols].j = j; col[ncols].i = i+1; v[ncols++] = -hy/hx;} if (j>0) {col[ncols].j = j-1; col[ncols].i = i; v[ncols++] = -hx/hy;} if (j<info.my-1) {col[ncols].j = j+1; col[ncols].i = i; v[ncols++] = -hx/hy;} ierr = MatSetValuesStencil(A,1,&row,ncols,col,v,INSERT_VALUES);CHKERRQ(ierr); } } /* Assemble matrix, using the 2-step process: MatAssemblyBegin(), MatAssemblyEnd() Computations can be done while messages are in transition by placing code between these two statements. */ ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); /* Create parallel vectors compatible with the DMDA. */ ierr = DMCreateGlobalVector(da,&u);CHKERRQ(ierr); ierr = VecDuplicate(u,&b);CHKERRQ(ierr); ierr = VecDuplicate(u,&x);CHKERRQ(ierr); /* Set exact solution; then compute right-hand-side vector. By default we use an exact solution of a vector with all elements of 1.0; Alternatively, using the runtime option -random_sol forms a solution vector with random components. */ ierr = PetscOptionsGetBool(NULL,"-random_exact_sol",&flg,NULL);CHKERRQ(ierr); if (flg) { ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr); ierr = VecSetRandom(u,rctx);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr); } else { ierr = VecSet(u,1.);CHKERRQ(ierr); } ierr = MatMult(A,u,b);CHKERRQ(ierr); /* View the exact solution vector if desired */ flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,"-view_exact_sol",&flg,NULL);CHKERRQ(ierr); if (flg) {ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);} /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create the linear solver and set various options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create linear solver context */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr); /* Set operators. Here the matrix that defines the linear system also serves as the preconditioning matrix. */ ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr); /* Set runtime options, e.g., -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol> These options will override those specified above as long as KSPSetFromOptions() is called _after_ any other customization routines. */ ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Solve the linear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Check solution and clean up - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Check the error */ ierr = VecAXPY(x,-1.,u);CHKERRQ(ierr); ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); /* Print convergence information. PetscPrintf() produces a single print statement from all processes that share a communicator. An alternative is PetscFPrintf(), which prints to a file. */ ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g iterations %D\n",(double)norm,its);CHKERRQ(ierr); /* Free work space. All PETSc objects should be destroyed when they are no longer needed. */ ierr = KSPDestroy(&ksp);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); /* Always call PetscFinalize() before exiting a program. This routine - finalizes the PETSc libraries as well as MPI - provides summary and diagnostic information if certain runtime options are chosen (e.g., -log_summary). */ ierr = PetscFinalize(); return 0; }
int main(int argc, char *argv[]) { PetscReal (*fList[])(Point) = {funConst, funSin, funStep}; PetscErrorCode ierr; PetscMPIInt rank,size; PetscReal m=0.0,n=4.0,k=0.0,l=4.0,h=0.1; PetscInt xSubs = 4, ySubs = 4; PetscReal mi=1e-3,ro=4,beta=1.5,M=1; PetscInitialize(&argc,&argv,0,help); PetscInt f = 2; PetscTruth flg; PetscLogStage assemblyStage, smaleStage; char fileName[PETSC_MAX_PATH_LEN]="matlab/out.m"; PetscOptionsGetReal("-test_h", &h, PETSC_NULL); PetscOptionsGetReal("-test_mi", &mi, PETSC_NULL); PetscOptionsGetReal("-test_ro", &ro, PETSC_NULL); PetscOptionsGetReal("-test_beta", &beta, PETSC_NULL); PetscOptionsGetReal("-test_M", &M, PETSC_NULL); PetscOptionsGetInt("-test_x", &xSubs, PETSC_NULL); PetscOptionsGetInt("-test_y", &ySubs, PETSC_NULL); PetscOptionsGetInt("-test_f", &f, PETSC_NULL); PetscOptionsGetString(PETSC_NULL, "-test_out_file", fileName, PETSC_MAX_PATH_LEN-1, &flg); //if (!flg) SETERRQ(1,"Must indicate binary file with the -test_out_file option"); { //n = xSubs; //l = ySubs; ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); MPI_Comm_size(PETSC_COMM_WORLD, &size); PetscViewer v; PetscLogStageRegister("Assembly", &assemblyStage); PetscLogStagePush(assemblyStage); Mesh *mesh = new Mesh(); //mesh->generateRectangularMesh(m, n, k, l, h); mesh->loadHDF5("mesh.med"); mesh->partition(size); mesh->tear(); SDSystem sdSystem(mesh,fList[f],fList[0]); PetscViewerBinaryOpen(PETSC_COMM_WORLD, "matlab/mesh.m", FILE_MODE_WRITE, &v); mesh->dumpForMatlab(v); mesh->saveHDF5("outmesh.med"); PetscViewerDestroy(v); delete mesh; PetscLogStagePop(); Smale smale(&sdSystem,mi,ro,beta,M); PetscLogStageRegister("Smale", &smaleStage); PetscLogStagePush(smaleStage); smale.solve(); PetscLogStagePop(); saveScalarResultHDF5("outmesh.med","smale",smale.getx()); PetscViewerBinaryOpen(PETSC_COMM_WORLD, fileName, FILE_MODE_WRITE, &v); smale.dumpSolution(v); //smale.dumpSolution(matLabView); PetscViewerDestroy(v); //PetscViewerDestroy(matLabView); } ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
PetscErrorCode PCSetFromOptions_MG(PetscOptionItems *PetscOptionsObject,PC pc) { PetscErrorCode ierr; PetscInt m,levels = 1,cycles; PetscBool flg,set; PC_MG *mg = (PC_MG*)pc->data; PC_MG_Levels **mglevels; PCMGType mgtype; PCMGCycleType mgctype; PetscFunctionBegin; ierr = PetscOptionsHead(PetscOptionsObject,"Multigrid options");CHKERRQ(ierr); if (!mg->levels) { ierr = PetscOptionsInt("-pc_mg_levels","Number of Levels","PCMGSetLevels",levels,&levels,&flg);CHKERRQ(ierr); if (!flg && pc->dm) { ierr = DMGetRefineLevel(pc->dm,&levels);CHKERRQ(ierr); levels++; mg->usedmfornumberoflevels = PETSC_TRUE; } ierr = PCMGSetLevels(pc,levels,NULL);CHKERRQ(ierr); } mglevels = mg->levels; mgctype = (PCMGCycleType) mglevels[0]->cycles; ierr = PetscOptionsEnum("-pc_mg_cycle_type","V cycle or for W-cycle","PCMGSetCycleType",PCMGCycleTypes,(PetscEnum)mgctype,(PetscEnum*)&mgctype,&flg);CHKERRQ(ierr); if (flg) { ierr = PCMGSetCycleType(pc,mgctype);CHKERRQ(ierr); } flg = PETSC_FALSE; ierr = PetscOptionsBool("-pc_mg_galerkin","Use Galerkin process to compute coarser operators","PCMGSetGalerkin",flg,&flg,&set);CHKERRQ(ierr); if (set) { ierr = PCMGSetGalerkin(pc,flg);CHKERRQ(ierr); } ierr = PetscOptionsInt("-pc_mg_smoothup","Number of post-smoothing steps","PCMGSetNumberSmoothUp",mg->default_smoothu,&m,&flg);CHKERRQ(ierr); if (flg) { ierr = PCMGSetNumberSmoothUp(pc,m);CHKERRQ(ierr); } ierr = PetscOptionsInt("-pc_mg_smoothdown","Number of pre-smoothing steps","PCMGSetNumberSmoothDown",mg->default_smoothd,&m,&flg);CHKERRQ(ierr); if (flg) { ierr = PCMGSetNumberSmoothDown(pc,m);CHKERRQ(ierr); } mgtype = mg->am; ierr = PetscOptionsEnum("-pc_mg_type","Multigrid type","PCMGSetType",PCMGTypes,(PetscEnum)mgtype,(PetscEnum*)&mgtype,&flg);CHKERRQ(ierr); if (flg) { ierr = PCMGSetType(pc,mgtype);CHKERRQ(ierr); } if (mg->am == PC_MG_MULTIPLICATIVE) { ierr = PetscOptionsInt("-pc_mg_multiplicative_cycles","Number of cycles for each preconditioner step","PCMGMultiplicativeSetCycles",mg->cyclesperpcapply,&cycles,&flg);CHKERRQ(ierr); if (flg) { ierr = PCMGMultiplicativeSetCycles(pc,cycles);CHKERRQ(ierr); } } flg = PETSC_FALSE; ierr = PetscOptionsBool("-pc_mg_log","Log times for each multigrid level","None",flg,&flg,NULL);CHKERRQ(ierr); if (flg) { PetscInt i; char eventname[128]; if (!mglevels) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must set MG levels before calling"); levels = mglevels[0]->levels; for (i=0; i<levels; i++) { sprintf(eventname,"MGSetup Level %d",(int)i); ierr = PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventsmoothsetup);CHKERRQ(ierr); sprintf(eventname,"MGSmooth Level %d",(int)i); ierr = PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventsmoothsolve);CHKERRQ(ierr); if (i) { sprintf(eventname,"MGResid Level %d",(int)i); ierr = PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventresidual);CHKERRQ(ierr); sprintf(eventname,"MGInterp Level %d",(int)i); ierr = PetscLogEventRegister(eventname,((PetscObject)pc)->classid,&mglevels[i]->eventinterprestrict);CHKERRQ(ierr); } } #if defined(PETSC_USE_LOG) { const char *sname = "MG Apply"; PetscStageLog stageLog; PetscInt st; PetscFunctionBegin; ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr); for (st = 0; st < stageLog->numStages; ++st) { PetscBool same; ierr = PetscStrcmp(stageLog->stageInfo[st].name, sname, &same);CHKERRQ(ierr); if (same) mg->stageApply = st; } if (!mg->stageApply) { ierr = PetscLogStageRegister(sname, &mg->stageApply);CHKERRQ(ierr); } } #endif } ierr = PetscOptionsTail();CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char ** argv) { PetscErrorCode ierr; PetscInt i, nbranch = 0, eStart, eEnd, vStart, vEnd; PetscInt seed = 0, nnode = 0; PetscMPIInt size, rank; DM networkdm; Vec x, b; Mat A; KSP ksp; PetscInt *edgelist = NULL; PetscInt componentkey[2]; Node *node; Branch *branch; #if defined(PETSC_USE_LOG) PetscLogStage stage[3]; #endif ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-seed",&seed,NULL);CHKERRQ(ierr); ierr = PetscLogStageRegister("Network Creation", &stage[0]);CHKERRQ(ierr); ierr = PetscLogStageRegister("DMNetwork data structures", &stage[1]);CHKERRQ(ierr); ierr = PetscLogStageRegister("KSP", &stage[2]);CHKERRQ(ierr); ierr = PetscLogStagePush(stage[0]);CHKERRQ(ierr); /* "read" data only for processor 0 */ if (!rank) { nnode = 100; ierr = PetscOptionsGetInt(NULL,NULL,"-n",&nnode,NULL);CHKERRQ(ierr); ierr = random_network(nnode, &nbranch, &node, &branch, &edgelist, seed);CHKERRQ(ierr); } ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStagePush(stage[1]);CHKERRQ(ierr); ierr = DMNetworkCreate(PETSC_COMM_WORLD,&networkdm);CHKERRQ(ierr); ierr = DMNetworkRegisterComponent(networkdm,"nstr",sizeof(Node),&componentkey[0]);CHKERRQ(ierr); ierr = DMNetworkRegisterComponent(networkdm,"bsrt",sizeof(Branch),&componentkey[1]);CHKERRQ(ierr); /* Set number of nodes/edges */ ierr = DMNetworkSetSizes(networkdm,1,0,&nnode,&nbranch,NULL,NULL);CHKERRQ(ierr); /* Add edge connectivity */ ierr = DMNetworkSetEdgeList(networkdm,&edgelist,NULL);CHKERRQ(ierr); /* Set up the network layout */ ierr = DMNetworkLayoutSetUp(networkdm);CHKERRQ(ierr); /* Add network components: physical parameters of nodes and branches*/ if (!rank) { ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr); for (i = eStart; i < eEnd; i++) { ierr = DMNetworkAddComponent(networkdm,i,componentkey[1],&branch[i-eStart]);CHKERRQ(ierr); ierr = DMNetworkAddNumVariables(networkdm,i,1);CHKERRQ(ierr); } ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr); for (i = vStart; i < vEnd; i++) { ierr = DMNetworkAddComponent(networkdm,i,componentkey[0],&node[i-vStart]);CHKERRQ(ierr); /* Add number of variables */ ierr = DMNetworkAddNumVariables(networkdm,i,1);CHKERRQ(ierr); } } /* Network partitioning and distribution of data */ ierr = DMSetUp(networkdm);CHKERRQ(ierr); ierr = DMNetworkDistribute(&networkdm,0);CHKERRQ(ierr); ierr = DMNetworkAssembleGraphStructures(networkdm);CHKERRQ(ierr); /* We don't use these data structures anymore since they have been copied to networkdm */ if (!rank) { ierr = PetscFree(edgelist);CHKERRQ(ierr); ierr = PetscFree2(node,branch);CHKERRQ(ierr); } /* Create vectors and matrix */ ierr = DMCreateGlobalVector(networkdm,&x);CHKERRQ(ierr); ierr = VecDuplicate(x,&b);CHKERRQ(ierr); ierr = DMCreateMatrix(networkdm,&A);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); ierr = PetscLogStagePush(stage[2]);CHKERRQ(ierr); /* Assembly system of equations */ ierr = FormOperator(networkdm,A,b);CHKERRQ(ierr); /* Solve linear system: A x = b */ ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);CHKERRQ(ierr); ierr = KSPSetOperators(ksp, A, A);CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); ierr = KSPSolve(ksp, b, x);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); /* Free work space */ ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = KSPDestroy(&ksp);CHKERRQ(ierr); ierr = DMDestroy(&networkdm);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }