/** * @brief Modify matrix A and RHS vector for all-Neumann BCs * * All Neumann BCs causes issues like indefinite matrix or infinite number of * solutions. We hence assume the exact solution at the node i = j = 0 is known. * (The same way we deal with the pressure in CFD applications.) The entry * A[0, 0] is modified to 1, and A[0, *] = A[*, 0] = 0; also all corresponding * entries in RHS vector is modified respectively. * * @param A Matrix A generated from the function generateA(...) * @param RHS The right-hand-side vector generated from generateRHS(...) * @param exact The exact solution vector generated from generateExt(...) * * @return */ PetscErrorCode applyNeumannBC(Mat &A, Vec &RHS, const Vec &exact) { PetscErrorCode ierr; PetscInt row[1] = {0}; ierr = MatZeroRowsColumns(A, 1, row, 1.0, exact, RHS); CHKERRQ(ierr); return 0; }
void StaggeredStokesPETScLevelSolver::initializeSolverStateSpecialized( const SAMRAIVectorReal<NDIM, double>& x, const SAMRAIVectorReal<NDIM, double>& /*b*/) { // Allocate DOF index data. Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(d_level_num); if (!level->checkAllocated(d_u_dof_index_idx)) level->allocatePatchData(d_u_dof_index_idx); if (!level->checkAllocated(d_p_dof_index_idx)) level->allocatePatchData(d_p_dof_index_idx); // Setup PETSc objects. int ierr; StaggeredStokesPETScVecUtilities::constructPatchLevelDOFIndices( d_num_dofs_per_proc, d_u_dof_index_idx, d_p_dof_index_idx, level); const int mpi_rank = SAMRAI_MPI::getRank(); ierr = VecCreateMPI( PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_x); IBTK_CHKERRQ(ierr); ierr = VecCreateMPI( PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_b); IBTK_CHKERRQ(ierr); StaggeredStokesPETScMatUtilities::constructPatchLevelMACStokesOp(d_petsc_mat, d_U_problem_coefs, d_U_bc_coefs, d_new_time, d_num_dofs_per_proc, d_u_dof_index_idx, d_p_dof_index_idx, level); ierr = MatDuplicate(d_petsc_mat, MAT_COPY_VALUES, &d_petsc_pc); IBTK_CHKERRQ(ierr); HierarchyDataOpsManager<NDIM>* hier_ops_manager = HierarchyDataOpsManager<NDIM>::getManager(); Pointer<HierarchyDataOpsInteger<NDIM> > hier_p_dof_index_ops = hier_ops_manager->getOperationsInteger(d_p_dof_index_var, d_hierarchy, true); hier_p_dof_index_ops->resetLevels(d_level_num, d_level_num); const int min_p_idx = hier_p_dof_index_ops->min( d_p_dof_index_idx); // NOTE: HierarchyDataOpsInteger::max() is broken ierr = MatZeroRowsColumns(d_petsc_pc, 1, &min_p_idx, 1.0, NULL, NULL); IBTK_CHKERRQ(ierr); d_petsc_ksp_ops_flag = SAME_PRECONDITIONER; const int u_idx = x.getComponentDescriptorIndex(0); const int p_idx = x.getComponentDescriptorIndex(1); d_data_synch_sched = StaggeredStokesPETScVecUtilities::constructDataSynchSchedule(u_idx, p_idx, level); d_ghost_fill_sched = StaggeredStokesPETScVecUtilities::constructGhostFillSchedule(u_idx, p_idx, level); return; } // initializeSolverStateSpecialized
/* Run with -build_twosided allreduce -pc_type bjacobi -sub_pc_type lu -q 16 -ksp_rtol 1.e-34 (or 1.e-14 for double precision) -q <q> number of spectral elements to use -N <N> maximum number of GLL points per element */ int main(int argc,char **args) { PetscErrorCode ierr; PetscGLL gll; PetscInt N = 80,n,q = 8,xs,xn,j,l; PetscReal **A; Mat K; KSP ksp; PC pc; Vec x,b; PetscInt *rows; PetscReal norm,xc,yc,h; PetscScalar *f; PetscDraw draw; PetscDrawLG lg; PetscDrawAxis axis; DM da; PetscMPIInt rank,size; ierr = PetscInitialize(&argc,&args,NULL,NULL);if (ierr) return ierr; ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-q",&q,NULL);CHKERRQ(ierr); ierr = PetscDrawCreate(PETSC_COMM_WORLD,NULL,"Log(Error norm) vs Number of GLL points",0,0,500,500,&draw);CHKERRQ(ierr); ierr = PetscDrawSetFromOptions(draw);CHKERRQ(ierr); ierr = PetscDrawLGCreate(draw,1,&lg);CHKERRQ(ierr); ierr = PetscDrawLGSetUseMarkers(lg,PETSC_TRUE);CHKERRQ(ierr); ierr = PetscDrawLGGetAxis(lg,&axis);CHKERRQ(ierr); ierr = PetscDrawAxisSetLabels(axis,NULL,"Number of GLL points","Log(Error Norm)");CHKERRQ(ierr); for (n=4; n<N; n+=2) { /* da contains the information about the parallel layout of the elements */ ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,q*(n-1)+1,1,1,NULL,&da);CHKERRQ(ierr); ierr = DMSetFromOptions(da);CHKERRQ(ierr); ierr = DMSetUp(da);CHKERRQ(ierr); ierr = DMDAGetInfo(da,NULL,&q,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);CHKERRQ(ierr); q = (q-1)/(n-1); /* number of spectral elements */ /* gll simply contains the GLL node and weight values */ ierr = PetscGLLCreate(n,PETSCGLL_VIA_LINEARALGEBRA,&gll);CHKERRQ(ierr); ierr = DMDASetGLLCoordinates(da,&gll);CHKERRQ(ierr); /* Creates the element stiffness matrix for the given gll */ ierr = PetscGLLElementLaplacianCreate(&gll,&A);CHKERRQ(ierr); /* Scale the element stiffness and weights by the size of the element */ h = 2.0/q; for (j=0; j<n; j++) { gll.weights[j] *= .5*h; for (l=0; l<n; l++) { A[j][l] = 2.*A[j][l]/h; } } /* Create the global stiffness matrix and add the element stiffness for each local element */ ierr = DMCreateMatrix(da,&K);CHKERRQ(ierr); ierr = MatSetOption(K,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr); ierr = DMDAGetCorners(da,&xs,NULL,NULL,&xn,NULL,NULL);CHKERRQ(ierr); xs = xs/(n-1); xn = xn/(n-1); ierr = PetscMalloc1(n,&rows);CHKERRQ(ierr); /* loop over local elements */ for (j=xs; j<xs+xn; j++) { for (l=0; l<n; l++) rows[l] = j*(n-1)+l; ierr = MatSetValues(K,n,rows,n,rows,&A[0][0],ADD_VALUES);CHKERRQ(ierr); } ierr = MatAssemblyBegin(K,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(K,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatCreateVecs(K,&x,&b);CHKERRQ(ierr); ierr = ComputeRhs(da,&gll,b);CHKERRQ(ierr); /* Replace the first and last rows/columns of the matrix with the identity to obtain the zero Dirichlet boundary conditions */ rows[0] = 0; rows[1] = q*(n-1); ierr = MatZeroRowsColumns(K,2,rows,1.0,x,b);CHKERRQ(ierr); ierr = PetscFree(rows);CHKERRQ(ierr); ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr); ierr = KSPSetOperators(ksp,K,K);CHKERRQ(ierr); ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr); ierr = PCSetType(pc,PCLU);CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr); /* compute the error to the continium problem */ ierr = ComputeSolution(da,&gll,b);CHKERRQ(ierr); ierr = VecAXPY(x,-1.0,b);CHKERRQ(ierr); /* compute the L^2 norm of the error */ ierr = VecGetArray(x,&f);CHKERRQ(ierr); ierr = PetscGLLIntegrate(&gll,f,&norm);CHKERRQ(ierr); ierr = VecRestoreArray(x,&f);CHKERRQ(ierr); norm = PetscSqrtReal(norm); ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD,"L^2 norm of the error %D %g\n",n,(double)norm);CHKERRQ(ierr); if (n > 10 && norm > 1.e-8) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"Slower convergence than expected"); xc = (PetscReal)n; yc = PetscLog10Real(norm); ierr = PetscDrawLGAddPoint(lg,&xc,&yc);CHKERRQ(ierr); ierr = PetscDrawLGDraw(lg);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = KSPDestroy(&ksp);CHKERRQ(ierr); ierr = MatDestroy(&K);CHKERRQ(ierr); ierr = PetscGLLElementLaplacianDestroy(&gll,&A);CHKERRQ(ierr); ierr = PetscGLLDestroy(&gll);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); } ierr = PetscDrawLGDestroy(&lg);CHKERRQ(ierr); ierr = PetscDrawDestroy(&draw);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }