/**
* @brief Modify matrix A and RHS vector for all-Neumann BCs
*
* All Neumann BCs causes issues like indefinite matrix or infinite number of
* solutions. We hence assume the exact solution at the node i = j = 0 is known.
* (The same way we deal with the pressure in CFD applications.) The entry
* A[0, 0] is modified to 1, and A[0, *] = A[*, 0] = 0; also all corresponding
* entries in RHS vector is modified respectively.
*
* @param A Matrix A generated from the function generateA(...)
* @param RHS The right-hand-side vector generated from generateRHS(...)
* @param exact The exact solution vector generated from generateExt(...)
*
* @return
*/
PetscErrorCode applyNeumannBC(Mat &A, Vec &RHS, const Vec &exact)
{
PetscErrorCode ierr;
PetscInt row[1] = {0};
ierr = MatZeroRowsColumns(A, 1, row, 1.0, exact, RHS); CHKERRQ(ierr);
return 0;
}
void StaggeredStokesPETScLevelSolver::initializeSolverStateSpecialized(
const SAMRAIVectorReal<NDIM, double>& x,
const SAMRAIVectorReal<NDIM, double>& /*b*/)
{
// Allocate DOF index data.
Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(d_level_num);
if (!level->checkAllocated(d_u_dof_index_idx)) level->allocatePatchData(d_u_dof_index_idx);
if (!level->checkAllocated(d_p_dof_index_idx)) level->allocatePatchData(d_p_dof_index_idx);
// Setup PETSc objects.
int ierr;
StaggeredStokesPETScVecUtilities::constructPatchLevelDOFIndices(
d_num_dofs_per_proc, d_u_dof_index_idx, d_p_dof_index_idx, level);
const int mpi_rank = SAMRAI_MPI::getRank();
ierr = VecCreateMPI(
PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_x);
IBTK_CHKERRQ(ierr);
ierr = VecCreateMPI(
PETSC_COMM_WORLD, d_num_dofs_per_proc[mpi_rank], PETSC_DETERMINE, &d_petsc_b);
IBTK_CHKERRQ(ierr);
StaggeredStokesPETScMatUtilities::constructPatchLevelMACStokesOp(d_petsc_mat,
d_U_problem_coefs,
d_U_bc_coefs,
d_new_time,
d_num_dofs_per_proc,
d_u_dof_index_idx,
d_p_dof_index_idx,
level);
ierr = MatDuplicate(d_petsc_mat, MAT_COPY_VALUES, &d_petsc_pc);
IBTK_CHKERRQ(ierr);
HierarchyDataOpsManager<NDIM>* hier_ops_manager =
HierarchyDataOpsManager<NDIM>::getManager();
Pointer<HierarchyDataOpsInteger<NDIM> > hier_p_dof_index_ops =
hier_ops_manager->getOperationsInteger(d_p_dof_index_var, d_hierarchy, true);
hier_p_dof_index_ops->resetLevels(d_level_num, d_level_num);
const int min_p_idx = hier_p_dof_index_ops->min(
d_p_dof_index_idx); // NOTE: HierarchyDataOpsInteger::max() is broken
ierr = MatZeroRowsColumns(d_petsc_pc, 1, &min_p_idx, 1.0, NULL, NULL);
IBTK_CHKERRQ(ierr);
d_petsc_ksp_ops_flag = SAME_PRECONDITIONER;
const int u_idx = x.getComponentDescriptorIndex(0);
const int p_idx = x.getComponentDescriptorIndex(1);
d_data_synch_sched =
StaggeredStokesPETScVecUtilities::constructDataSynchSchedule(u_idx, p_idx, level);
d_ghost_fill_sched =
StaggeredStokesPETScVecUtilities::constructGhostFillSchedule(u_idx, p_idx, level);
return;
} // initializeSolverStateSpecialized
/*
Run with -build_twosided allreduce -pc_type bjacobi -sub_pc_type lu -q 16 -ksp_rtol 1.e-34 (or 1.e-14 for double precision)
-q <q> number of spectral elements to use
-N <N> maximum number of GLL points per element
*/
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscGLL gll;
PetscInt N = 80,n,q = 8,xs,xn,j,l;
PetscReal **A;
Mat K;
KSP ksp;
PC pc;
Vec x,b;
PetscInt *rows;
PetscReal norm,xc,yc,h;
PetscScalar *f;
PetscDraw draw;
PetscDrawLG lg;
PetscDrawAxis axis;
DM da;
PetscMPIInt rank,size;
ierr = PetscInitialize(&argc,&args,NULL,NULL);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-q",&q,NULL);CHKERRQ(ierr);
ierr = PetscDrawCreate(PETSC_COMM_WORLD,NULL,"Log(Error norm) vs Number of GLL points",0,0,500,500,&draw);CHKERRQ(ierr);
ierr = PetscDrawSetFromOptions(draw);CHKERRQ(ierr);
ierr = PetscDrawLGCreate(draw,1,&lg);CHKERRQ(ierr);
ierr = PetscDrawLGSetUseMarkers(lg,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscDrawLGGetAxis(lg,&axis);CHKERRQ(ierr);
ierr = PetscDrawAxisSetLabels(axis,NULL,"Number of GLL points","Log(Error Norm)");CHKERRQ(ierr);
for (n=4; n<N; n+=2) {
/*
da contains the information about the parallel layout of the elements
*/
ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,q*(n-1)+1,1,1,NULL,&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,NULL,&q,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);CHKERRQ(ierr);
q = (q-1)/(n-1); /* number of spectral elements */
/*
gll simply contains the GLL node and weight values
*/
ierr = PetscGLLCreate(n,PETSCGLL_VIA_LINEARALGEBRA,&gll);CHKERRQ(ierr);
ierr = DMDASetGLLCoordinates(da,&gll);CHKERRQ(ierr);
/*
Creates the element stiffness matrix for the given gll
*/
ierr = PetscGLLElementLaplacianCreate(&gll,&A);CHKERRQ(ierr);
/*
Scale the element stiffness and weights by the size of the element
*/
h = 2.0/q;
for (j=0; j<n; j++) {
gll.weights[j] *= .5*h;
for (l=0; l<n; l++) {
A[j][l] = 2.*A[j][l]/h;
}
}
/*
Create the global stiffness matrix and add the element stiffness for each local element
*/
ierr = DMCreateMatrix(da,&K);CHKERRQ(ierr);
ierr = MatSetOption(K,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,NULL,NULL,&xn,NULL,NULL);CHKERRQ(ierr);
xs = xs/(n-1);
xn = xn/(n-1);
ierr = PetscMalloc1(n,&rows);CHKERRQ(ierr);
/*
loop over local elements
*/
for (j=xs; j<xs+xn; j++) {
for (l=0; l<n; l++) rows[l] = j*(n-1)+l;
ierr = MatSetValues(K,n,rows,n,rows,&A[0][0],ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(K,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(K,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCreateVecs(K,&x,&b);CHKERRQ(ierr);
ierr = ComputeRhs(da,&gll,b);CHKERRQ(ierr);
/*
Replace the first and last rows/columns of the matrix with the identity to obtain the zero Dirichlet boundary conditions
*/
rows[0] = 0;
rows[1] = q*(n-1);
ierr = MatZeroRowsColumns(K,2,rows,1.0,x,b);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,K,K);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCLU);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* compute the error to the continium problem */
ierr = ComputeSolution(da,&gll,b);CHKERRQ(ierr);
ierr = VecAXPY(x,-1.0,b);CHKERRQ(ierr);
/* compute the L^2 norm of the error */
ierr = VecGetArray(x,&f);CHKERRQ(ierr);
ierr = PetscGLLIntegrate(&gll,f,&norm);CHKERRQ(ierr);
ierr = VecRestoreArray(x,&f);CHKERRQ(ierr);
norm = PetscSqrtReal(norm);
ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD,"L^2 norm of the error %D %g\n",n,(double)norm);CHKERRQ(ierr);
if (n > 10 && norm > 1.e-8) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"Slower convergence than expected");
xc = (PetscReal)n;
yc = PetscLog10Real(norm);
ierr = PetscDrawLGAddPoint(lg,&xc,&yc);CHKERRQ(ierr);
ierr = PetscDrawLGDraw(lg);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = MatDestroy(&K);CHKERRQ(ierr);
ierr = PetscGLLElementLaplacianDestroy(&gll,&A);CHKERRQ(ierr);
ierr = PetscGLLDestroy(&gll);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
}
ierr = PetscDrawLGDestroy(&lg);CHKERRQ(ierr);
ierr = PetscDrawDestroy(&draw);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}