int main(int argc, char **argv)
{
Mat mat;
MatNullSpace nsp;
PetscBool prefix = PETSC_FALSE, flg;
PetscErrorCode ierr;
PetscInt zero = 0;
PetscScalar value = 0;
ierr = PetscInitialize(&argc, &argv, NULL, help); if (ierr) return ierr;
ierr = PetscOptionsGetBool(NULL, NULL, "-with_prefix",&prefix,NULL);CHKERRQ(ierr);
ierr = MatCreateDense(PETSC_COMM_WORLD, 1, 1, 1, 1, NULL, &mat);CHKERRQ(ierr);
ierr = MatSetOptionsPrefix(mat, prefix ? "prefix_" : NULL);CHKERRQ(ierr);
ierr = MatSetUp(mat);CHKERRQ(ierr);
ierr = MatSetValues(mat, 1, &zero, 1, &zero, &value, INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, NULL, &nsp);CHKERRQ(ierr);
ierr = MatNullSpaceTest(nsp, mat, &flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"Null space test failed!");
ierr = MatNullSpaceDestroy(&nsp);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
void PotentialSolve::SetupOperator(PotentialOp optype, double fval, vector<double> origin) {
//Set the matrix
switch (optype) {
case REALPBC :
_BuildCARTPBC();
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &mnull);
break;
case CARTPBC :
_BuildCARTPBC(fval);
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &mnull);
break;
case RADIAL :
_BuildRADIAL(fval, origin);
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &mnull);
break;
default :
RAISE_ERR(99, "Unknown operator type");
}
// Sanity check
PetscTruth isNull;
MatNullSpaceTest(mnull, a, &isNull);
if (isNull != PETSC_TRUE) {
PetscPrintf(PETSC_COMM_WORLD, "Warning -- Null space is not truly a null space!\n");
}
PetscPrintf(PETSC_COMM_WORLD, "Completed operator initialization\n");
// Initialize the solver
KSPCreate(PETSC_COMM_WORLD,&solver);
KSPSetOperators(solver, a, a,SAME_PRECONDITIONER);
KSPSetTolerances(solver, rtol, atol, PETSC_DEFAULT, maxit);
// Set some defaults
KSPSetType(solver,KSPLGMRES);
KSPGMRESSetRestart(solver, 10);
// Possible for user to override
KSPSetFromOptions(solver);
// Set up the null space
KSPSetNullSpace(solver, mnull);
PetscPrintf(PETSC_COMM_WORLD, "Solver setup\n");
}
int main(int argc,char **argv)
{
AppCtx appctx; /* user-defined application context */
PetscErrorCode ierr;
PetscInt i, xs, xm, ind, j, lenglob;
PetscReal x, *wrk_ptr1, *wrk_ptr2;
MatNullSpace nsp;
PetscMPIInt size;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Initialize program and set problem parameters
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
PetscFunctionBegin;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
/*initialize parameters */
appctx.param.N = 10; /* order of the spectral element */
appctx.param.E = 10; /* number of elements */
appctx.param.L = 4.0; /* length of the domain */
appctx.param.mu = 0.01; /* diffusion coefficient */
appctx.initial_dt = 5e-3;
appctx.param.steps = PETSC_MAX_INT;
appctx.param.Tend = 4;
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&appctx.param.N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-E",&appctx.param.E,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-Tend",&appctx.param.Tend,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-mu",&appctx.param.mu,NULL);CHKERRQ(ierr);
appctx.param.Le = appctx.param.L/appctx.param.E;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (appctx.param.E % size) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG,"Number of elements must be divisible by number of processes");
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create GLL data structures
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscGLLCreate(appctx.param.N,PETSCGLL_VIA_LINEARALGEBRA,&appctx.SEMop.gll);CHKERRQ(ierr);
lenglob = appctx.param.E*(appctx.param.N-1);
/*
Create distributed array (DMDA) to manage parallel grid and vectors
and to set up the ghost point communication pattern. There are E*(Nl-1)+1
total grid values spread equally among all the processors, except first and last
*/
ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_PERIODIC,lenglob,1,1,NULL,&appctx.da);CHKERRQ(ierr);
ierr = DMSetFromOptions(appctx.da);CHKERRQ(ierr);
ierr = DMSetUp(appctx.da);CHKERRQ(ierr);
/*
Extract global and local vectors from DMDA; we use these to store the
approximate solution. Then duplicate these for remaining vectors that
have the same types.
*/
ierr = DMCreateGlobalVector(appctx.da,&appctx.dat.curr_sol);CHKERRQ(ierr);
ierr = VecDuplicate(appctx.dat.curr_sol,&appctx.SEMop.grid);CHKERRQ(ierr);
ierr = VecDuplicate(appctx.dat.curr_sol,&appctx.SEMop.mass);CHKERRQ(ierr);
ierr = DMDAGetCorners(appctx.da,&xs,NULL,NULL,&xm,NULL,NULL);CHKERRQ(ierr);
ierr = DMDAVecGetArray(appctx.da,appctx.SEMop.grid,&wrk_ptr1);CHKERRQ(ierr);
ierr = DMDAVecGetArray(appctx.da,appctx.SEMop.mass,&wrk_ptr2);CHKERRQ(ierr);
/* Compute function over the locally owned part of the grid */
xs=xs/(appctx.param.N-1);
xm=xm/(appctx.param.N-1);
/*
Build total grid and mass over entire mesh (multi-elemental)
*/
for (i=xs; i<xs+xm; i++) {
for (j=0; j<appctx.param.N-1; j++) {
x = (appctx.param.Le/2.0)*(appctx.SEMop.gll.nodes[j]+1.0)+appctx.param.Le*i;
ind=i*(appctx.param.N-1)+j;
wrk_ptr1[ind]=x;
wrk_ptr2[ind]=.5*appctx.param.Le*appctx.SEMop.gll.weights[j];
if (j==0) wrk_ptr2[ind]+=.5*appctx.param.Le*appctx.SEMop.gll.weights[j];
}
}
ierr = DMDAVecRestoreArray(appctx.da,appctx.SEMop.grid,&wrk_ptr1);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(appctx.da,appctx.SEMop.mass,&wrk_ptr2);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create matrix data structure; set matrix evaluation routine.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DMSetMatrixPreallocateOnly(appctx.da, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMCreateMatrix(appctx.da,&appctx.SEMop.stiff);CHKERRQ(ierr);
ierr = DMCreateMatrix(appctx.da,&appctx.SEMop.grad);CHKERRQ(ierr);
/*
For linear problems with a time-dependent f(u,t) in the equation
u_t = f(u,t), the user provides the discretized right-hand-side
as a time-dependent matrix.
*/
ierr = RHSMatrixLaplaciangllDM(appctx.ts,0.0,appctx.dat.curr_sol,appctx.SEMop.stiff,appctx.SEMop.stiff,&appctx);CHKERRQ(ierr);
ierr = RHSMatrixAdvectiongllDM(appctx.ts,0.0,appctx.dat.curr_sol,appctx.SEMop.grad,appctx.SEMop.grad,&appctx);CHKERRQ(ierr);
/*
For linear problems with a time-dependent f(u,t) in the equation
u_t = f(u,t), the user provides the discretized right-hand-side
as a time-dependent matrix.
*/
ierr = MatDuplicate(appctx.SEMop.stiff,MAT_COPY_VALUES,&appctx.SEMop.keptstiff);CHKERRQ(ierr);
/* attach the null space to the matrix, this probably is not needed but does no harm */
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,NULL,&nsp);CHKERRQ(ierr);
ierr = MatSetNullSpace(appctx.SEMop.stiff,nsp);CHKERRQ(ierr);
ierr = MatSetNullSpace(appctx.SEMop.keptstiff,nsp);CHKERRQ(ierr);
ierr = MatNullSpaceTest(nsp,appctx.SEMop.stiff,NULL);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nsp);CHKERRQ(ierr);
/* attach the null space to the matrix, this probably is not needed but does no harm */
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,NULL,&nsp);CHKERRQ(ierr);
ierr = MatSetNullSpace(appctx.SEMop.grad,nsp);CHKERRQ(ierr);
ierr = MatNullSpaceTest(nsp,appctx.SEMop.grad,NULL);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nsp);CHKERRQ(ierr);
/* Create the TS solver that solves the ODE and its adjoint; set its options */
ierr = TSCreate(PETSC_COMM_WORLD,&appctx.ts);CHKERRQ(ierr);
ierr = TSSetProblemType(appctx.ts,TS_NONLINEAR);CHKERRQ(ierr);
ierr = TSSetType(appctx.ts,TSRK);CHKERRQ(ierr);
ierr = TSSetDM(appctx.ts,appctx.da);CHKERRQ(ierr);
ierr = TSSetTime(appctx.ts,0.0);CHKERRQ(ierr);
ierr = TSSetTimeStep(appctx.ts,appctx.initial_dt);CHKERRQ(ierr);
ierr = TSSetMaxSteps(appctx.ts,appctx.param.steps);CHKERRQ(ierr);
ierr = TSSetMaxTime(appctx.ts,appctx.param.Tend);CHKERRQ(ierr);
ierr = TSSetExactFinalTime(appctx.ts,TS_EXACTFINALTIME_MATCHSTEP);CHKERRQ(ierr);
ierr = TSSetTolerances(appctx.ts,1e-7,NULL,1e-7,NULL);CHKERRQ(ierr);
ierr = TSSetSaveTrajectory(appctx.ts);CHKERRQ(ierr);
ierr = TSSetFromOptions(appctx.ts);CHKERRQ(ierr);
ierr = TSSetRHSFunction(appctx.ts,NULL,RHSFunction,&appctx);CHKERRQ(ierr);
ierr = TSSetRHSJacobian(appctx.ts,appctx.SEMop.stiff,appctx.SEMop.stiff,RHSJacobian,&appctx);CHKERRQ(ierr);
/* Set Initial conditions for the problem */
ierr = TrueSolution(appctx.ts,0,appctx.dat.curr_sol,&appctx);CHKERRQ(ierr);
ierr = TSSetSolutionFunction(appctx.ts,(PetscErrorCode (*)(TS,PetscReal,Vec,void *))TrueSolution,&appctx);CHKERRQ(ierr);
ierr = TSSetTime(appctx.ts,0.0);CHKERRQ(ierr);
ierr = TSSetStepNumber(appctx.ts,0);CHKERRQ(ierr);
ierr = TSSolve(appctx.ts,appctx.dat.curr_sol);CHKERRQ(ierr);
ierr = MatDestroy(&appctx.SEMop.stiff);CHKERRQ(ierr);
ierr = MatDestroy(&appctx.SEMop.keptstiff);CHKERRQ(ierr);
ierr = MatDestroy(&appctx.SEMop.grad);CHKERRQ(ierr);
ierr = VecDestroy(&appctx.SEMop.grid);CHKERRQ(ierr);
ierr = VecDestroy(&appctx.SEMop.mass);CHKERRQ(ierr);
ierr = VecDestroy(&appctx.dat.curr_sol);CHKERRQ(ierr);
ierr = PetscGLLDestroy(&appctx.SEMop.gll);CHKERRQ(ierr);
ierr = DMDestroy(&appctx.da);CHKERRQ(ierr);
ierr = TSDestroy(&appctx.ts);CHKERRQ(ierr);
/*
Always call PetscFinalize() before exiting a program. This routine
- finalizes the PETSc libraries as well as MPI
- provides summary and diagnostic information if certain runtime
options are chosen (e.g., -log_summary).
*/
ierr = PetscFinalize();
return ierr;
}
int main(int argc, char **argv)
{
SNES snes; /* nonlinear solver */
DM dm; /* problem definition */
Vec u,r; /* solution, residual vectors */
Mat A,J; /* Jacobian matrix */
MatNullSpace nullSpace; /* May be necessary for pressure */
AppCtx user; /* user-defined work context */
JacActionCtx userJ; /* context for Jacobian MF action */
PetscInt its; /* iterations for convergence */
PetscReal error = 0.0; /* L_2 error in the solution */
PetscInt numComponents = 0, f;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, NULL, help);CHKERRQ(ierr);
ierr = ProcessOptions(PETSC_COMM_WORLD, &user);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD, &snes);CHKERRQ(ierr);
ierr = CreateMesh(PETSC_COMM_WORLD, &user, &dm);CHKERRQ(ierr);
ierr = SNESSetDM(snes, dm);CHKERRQ(ierr);
ierr = SetupElement(dm, &user);CHKERRQ(ierr);
for (f = 0; f < NUM_FIELDS; ++f) {
PetscInt numComp;
ierr = PetscFEGetNumComponents(user.fe[f], &numComp);CHKERRQ(ierr);
numComponents += numComp;
}
ierr = PetscMalloc(NUM_FIELDS * sizeof(void (*)(const PetscReal[], PetscScalar *)), &user.exactFuncs);CHKERRQ(ierr);
user.fem.bcFuncs = (void (**)(const PetscReal[], PetscScalar *)) user.exactFuncs;
ierr = SetupExactSolution(dm, &user);CHKERRQ(ierr);
ierr = SetupSection(dm, &user);CHKERRQ(ierr);
ierr = DMPlexCreateClosureIndex(dm, NULL);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dm, &u);CHKERRQ(ierr);
ierr = VecDuplicate(u, &r);CHKERRQ(ierr);
ierr = DMSetMatType(dm,MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(dm, &J);CHKERRQ(ierr);
if (user.jacobianMF) {
PetscInt M, m, N, n;
ierr = MatGetSize(J, &M, &N);CHKERRQ(ierr);
ierr = MatGetLocalSize(J, &m, &n);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD, &A);CHKERRQ(ierr);
ierr = MatSetSizes(A, m, n, M, N);CHKERRQ(ierr);
ierr = MatSetType(A, MATSHELL);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatShellSetOperation(A, MATOP_MULT, (void (*)(void))FormJacobianAction);CHKERRQ(ierr);
userJ.dm = dm;
userJ.J = J;
userJ.user = &user;
ierr = DMCreateLocalVector(dm, &userJ.u);CHKERRQ(ierr);
ierr = DMPlexProjectFunctionLocal(dm, user.fe, user.exactFuncs, INSERT_BC_VALUES, userJ.u);CHKERRQ(ierr);
ierr = MatShellSetContext(A, &userJ);CHKERRQ(ierr);
} else {
A = J;
}
ierr = CreatePressureNullSpace(dm, &user, &nullSpace);CHKERRQ(ierr);
ierr = MatSetNullSpace(J, nullSpace);CHKERRQ(ierr);
if (A != J) {
ierr = MatSetNullSpace(A, nullSpace);CHKERRQ(ierr);
}
ierr = DMSNESSetFunctionLocal(dm, (PetscErrorCode (*)(DM,Vec,Vec,void*))DMPlexComputeResidualFEM,&user);CHKERRQ(ierr);
ierr = DMSNESSetJacobianLocal(dm, (PetscErrorCode (*)(DM,Vec,Mat,Mat,MatStructure*,void*))DMPlexComputeJacobianFEM,&user);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, A, J, NULL, NULL);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = DMPlexProjectFunction(dm, user.fe, user.exactFuncs, INSERT_ALL_VALUES, u);CHKERRQ(ierr);
if (user.showInitial) {ierr = DMVecViewLocal(dm, u, PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
if (user.runType == RUN_FULL) {
ierr = DMPlexProjectFunction(dm, user.fe, user.initialGuess, INSERT_VALUES, u);CHKERRQ(ierr);
if (user.showInitial) {ierr = DMVecViewLocal(dm, u, PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
if (user.debug) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial guess\n");CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = SNESSolve(snes, NULL, u);CHKERRQ(ierr);
ierr = SNESGetIterationNumber(snes, &its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Number of SNES iterations = %D\n", its);CHKERRQ(ierr);
ierr = DMPlexComputeL2Diff(dm, user.fe, user.exactFuncs, u, &error);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: %.3g\n", error);CHKERRQ(ierr);
if (user.showSolution) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Solution\n");CHKERRQ(ierr);
ierr = VecChop(u, 3.0e-9);CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
} else {
PetscReal res = 0.0;
/* Check discretization error */
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial guess\n");CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMPlexComputeL2Diff(dm, user.fe, user.exactFuncs, u, &error);CHKERRQ(ierr);
if (error >= 1.0e-11) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: %g\n", error);CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: < 1.0e-11\n", error);CHKERRQ(ierr);
}
/* Check residual */
ierr = SNESComputeFunction(snes, u, r);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial Residual\n");CHKERRQ(ierr);
ierr = VecChop(r, 1.0e-10);CHKERRQ(ierr);
ierr = VecView(r, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecNorm(r, NORM_2, &res);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Residual: %g\n", res);CHKERRQ(ierr);
/* Check Jacobian */
{
Vec b;
MatStructure flag;
PetscBool isNull;
ierr = SNESComputeJacobian(snes, u, &A, &A, &flag);CHKERRQ(ierr);
ierr = MatNullSpaceTest(nullSpace, J, &isNull);CHKERRQ(ierr);
if (!isNull) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_PLIB, "The null space calculated for the system operator is invalid.");
ierr = VecDuplicate(u, &b);CHKERRQ(ierr);
ierr = VecSet(r, 0.0);CHKERRQ(ierr);
ierr = SNESComputeFunction(snes, r, b);CHKERRQ(ierr);
ierr = MatMult(A, u, r);CHKERRQ(ierr);
ierr = VecAXPY(r, 1.0, b);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Au - b = Au + F(0)\n");CHKERRQ(ierr);
ierr = VecChop(r, 1.0e-10);CHKERRQ(ierr);
ierr = VecView(r, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecNorm(r, NORM_2, &res);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Linear L_2 Residual: %g\n", res);CHKERRQ(ierr);
}
}
if (user.runType == RUN_FULL) {
PetscViewer viewer;
Vec uLocal;
const char *name;
ierr = PetscViewerCreate(PETSC_COMM_WORLD, &viewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(viewer, PETSCVIEWERVTK);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(viewer, "ex62_sol.vtk");CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &uLocal);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) u, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) uLocal, name);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm, u, INSERT_VALUES, uLocal);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm, u, INSERT_VALUES, uLocal);CHKERRQ(ierr);
ierr = VecView(uLocal, viewer);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &uLocal);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = PetscFree(user.exactFuncs);CHKERRQ(ierr);
ierr = DestroyElement(&user);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
if (user.jacobianMF) {
ierr = VecDestroy(&userJ.u);CHKERRQ(ierr);
}
if (A != J) {
ierr = MatDestroy(&A);CHKERRQ(ierr);
}
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&r);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PCBDDCNullSpaceAdaptGlobal(PC pc)
{
PC_IS* pcis = (PC_IS*)(pc->data);
PC_BDDC* pcbddc = (PC_BDDC*)(pc->data);
KSP inv_change;
const Vec *nsp_vecs;
Vec *new_nsp_vecs;
PetscInt i,nsp_size,new_nsp_size,start_new;
PetscBool nsp_has_cnst;
MatNullSpace new_nsp;
PetscErrorCode ierr;
PetscFunctionBegin;
/* create KSP for change of basis */
ierr = MatGetSize(pcbddc->ChangeOfBasisMatrix,&i,NULL);CHKERRQ(ierr);
ierr = KSPCreate(PetscObjectComm((PetscObject)pc),&inv_change);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(inv_change,pc->erroriffailure);CHKERRQ(ierr);
ierr = KSPSetOperators(inv_change,pcbddc->ChangeOfBasisMatrix,pcbddc->ChangeOfBasisMatrix);CHKERRQ(ierr);
ierr = KSPSetTolerances(inv_change,1.e-8,1.e-8,PETSC_DEFAULT,2*i);CHKERRQ(ierr);
if (pcbddc->dbg_flag) {
ierr = KSPMonitorSet(inv_change,KSPMonitorDefault,pcbddc->dbg_viewer,NULL);CHKERRQ(ierr);
}
ierr = KSPSetUp(inv_change);CHKERRQ(ierr);
/* get nullspace and transform it */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nsp_has_cnst,&nsp_size,&nsp_vecs);CHKERRQ(ierr);
new_nsp_size = nsp_size;
if (nsp_has_cnst) {
new_nsp_size++;
}
ierr = VecDuplicateVecs(pcis->vec1_global,new_nsp_size,&new_nsp_vecs);CHKERRQ(ierr);
start_new = 0;
if (nsp_has_cnst) {
start_new = 1;
ierr = VecSet(new_nsp_vecs[0],1.0);CHKERRQ(ierr);
if (pcbddc->dbg_flag) {
ierr = PetscViewerFlush(pcbddc->dbg_viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pcbddc->dbg_viewer,"Mapping constant in nullspace\n");CHKERRQ(ierr);
}
ierr = KSPSolve(inv_change,new_nsp_vecs[0],new_nsp_vecs[0]);CHKERRQ(ierr);
}
for (i=0;i<nsp_size;i++) {
ierr = PetscViewerFlush(pcbddc->dbg_viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pcbddc->dbg_viewer,"Mapping %dth vector in nullspace\n",i);CHKERRQ(ierr);
ierr = KSPSolve(inv_change,nsp_vecs[i],new_nsp_vecs[i+start_new]);CHKERRQ(ierr);
}
ierr = PCBDDCOrthonormalizeVecs(new_nsp_size,new_nsp_vecs);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)pc),PETSC_FALSE,new_nsp_size,new_nsp_vecs,&new_nsp);CHKERRQ(ierr);
ierr = PCBDDCSetNullSpace(pc,new_nsp);CHKERRQ(ierr);
/* free */
ierr = KSPDestroy(&inv_change);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&new_nsp);CHKERRQ(ierr);
ierr = VecDestroyVecs(new_nsp_size,&new_nsp_vecs);CHKERRQ(ierr);
/* check */
if (pcbddc->dbg_flag) {
PetscBool nsp_t=PETSC_FALSE;
Mat temp_mat;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
ierr = PetscPrintf(PetscObjectComm((PetscObject)(pc->pmat)),"Check nullspace with change of basis: %d\n",nsp_t);CHKERRQ(ierr);
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
}
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
SNES snes; /* nonlinear solver */
Vec u,r; /* solution, residual vectors */
Mat A,J; /* Jacobian matrix */
MatNullSpace nullSpace; /* May be necessary for pressure */
AppCtx user; /* user-defined work context */
JacActionCtx userJ; /* context for Jacobian MF action */
PetscInt its; /* iterations for convergence */
PetscReal error = 0.0; /* L_2 error in the solution */
const PetscInt numComponents = NUM_BASIS_COMPONENTS_TOTAL;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, NULL, help);CHKERRQ(ierr);
ierr = ProcessOptions(PETSC_COMM_WORLD, &user);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD, &snes);CHKERRQ(ierr);
ierr = CreateMesh(PETSC_COMM_WORLD, &user, &user.dm);CHKERRQ(ierr);
ierr = SNESSetDM(snes, user.dm);CHKERRQ(ierr);
ierr = SetupExactSolution(user.dm, &user);CHKERRQ(ierr);
ierr = SetupQuadrature(&user);CHKERRQ(ierr);
ierr = SetupSection(user.dm, &user);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user.dm, &u);CHKERRQ(ierr);
ierr = VecDuplicate(u, &r);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.dm, MATAIJ, &J);CHKERRQ(ierr);
if (user.jacobianMF) {
PetscInt M, m, N, n;
ierr = MatGetSize(J, &M, &N);CHKERRQ(ierr);
ierr = MatGetLocalSize(J, &m, &n);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD, &A);CHKERRQ(ierr);
ierr = MatSetSizes(A, m, n, M, N);CHKERRQ(ierr);
ierr = MatSetType(A, MATSHELL);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatShellSetOperation(A, MATOP_MULT, (void (*)(void))FormJacobianAction);CHKERRQ(ierr);
userJ.dm = user.dm;
userJ.J = J;
userJ.user = &user;
ierr = DMCreateLocalVector(user.dm, &userJ.u);CHKERRQ(ierr);
ierr = DMPlexProjectFunctionLocal(user.dm, numComponents, user.exactFuncs, INSERT_BC_VALUES, userJ.u);CHKERRQ(ierr);
ierr = MatShellSetContext(A, &userJ);CHKERRQ(ierr);
} else {
A = J;
}
ierr = CreatePressureNullSpace(user.dm, &user, &nullSpace);CHKERRQ(ierr);
ierr = MatSetNullSpace(J, nullSpace);CHKERRQ(ierr);
if (A != J) {
ierr = MatSetNullSpace(A, nullSpace);CHKERRQ(ierr);
}
ierr = DMSNESSetFunctionLocal(user.dm, (PetscErrorCode (*)(DM,Vec,Vec,void*))DMPlexComputeResidualFEM,&user);CHKERRQ(ierr);
ierr = DMSNESSetJacobianLocal(user.dm, (PetscErrorCode (*)(DM,Vec,Mat,Mat,MatStructure*,void*))DMPlexComputeJacobianFEM,&user);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, A, J, NULL, NULL);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = DMPlexProjectFunction(user.dm, numComponents, user.exactFuncs, INSERT_ALL_VALUES, u);CHKERRQ(ierr);
if (user.showInitial) {ierr = DMVecViewLocal(user.dm, u, PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
if (user.runType == RUN_FULL) {
PetscScalar (*initialGuess[numComponents])(const PetscReal x[]);
PetscInt c;
for (c = 0; c < numComponents; ++c) initialGuess[c] = zero;
ierr = DMPlexProjectFunction(user.dm, numComponents, initialGuess, INSERT_VALUES, u);CHKERRQ(ierr);
if (user.showInitial) {ierr = DMVecViewLocal(user.dm, u, PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
if (user.debug) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial guess\n");CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = SNESSolve(snes, NULL, u);CHKERRQ(ierr);
ierr = SNESGetIterationNumber(snes, &its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Number of SNES iterations = %D\n", its);CHKERRQ(ierr);
ierr = DMPlexComputeL2Diff(user.dm, user.fem.quad, user.exactFuncs, u, &error);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: %.3g\n", error);CHKERRQ(ierr);
if (user.showSolution) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Solution\n");CHKERRQ(ierr);
ierr = VecChop(u, 3.0e-9);CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
} else {
PetscReal res = 0.0;
/* Check discretization error */
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial guess\n");CHKERRQ(ierr);
ierr = VecView(u, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMPlexComputeL2Diff(user.dm, user.fem.quad, user.exactFuncs, u, &error);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Error: %g\n", error);CHKERRQ(ierr);
/* Check residual */
ierr = SNESComputeFunction(snes, u, r);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Initial Residual\n");CHKERRQ(ierr);
ierr = VecChop(r, 1.0e-10);CHKERRQ(ierr);
ierr = VecView(r, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecNorm(r, NORM_2, &res);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "L_2 Residual: %g\n", res);CHKERRQ(ierr);
/* Check Jacobian */
{
Vec b;
MatStructure flag;
PetscBool isNull;
ierr = SNESComputeJacobian(snes, u, &A, &A, &flag);CHKERRQ(ierr);
ierr = MatNullSpaceTest(nullSpace, J, &isNull);CHKERRQ(ierr);
if (!isNull) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_PLIB, "The null space calculated for the system operator is invalid.");
ierr = VecDuplicate(u, &b);CHKERRQ(ierr);
ierr = VecSet(r, 0.0);CHKERRQ(ierr);
ierr = SNESComputeFunction(snes, r, b);CHKERRQ(ierr);
ierr = MatMult(A, u, r);CHKERRQ(ierr);
ierr = VecAXPY(r, 1.0, b);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Au - b = Au + F(0)\n");CHKERRQ(ierr);
ierr = VecChop(r, 1.0e-10);CHKERRQ(ierr);
ierr = VecView(r, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecNorm(r, NORM_2, &res);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Linear L_2 Residual: %g\n", res);CHKERRQ(ierr);
}
}
if (user.runType == RUN_FULL) {
PetscViewer viewer;
Vec uLocal;
ierr = PetscViewerCreate(PETSC_COMM_WORLD, &viewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(viewer, PETSCVIEWERVTK);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(viewer, "ex62_sol.vtk");CHKERRQ(ierr);
ierr = DMGetLocalVector(user.dm, &uLocal);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(user.dm, u, INSERT_VALUES, uLocal);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user.dm, u, INSERT_VALUES, uLocal);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject) user.dm);CHKERRQ(ierr); /* Needed because viewer destroys the DM */
ierr = PetscObjectReference((PetscObject) uLocal);CHKERRQ(ierr); /* Needed because viewer destroys the Vec */
ierr = PetscViewerVTKAddField(viewer, (PetscObject) user.dm, DMPlexVTKWriteAll, PETSC_VTK_POINT_FIELD, (PetscObject) uLocal);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(user.dm, &uLocal);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = DestroyQuadrature(&user);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
if (user.jacobianMF) {
ierr = VecDestroy(&userJ.u);CHKERRQ(ierr);
}
if (A != J) {
ierr = MatDestroy(&A);CHKERRQ(ierr);
}
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&r);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = DMDestroy(&user.dm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PCBDDCNullSpaceAdaptGlobal(PC pc)
{
PC_IS* pcis = (PC_IS*) (pc->data);
PC_BDDC* pcbddc = (PC_BDDC*)(pc->data);
KSP inv_change;
PC pc_change;
const Vec *nsp_vecs;
Vec *new_nsp_vecs;
PetscInt i,nsp_size,new_nsp_size,start_new;
PetscBool nsp_has_cnst;
MatNullSpace new_nsp;
PetscErrorCode ierr;
MPI_Comm comm;
PetscFunctionBegin;
/* create KSP for change of basis */
ierr = KSPCreate(PETSC_COMM_SELF,&inv_change);CHKERRQ(ierr);
ierr = KSPSetOperators(inv_change,pcbddc->ChangeOfBasisMatrix,pcbddc->ChangeOfBasisMatrix,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetType(inv_change,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(inv_change,&pc_change);CHKERRQ(ierr);
ierr = PCSetType(pc_change,PCLU);CHKERRQ(ierr);
ierr = KSPSetUp(inv_change);CHKERRQ(ierr);
/* get nullspace and transform it */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nsp_has_cnst,&nsp_size,&nsp_vecs);CHKERRQ(ierr);
new_nsp_size = nsp_size;
if (nsp_has_cnst) {
new_nsp_size++;
}
ierr = PetscMalloc(new_nsp_size*sizeof(Vec),&new_nsp_vecs);CHKERRQ(ierr);
for (i=0;i<new_nsp_size;i++) {
ierr = VecDuplicate(pcis->vec1_global,&new_nsp_vecs[i]);CHKERRQ(ierr);
}
start_new = 0;
if (nsp_has_cnst) {
start_new = 1;
ierr = VecSet(new_nsp_vecs[0],1.0);CHKERRQ(ierr);
ierr = VecSet(pcis->vec1_B,1.0);CHKERRQ(ierr);
ierr = KSPSolve(inv_change,pcis->vec1_B,pcis->vec1_B);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[0],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[0],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
for (i=0;i<nsp_size;i++) {
ierr = VecCopy(nsp_vecs[i],new_nsp_vecs[i+start_new]);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,nsp_vecs[i],pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,nsp_vecs[i],pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = KSPSolve(inv_change,pcis->vec1_B,pcis->vec1_B);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[i+start_new],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[i+start_new],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
ierr = PCBDDCOrthonormalizeVecs(new_nsp_size,new_nsp_vecs);CHKERRQ(ierr);
#if 0
PetscBool nsp_t=PETSC_FALSE;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("Original Null Space test: %d\n",nsp_t);
Mat temp_mat;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("Original Null Space, mat changed test: %d\n",nsp_t);
{
PetscReal test_norm;
for (i=0;i<new_nsp_size;i++) {
ierr = MatMult(pc->pmat,new_nsp_vecs[i],pcis->vec1_global);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_global,NORM_2,&test_norm);CHKERRQ(ierr);
if (test_norm > 1.e-12) {
printf("------------ERROR VEC %d------------------\n",i);
ierr = VecView(pcis->vec1_global,PETSC_VIEWER_STDOUT_WORLD);
printf("------------------------------------------\n");
}
}
}
#endif
ierr = KSPDestroy(&inv_change);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(comm,PETSC_FALSE,new_nsp_size,new_nsp_vecs,&new_nsp);CHKERRQ(ierr);
ierr = PCBDDCSetNullSpace(pc,new_nsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&new_nsp);CHKERRQ(ierr);
#if 0
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("New Null Space, mat changed: %d\n",nsp_t);
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("New Null Space, mat original: %d\n",nsp_t);
#endif
for (i=0;i<new_nsp_size;i++) {
ierr = VecDestroy(&new_nsp_vecs[i]);CHKERRQ(ierr);
}
ierr = PetscFree(new_nsp_vecs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
