int main(int argc, char** argv)
{
/* the total number of grid points in each spatial direction is (n+1) */
/* the total number of degrees-of-freedom in each spatial direction is (n-1) */
/* this version requires n to be a power of 2 */
if( argc < 2 ) {
printf("need a problem size\n");
return 1;
}
int n = atoi(argv[1]);
int m = n-1;
// Initialize Petsc
PetscInitialize(&argc,&argv,0,PETSC_NULL);
// Create our vector
Vec b;
VecCreate(PETSC_COMM_WORLD,&b);
VecSetSizes(b,m*m,PETSC_DECIDE);
VecSetFromOptions(b);
VecSetUp(b);
// Create our matrix
Mat A;
MatCreate(PETSC_COMM_WORLD,&A);
MatSetType(A,MATSEQAIJ);
MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,m*m,m*m);
MatSetUp(A);
// Create linear solver object
KSP ksp;
KSPCreate(PETSC_COMM_WORLD,&ksp);
// setup rhs
PetscInt low, high;
VecGetOwnershipRange(b,&low,&high);
PetscInt* inds = (PetscInt*)malloc((high-low)*sizeof(PetscInt));
double* vals = (double*)malloc((high-low)*sizeof(double));
double h = 1./(double)n;
for (int i=0;i<high-1;++i) {
inds[i] = low+i;
vals[i] = h*h;
}
VecSetValues(b,high-low,inds,vals,INSERT_VALUES);
free(inds);
free(vals);
// setup matrix
for (int i=0;i<m*m;++i) {
MatSetValue(A,i,i,4.f,INSERT_VALUES);
if (i%m != m-1)
MatSetValue(A,i,i+1,-1.f,INSERT_VALUES);
if (i%m)
MatSetValue(A,i,i-1,-1.f,INSERT_VALUES);
if (i > m)
MatSetValue(A,i,i-m,-1.f,INSERT_VALUES);
if (i < m*(m-1))
MatSetValue(A,i,i+m,-1.f,INSERT_VALUES);
}
// sync processes
VecAssemblyBegin(b);
VecAssemblyEnd(b);
MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
// solve
KSPSetType(ksp,"cg");
KSPSetTolerances(ksp,1.e-10,1.e-10,1.e6,10000);
KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);
PC pc;
KSPGetPC(ksp,&pc);
PCSetType(pc,"ilu");
PCSetFromOptions(pc);
PCSetUp(pc);
// setup solver
KSPSetFromOptions(ksp);
KSPSetUp(ksp);
Vec x;
VecDuplicate(b,&x);
KSPSolve(ksp,b,x);
double val;
VecNorm(x,NORM_INFINITY,&val);
printf (" umax = %e \n",val);
PetscFinalize();
return 0;
}
krylov_petsc_info_t
krylov_petsc_solve
(
p4est_t* p4est,
problem_data_t* vecs,
weakeqn_ptrs_t* fcns,
p4est_ghost_t** ghost,
element_data_t** ghost_data,
dgmath_jit_dbase_t* dgmath_jit_dbase,
krylov_petsc_params_t* krylov_params
)
{
krylov_petsc_info_t info;
KSP ksp;
Vec x,b;
PC pc;
/* double* u_temp; */
/* double* rhs_temp; */
KSPCreate(PETSC_COMM_WORLD,&ksp);
VecCreate(PETSC_COMM_WORLD,&x);//CHKERRQ(ierr);
VecSetSizes(x, vecs->local_nodes, PETSC_DECIDE);//CHKERRQ(ierr);
VecSetFromOptions(x);//CHKERRQ(ierr);
VecDuplicate(x,&b);//CHKERRQ(ierr);
/* VecGetArray(x,&u_temp); */
/* VecGetArray(b,&rhs_temp); */
krylov_pc_ctx_t kct;
kct.p4est = p4est;
kct.vecs = vecs;
kct.fcns = fcns;
kct.ghost = ghost;
kct.ghost_data = ghost_data;
kct.dgmath_jit_dbase = dgmath_jit_dbase;
kct.pc_data = krylov_params->pc_data;
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_monitor","");
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_view","");
/* KSPMonitorSet(ksp, KSPMonitorDefault, NULL, NULL); */
/* PetscOptionsSetValue(NULL,"-ksp_converged_reason",""); */
PetscOptionsSetValue(NULL,"-ksp_atol","1e-20");
/* PetscOptionsSetValue(NULL,"-with-debugging","1"); */
PetscOptionsSetValue(NULL,"-ksp_rtol","1e-20");
PetscOptionsSetValue(NULL,"-ksp_max_it","1000000");
KSPGetPC(ksp,&pc);
krylov_pc_t* kp = NULL;
if (krylov_params != NULL && krylov_params->user_defined_pc) {
PCSetType(pc,PCSHELL);//CHKERRQ(ierr);
kp = krylov_params->pc_create(&kct);
PCShellSetApply(pc, krylov_petsc_pc_apply);//CHKERRQ(ierr);
PCShellSetSetUp(pc, krylov_petsc_pc_setup);
PCShellSetContext(pc, kp);//CHKERRQ(ierr);
}
else {
PCSetType(pc,PCNONE);//CHKERRQ(ierr);
}
KSPSetType(ksp, krylov_params->krylov_type);
KSPSetFromOptions(ksp);
/* Create matrix-free shell for Aij */
Mat A;
MatCreateShell
(
PETSC_COMM_WORLD,
vecs->local_nodes,
vecs->local_nodes,
PETSC_DETERMINE,
PETSC_DETERMINE,
(void*)&kct,
&A
);
MatShellSetOperation(A,MATOP_MULT,(void(*)())krylov_petsc_apply_aij);
/* Set Amat and Pmat, where Pmat is the matrix the Preconditioner needs */
KSPSetOperators(ksp,A,A);
/* linalg_copy_1st_to_2nd(vecs->u, u_temp, vecs->local_nodes); */
/* linalg_copy_1st_to_2nd(vecs->rhs, rhs_temp, vecs->local_nodes); */
VecPlaceArray(b, vecs->rhs);
VecPlaceArray(x, vecs->u);
KSPSolve(ksp,b,x);
if (krylov_params != NULL && krylov_params->user_defined_pc) {
krylov_params->pc_destroy(kp);
}
KSPGetIterationNumber(ksp, &(info.iterations));
KSPGetResidualNorm(ksp, &(info.residual_norm));
/* linalg_copy_1st_to_2nd(u_temp, vecs->u, vecs->local_nodes); */
/* VecRestoreArray(x,&u_temp); */
/* VecRestoreArray(b,&rhs_temp); */
VecResetArray(b);
VecResetArray(x);
VecDestroy(&x);
VecDestroy(&b);
KSPDestroy(&ksp);
return info;
}
static PetscErrorCode TaoSolve_NTR(Tao tao)
{
TAO_NTR *tr = (TAO_NTR *)tao->data;
PC pc;
KSPConvergedReason ksp_reason;
TaoConvergedReason reason;
PetscReal fmin, ftrial, prered, actred, kappa, sigma, beta;
PetscReal tau, tau_1, tau_2, tau_max, tau_min, max_radius;
PetscReal f, gnorm;
PetscReal delta;
PetscReal norm_d;
PetscErrorCode ierr;
PetscInt iter = 0;
PetscInt bfgsUpdates = 0;
PetscInt needH;
PetscInt i_max = 5;
PetscInt j_max = 1;
PetscInt i, j, N, n, its;
PetscFunctionBegin;
if (tao->XL || tao->XU || tao->ops->computebounds) {
ierr = PetscPrintf(((PetscObject)tao)->comm,"WARNING: Variable bounds have been set but will be ignored by ntr algorithm\n");CHKERRQ(ierr);
}
tao->trust = tao->trust0;
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
if (NTR_PC_BFGS == tr->pc_type && !tr->M) {
ierr = VecGetLocalSize(tao->solution,&n);CHKERRQ(ierr);
ierr = VecGetSize(tao->solution,&N);CHKERRQ(ierr);
ierr = MatCreateLMVM(((PetscObject)tao)->comm,n,N,&tr->M);CHKERRQ(ierr);
ierr = MatLMVMAllocateVectors(tr->M,tao->solution);CHKERRQ(ierr);
}
/* Check convergence criteria */
ierr = TaoComputeObjectiveAndGradient(tao, tao->solution, &f, tao->gradient);CHKERRQ(ierr);
ierr = VecNorm(tao->gradient,NORM_2,&gnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
needH = 1;
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, 1.0, &reason);CHKERRQ(ierr);
if (reason != TAO_CONTINUE_ITERATING) PetscFunctionReturn(0);
/* Create vectors for the limited memory preconditioner */
if ((NTR_PC_BFGS == tr->pc_type) &&
(BFGS_SCALE_BFGS != tr->bfgs_scale_type)) {
if (!tr->Diag) {
ierr = VecDuplicate(tao->solution, &tr->Diag);CHKERRQ(ierr);
}
}
switch(tr->ksp_type) {
case NTR_KSP_NASH:
ierr = KSPSetType(tao->ksp, KSPNASH);CHKERRQ(ierr);
if (tao->ksp->ops->setfromoptions) {
(*tao->ksp->ops->setfromoptions)(tao->ksp);
}
break;
case NTR_KSP_STCG:
ierr = KSPSetType(tao->ksp, KSPSTCG);CHKERRQ(ierr);
if (tao->ksp->ops->setfromoptions) {
(*tao->ksp->ops->setfromoptions)(tao->ksp);
}
break;
default:
ierr = KSPSetType(tao->ksp, KSPGLTR);CHKERRQ(ierr);
if (tao->ksp->ops->setfromoptions) {
(*tao->ksp->ops->setfromoptions)(tao->ksp);
}
break;
}
/* Modify the preconditioner to use the bfgs approximation */
ierr = KSPGetPC(tao->ksp, &pc);CHKERRQ(ierr);
switch(tr->pc_type) {
case NTR_PC_NONE:
ierr = PCSetType(pc, PCNONE);CHKERRQ(ierr);
if (pc->ops->setfromoptions) {
(*pc->ops->setfromoptions)(pc);
}
break;
case NTR_PC_AHESS:
ierr = PCSetType(pc, PCJACOBI);CHKERRQ(ierr);
if (pc->ops->setfromoptions) {
(*pc->ops->setfromoptions)(pc);
}
ierr = PCJacobiSetUseAbs(pc);CHKERRQ(ierr);
break;
case NTR_PC_BFGS:
ierr = PCSetType(pc, PCSHELL);CHKERRQ(ierr);
if (pc->ops->setfromoptions) {
(*pc->ops->setfromoptions)(pc);
}
ierr = PCShellSetName(pc, "bfgs");CHKERRQ(ierr);
ierr = PCShellSetContext(pc, tr->M);CHKERRQ(ierr);
ierr = PCShellSetApply(pc, MatLMVMSolveShell);CHKERRQ(ierr);
break;
default:
/* Use the pc method set by pc_type */
break;
}
/* Initialize trust-region radius */
switch(tr->init_type) {
case NTR_INIT_CONSTANT:
/* Use the initial radius specified */
break;
case NTR_INIT_INTERPOLATION:
/* Use the initial radius specified */
max_radius = 0.0;
for (j = 0; j < j_max; ++j) {
fmin = f;
sigma = 0.0;
if (needH) {
ierr = TaoComputeHessian(tao,tao->solution,tao->hessian,tao->hessian_pre);CHKERRQ(ierr);
needH = 0;
}
for (i = 0; i < i_max; ++i) {
ierr = VecCopy(tao->solution, tr->W);CHKERRQ(ierr);
ierr = VecAXPY(tr->W, -tao->trust/gnorm, tao->gradient);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao, tr->W, &ftrial);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(ftrial)) {
tau = tr->gamma1_i;
}
else {
if (ftrial < fmin) {
fmin = ftrial;
sigma = -tao->trust / gnorm;
}
ierr = MatMult(tao->hessian, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = VecDot(tao->gradient, tao->stepdirection, &prered);CHKERRQ(ierr);
prered = tao->trust * (gnorm - 0.5 * tao->trust * prered / (gnorm * gnorm));
actred = f - ftrial;
if ((PetscAbsScalar(actred) <= tr->epsilon) &&
(PetscAbsScalar(prered) <= tr->epsilon)) {
kappa = 1.0;
}
else {
kappa = actred / prered;
}
tau_1 = tr->theta_i * gnorm * tao->trust / (tr->theta_i * gnorm * tao->trust + (1.0 - tr->theta_i) * prered - actred);
tau_2 = tr->theta_i * gnorm * tao->trust / (tr->theta_i * gnorm * tao->trust - (1.0 + tr->theta_i) * prered + actred);
tau_min = PetscMin(tau_1, tau_2);
tau_max = PetscMax(tau_1, tau_2);
if (PetscAbsScalar(kappa - 1.0) <= tr->mu1_i) {
/* Great agreement */
max_radius = PetscMax(max_radius, tao->trust);
if (tau_max < 1.0) {
tau = tr->gamma3_i;
}
else if (tau_max > tr->gamma4_i) {
tau = tr->gamma4_i;
}
else {
tau = tau_max;
}
}
else if (PetscAbsScalar(kappa - 1.0) <= tr->mu2_i) {
/* Good agreement */
max_radius = PetscMax(max_radius, tao->trust);
if (tau_max < tr->gamma2_i) {
tau = tr->gamma2_i;
}
else if (tau_max > tr->gamma3_i) {
tau = tr->gamma3_i;
}
else {
tau = tau_max;
}
}
else {
/* Not good agreement */
if (tau_min > 1.0) {
tau = tr->gamma2_i;
}
else if (tau_max < tr->gamma1_i) {
tau = tr->gamma1_i;
}
else if ((tau_min < tr->gamma1_i) && (tau_max >= 1.0)) {
tau = tr->gamma1_i;
}
else if ((tau_1 >= tr->gamma1_i) && (tau_1 < 1.0) &&
((tau_2 < tr->gamma1_i) || (tau_2 >= 1.0))) {
tau = tau_1;
}
else if ((tau_2 >= tr->gamma1_i) && (tau_2 < 1.0) &&
((tau_1 < tr->gamma1_i) || (tau_2 >= 1.0))) {
tau = tau_2;
}
else {
tau = tau_max;
}
}
}
tao->trust = tau * tao->trust;
}
if (fmin < f) {
f = fmin;
ierr = VecAXPY(tao->solution, sigma, tao->gradient);CHKERRQ(ierr);
ierr = TaoComputeGradient(tao,tao->solution, tao->gradient);CHKERRQ(ierr);
ierr = VecNorm(tao->gradient, NORM_2, &gnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
needH = 1;
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, 1.0, &reason);CHKERRQ(ierr);
if (reason != TAO_CONTINUE_ITERATING) {
PetscFunctionReturn(0);
}
}
}
tao->trust = PetscMax(tao->trust, max_radius);
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
break;
default:
/* Norm of the first direction will initialize radius */
tao->trust = 0.0;
break;
}
/* Set initial scaling for the BFGS preconditioner
This step is done after computing the initial trust-region radius
since the function value may have decreased */
if (NTR_PC_BFGS == tr->pc_type) {
if (f != 0.0) {
delta = 2.0 * PetscAbsScalar(f) / (gnorm*gnorm);
}
else {
delta = 2.0 / (gnorm*gnorm);
}
ierr = MatLMVMSetDelta(tr->M,delta);CHKERRQ(ierr);
}
/* Have not converged; continue with Newton method */
while (reason == TAO_CONTINUE_ITERATING) {
++iter;
tao->ksp_its=0;
/* Compute the Hessian */
if (needH) {
ierr = TaoComputeHessian(tao,tao->solution,tao->hessian,tao->hessian_pre);CHKERRQ(ierr);
needH = 0;
}
if (NTR_PC_BFGS == tr->pc_type) {
if (BFGS_SCALE_AHESS == tr->bfgs_scale_type) {
/* Obtain diagonal for the bfgs preconditioner */
ierr = MatGetDiagonal(tao->hessian, tr->Diag);CHKERRQ(ierr);
ierr = VecAbs(tr->Diag);CHKERRQ(ierr);
ierr = VecReciprocal(tr->Diag);CHKERRQ(ierr);
ierr = MatLMVMSetScale(tr->M,tr->Diag);CHKERRQ(ierr);
}
/* Update the limited memory preconditioner */
ierr = MatLMVMUpdate(tr->M, tao->solution, tao->gradient);CHKERRQ(ierr);
++bfgsUpdates;
}
while (reason == TAO_CONTINUE_ITERATING) {
ierr = KSPSetOperators(tao->ksp, tao->hessian, tao->hessian_pre);CHKERRQ(ierr);
/* Solve the trust region subproblem */
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPNASHGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPSTCGGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else { /* NTR_KSP_GLTR */
ierr = KSPGLTRSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPGLTRGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
}
if (0.0 == tao->trust) {
/* Radius was uninitialized; use the norm of the direction */
if (norm_d > 0.0) {
tao->trust = norm_d;
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
}
else {
/* The direction was bad; set radius to default value and re-solve
the trust-region subproblem to get a direction */
tao->trust = tao->trust0;
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPNASHGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPSTCGGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else { /* NTR_KSP_GLTR */
ierr = KSPGLTRSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPGLTRGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
}
if (norm_d == 0.0) SETERRQ(PETSC_COMM_SELF,1, "Initial direction zero");
}
}
ierr = VecScale(tao->stepdirection, -1.0);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(tao->ksp, &ksp_reason);CHKERRQ(ierr);
if ((KSP_DIVERGED_INDEFINITE_PC == ksp_reason) &&
(NTR_PC_BFGS == tr->pc_type) && (bfgsUpdates > 1)) {
/* Preconditioner is numerically indefinite; reset the
approximate if using BFGS preconditioning. */
if (f != 0.0) {
delta = 2.0 * PetscAbsScalar(f) / (gnorm*gnorm);
}
else {
delta = 2.0 / (gnorm*gnorm);
}
ierr = MatLMVMSetDelta(tr->M, delta);CHKERRQ(ierr);
ierr = MatLMVMReset(tr->M);CHKERRQ(ierr);
ierr = MatLMVMUpdate(tr->M, tao->solution, tao->gradient);CHKERRQ(ierr);
bfgsUpdates = 1;
}
if (NTR_UPDATE_REDUCTION == tr->update_type) {
/* Get predicted reduction */
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else { /* gltr */
ierr = KSPGLTRGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
}
if (prered >= 0.0) {
/* The predicted reduction has the wrong sign. This cannot
happen in infinite precision arithmetic. Step should
be rejected! */
tao->trust = tr->alpha1 * PetscMin(tao->trust, norm_d);
}
else {
/* Compute trial step and function value */
ierr = VecCopy(tao->solution,tr->W);CHKERRQ(ierr);
ierr = VecAXPY(tr->W, 1.0, tao->stepdirection);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao, tr->W, &ftrial);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(ftrial)) {
tao->trust = tr->alpha1 * PetscMin(tao->trust, norm_d);
} else {
/* Compute and actual reduction */
actred = f - ftrial;
prered = -prered;
if ((PetscAbsScalar(actred) <= tr->epsilon) &&
(PetscAbsScalar(prered) <= tr->epsilon)) {
kappa = 1.0;
}
else {
kappa = actred / prered;
}
/* Accept or reject the step and update radius */
if (kappa < tr->eta1) {
/* Reject the step */
tao->trust = tr->alpha1 * PetscMin(tao->trust, norm_d);
}
else {
/* Accept the step */
if (kappa < tr->eta2) {
/* Marginal bad step */
tao->trust = tr->alpha2 * PetscMin(tao->trust, norm_d);
}
else if (kappa < tr->eta3) {
/* Reasonable step */
tao->trust = tr->alpha3 * tao->trust;
}
else if (kappa < tr->eta4) {
/* Good step */
tao->trust = PetscMax(tr->alpha4 * norm_d, tao->trust);
}
else {
/* Very good step */
tao->trust = PetscMax(tr->alpha5 * norm_d, tao->trust);
}
break;
}
}
}
}
else {
/* Get predicted reduction */
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else { /* gltr */
ierr = KSPGLTRGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
}
if (prered >= 0.0) {
/* The predicted reduction has the wrong sign. This cannot
happen in infinite precision arithmetic. Step should
be rejected! */
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else {
ierr = VecCopy(tao->solution, tr->W);CHKERRQ(ierr);
ierr = VecAXPY(tr->W, 1.0, tao->stepdirection);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao, tr->W, &ftrial);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(ftrial)) {
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else {
ierr = VecDot(tao->gradient, tao->stepdirection, &beta);CHKERRQ(ierr);
actred = f - ftrial;
prered = -prered;
if ((PetscAbsScalar(actred) <= tr->epsilon) &&
(PetscAbsScalar(prered) <= tr->epsilon)) {
kappa = 1.0;
}
else {
kappa = actred / prered;
}
tau_1 = tr->theta * beta / (tr->theta * beta - (1.0 - tr->theta) * prered + actred);
tau_2 = tr->theta * beta / (tr->theta * beta + (1.0 + tr->theta) * prered - actred);
tau_min = PetscMin(tau_1, tau_2);
tau_max = PetscMax(tau_1, tau_2);
if (kappa >= 1.0 - tr->mu1) {
/* Great agreement; accept step and update radius */
if (tau_max < 1.0) {
tao->trust = PetscMax(tao->trust, tr->gamma3 * norm_d);
}
else if (tau_max > tr->gamma4) {
tao->trust = PetscMax(tao->trust, tr->gamma4 * norm_d);
}
else {
tao->trust = PetscMax(tao->trust, tau_max * norm_d);
}
break;
}
else if (kappa >= 1.0 - tr->mu2) {
/* Good agreement */
if (tau_max < tr->gamma2) {
tao->trust = tr->gamma2 * PetscMin(tao->trust, norm_d);
}
else if (tau_max > tr->gamma3) {
tao->trust = PetscMax(tao->trust, tr->gamma3 * norm_d);
}
else if (tau_max < 1.0) {
tao->trust = tau_max * PetscMin(tao->trust, norm_d);
}
else {
tao->trust = PetscMax(tao->trust, tau_max * norm_d);
}
break;
}
else {
/* Not good agreement */
if (tau_min > 1.0) {
tao->trust = tr->gamma2 * PetscMin(tao->trust, norm_d);
}
else if (tau_max < tr->gamma1) {
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else if ((tau_min < tr->gamma1) && (tau_max >= 1.0)) {
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else if ((tau_1 >= tr->gamma1) && (tau_1 < 1.0) &&
((tau_2 < tr->gamma1) || (tau_2 >= 1.0))) {
tao->trust = tau_1 * PetscMin(tao->trust, norm_d);
}
else if ((tau_2 >= tr->gamma1) && (tau_2 < 1.0) &&
((tau_1 < tr->gamma1) || (tau_2 >= 1.0))) {
tao->trust = tau_2 * PetscMin(tao->trust, norm_d);
}
else {
tao->trust = tau_max * PetscMin(tao->trust, norm_d);
}
}
}
}
}
/* The step computed was not good and the radius was decreased.
Monitor the radius to terminate. */
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, tao->trust, &reason);CHKERRQ(ierr);
}
/* The radius may have been increased; modify if it is too large */
tao->trust = PetscMin(tao->trust, tr->max_radius);
if (reason == TAO_CONTINUE_ITERATING) {
ierr = VecCopy(tr->W, tao->solution);CHKERRQ(ierr);
f = ftrial;
ierr = TaoComputeGradient(tao, tao->solution, tao->gradient);
ierr = VecNorm(tao->gradient, NORM_2, &gnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
needH = 1;
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, tao->trust, &reason);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
/*
* steady_state solves for the steady_state of the system
* that was previously setup using the add_to_ham and add_lin
* routines. Solver selection and parameterscan be controlled via PETSc
* command line options.
*/
void steady_state(Vec x){
PetscViewer mat_view;
PC pc;
Vec b;
KSP ksp; /* linear solver context */
PetscInt row,col,its,j,i,Istart,Iend;
PetscScalar mat_tmp;
long dim;
int num_pop;
double *populations;
Mat solve_A;
if (_lindblad_terms) {
dim = total_levels*total_levels;
solve_A = full_A;
if (nid==0) {
printf("Lindblad terms found, using Lindblad solver.");
}
} else {
if (nid==0) {
printf("Warning! Steady state not supported for Schrodinger.\n");
printf(" Defaulting to (less efficient) Lindblad Solver\n");
exit(0);
}
dim = total_levels*total_levels;
solve_A = ham_A;
}
if (!stab_added){
if (nid==0) printf("Adding stabilization...\n");
/*
* Add elements to the matrix to make the normalization work
* I have no idea why this works, I am copying it from qutip
* We add 1.0 in the 0th spot and every n+1 after
*/
if (nid==0) {
row = 0;
for (i=0;i<total_levels;i++){
col = i*(total_levels+1);
mat_tmp = 1.0 + 0.*PETSC_i;
MatSetValue(full_A,row,col,mat_tmp,ADD_VALUES);
}
/* Print dense ham, if it was asked for */
if (_print_dense_ham){
FILE *fp_ham;
fp_ham = fopen("ham","w");
if (nid==0){
for (i=0;i<total_levels;i++){
for (j=0;j<total_levels;j++){
fprintf(fp_ham,"%e %e ",PetscRealPart(_hamiltonian[i][j]),PetscImaginaryPart(_hamiltonian[i][j]));
}
fprintf(fp_ham,"\n");
}
}
fclose(fp_ham);
for (i=0;i<total_levels;i++){
free(_hamiltonian[i]);
}
free(_hamiltonian);
_print_dense_ham = 0;
}
}
stab_added = 1;
}
// if (!matrix_assembled) {
MatGetOwnershipRange(full_A,&Istart,&Iend);
/*
* Explicitly add 0.0 to all diagonal elements;
* this fixes a 'matrix in wrong state' message that PETSc
* gives if the diagonal was never initialized.
*/
if (nid==0) printf("Adding 0 to diagonal elements...\n");
for (i=Istart;i<Iend;i++){
mat_tmp = 0 + 0.*PETSC_i;
MatSetValue(full_A,i,i,mat_tmp,ADD_VALUES);
}
/* Tell PETSc to assemble the matrix */
MatAssemblyBegin(full_A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(full_A,MAT_FINAL_ASSEMBLY);
if (nid==0) printf("Matrix Assembled.\n");
matrix_assembled = 1;
// }
/* Print information about the matrix. */
PetscViewerASCIIOpen(PETSC_COMM_WORLD,NULL,&mat_view);
PetscViewerPushFormat(mat_view,PETSC_VIEWER_ASCII_INFO);
MatView(full_A,mat_view);
PetscViewerPopFormat(mat_view);
PetscViewerDestroy(&mat_view);
/*
* Create parallel vectors.
* - When using VecCreate(), VecSetSizes() and VecSetFromOptions(),
* we specify only the vector's global
* dimension; the parallel partitioning is determined at runtime.
* - Note: We form 1 vector from scratch and then duplicate as needed.
*/
VecCreate(PETSC_COMM_WORLD,&b);
VecSetSizes(b,PETSC_DECIDE,dim);
VecSetFromOptions(b);
// VecDuplicate(b,&x); Assume x is passed in
/*
* Set rhs, b, and solution, x to 1.0 in the first
* element, 0.0 elsewhere.
*/
VecSet(b,0.0);
VecSet(x,0.0);
if(nid==0) {
row = 0;
mat_tmp = 1.0 + 0.0*PETSC_i;
VecSetValue(x,row,mat_tmp,INSERT_VALUES);
VecSetValue(b,row,mat_tmp,INSERT_VALUES);
}
/* Assemble x and b */
VecAssemblyBegin(x);
VecAssemblyEnd(x);
VecAssemblyBegin(b);
VecAssemblyEnd(b);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*
* Create the linear solver and set various options *
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
* Create linear solver context
*/
KSPCreate(PETSC_COMM_WORLD,&ksp);
/*
* Set operators. Here the matrix that defines the linear system
* also serves as the preconditioning matrix.
*/
KSPSetOperators(ksp,full_A,full_A);
/*
* Set good default options for solver
*/
/* relative tolerance */
KSPSetTolerances(ksp,default_rtol,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
/* bjacobi preconditioner */
KSPGetPC(ksp,&pc);
PCSetType(pc,PCASM);
/* gmres solver with 100 restart*/
KSPSetType(ksp,KSPGMRES);
KSPGMRESSetRestart(ksp,default_restart);
/*
* Set runtime options, e.g.,
* -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
*/
KSPSetFromOptions(ksp);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
if (nid==0) printf("KSP set. Solving for steady state...\n");
KSPSolve(ksp,b,x);
num_pop = get_num_populations();
populations = malloc(num_pop*sizeof(double));
get_populations(x,&populations);
if(nid==0){
printf("Final populations: ");
for(i=0;i<num_pop;i++){
printf(" %e ",populations[i]);
}
printf("\n");
}
KSPGetIterationNumber(ksp,&its);
PetscPrintf(PETSC_COMM_WORLD,"Iterations %D\n",its);
/* Free work space */
KSPDestroy(&ksp);
// VecDestroy(&x);
VecDestroy(&b);
return;
}
int main(int argc,char **argv)
{
KSP solver;
PC pc;
Mat A,B;
Vec X,Y,Z;
MatScalar *a;
PetscScalar *b,*x,*y,*z;
PetscReal nrm;
PetscErrorCode ierr,size=8,lda=10, i,j;
PetscInitialize(&argc,&argv,0,help);
/* Create matrix and three vectors: these are all normal */
ierr = PetscMalloc1(lda*size,&b);CHKERRQ(ierr);
for (i=0; i<size; i++) {
for (j=0; j<size; j++) {
b[i+j*lda] = rand();
}
}
ierr = MatCreate(MPI_COMM_SELF,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,size,size,size,size);CHKERRQ(ierr);
ierr = MatSetType(A,MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(A,NULL);CHKERRQ(ierr);
ierr = MatDenseGetArray(A,&a);CHKERRQ(ierr);
for (i=0; i<size; i++) {
for (j=0; j<size; j++) {
a[i+j*size] = b[i+j*lda];
}
}
ierr = MatDenseRestoreArray(A,&a);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCreate(MPI_COMM_SELF,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,size,size,size,size);CHKERRQ(ierr);
ierr = MatSetType(B,MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(B,b);CHKERRQ(ierr);
ierr = MatSeqDenseSetLDA(B,lda);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscMalloc1(size,&x);CHKERRQ(ierr);
for (i=0; i<size; i++) x[i] = 1.0;
ierr = VecCreateSeqWithArray(MPI_COMM_SELF,1,size,x,&X);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
ierr = PetscMalloc1(size,&y);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(MPI_COMM_SELF,1,size,y,&Y);CHKERRQ(ierr);
ierr = VecAssemblyBegin(Y);CHKERRQ(ierr);
ierr = VecAssemblyEnd(Y);CHKERRQ(ierr);
ierr = PetscMalloc1(size,&z);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(MPI_COMM_SELF,1,size,z,&Z);CHKERRQ(ierr);
ierr = VecAssemblyBegin(Z);CHKERRQ(ierr);
ierr = VecAssemblyEnd(Z);CHKERRQ(ierr);
/*
* Solve with A and B
*/
ierr = KSPCreate(MPI_COMM_SELF,&solver);CHKERRQ(ierr);
ierr = KSPSetType(solver,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(solver,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCLU);CHKERRQ(ierr);
ierr = KSPSetOperators(solver,A,A);CHKERRQ(ierr);
ierr = KSPSolve(solver,X,Y);CHKERRQ(ierr);
ierr = KSPSetOperators(solver,B,B);CHKERRQ(ierr);
ierr = KSPSolve(solver,X,Z);CHKERRQ(ierr);
ierr = VecAXPY(Z,-1.0,Y);CHKERRQ(ierr);
ierr = VecNorm(Z,NORM_2,&nrm);
printf("Test1; error norm=%e\n",nrm);
/* Free spaces */
ierr = PetscFree(b);CHKERRQ(ierr);
ierr = PetscFree(x);CHKERRQ(ierr);
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = PetscFree(z);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&Y);CHKERRQ(ierr);
ierr = VecDestroy(&Z);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = KSPDestroy(&solver);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
KSP ksp;
PC pc;
Mat A,M;
Vec X,B,D;
MPI_Comm comm;
PetscScalar v;
KSPConvergedReason reason;
PetscInt i,j,its;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscInitialize(&argc,&argv,0,0);CHKERRQ(ierr);
ierr = PetscOptionsSetValue("-options_left",PETSC_NULL);CHKERRQ(ierr);
comm = MPI_COMM_SELF;
/*
* Construct the Kershaw matrix
* and a suitable rhs / initial guess
*/
ierr = MatCreateSeqAIJ(comm,4,4,4,0,&A);CHKERRQ(ierr);
ierr = VecCreateSeq(comm,4,&B);CHKERRQ(ierr);
ierr = VecDuplicate(B,&X);CHKERRQ(ierr);
for (i=0; i<4; i++) {
v=3;
ierr = MatSetValues(A,1,&i,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
v=1;
ierr = VecSetValues(B,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecSetValues(X,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
}
i=0; v=0;
ierr = VecSetValues(X,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
for (i=0; i<3; i++) {
v=-2; j=i+1;
ierr = MatSetValues(A,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(A,1,&j,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
}
i=0; j=3; v=2;
ierr = MatSetValues(A,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(A,1,&j,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecAssemblyBegin(B);CHKERRQ(ierr);
ierr = VecAssemblyEnd(B);CHKERRQ(ierr);
printf("\nThe Kershaw matrix:\n\n"); MatView(A,0);
/*
* A Conjugate Gradient method
* with ILU(0) preconditioning
*/
ierr = KSPCreate(comm,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPCG);CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp,PETSC_TRUE);CHKERRQ(ierr);
/*
* ILU preconditioner;
* The iterative method will break down unless you comment in the SetShift
* line below, or use the -pc_factor_shift_positive_definite option.
* Run the code twice: once as given to see the negative pivot and the
* divergence behaviour, then comment in the Shift line, or add the
* command line option, and see that the pivots are all positive and
* the method converges.
*/
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCICC);CHKERRQ(ierr);
/* ierr = PCFactorSetShiftType(prec,MAT_SHIFT_POSITIVE_DEFINITE);CHKERRQ(ierr); */
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
/*
* Now that the factorisation is done, show the pivots;
* note that the last one is negative. This in itself is not an error,
* but it will make the iterative method diverge.
*/
ierr = PCFactorGetMatrix(pc,&M);CHKERRQ(ierr);
ierr = VecDuplicate(B,&D);CHKERRQ(ierr);
ierr = MatGetDiagonal(M,D);CHKERRQ(ierr);
printf("\nPivots:\n\n"); VecView(D,0);
/*
* Solve the system;
* without the shift this will diverge with
* an indefinite preconditioner
*/
ierr = KSPSolve(ksp,B,X);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(ksp,&reason);CHKERRQ(ierr);
if (reason==KSP_DIVERGED_INDEFINITE_PC) {
printf("\nDivergence because of indefinite preconditioner;\n");
printf("Run the executable again but with -pc_factor_shift_positive_definite option.\n");
} else if (reason<0) {
printf("\nOther kind of divergence: this should not happen.\n");
} else {
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
printf("\nConvergence in %d iterations.\n",(int)its);
}
printf("\n");
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
ierr = VecDestroy(B);CHKERRQ(ierr);
ierr = VecDestroy(X);CHKERRQ(ierr);
ierr = VecDestroy(D);CHKERRQ(ierr);
PetscFinalize();
PetscFunctionReturn(0);
}
int implicit_solver(HashTable* El_Table, HashTable* NodeTable, double delta_t, double LapCoef,
TimeProps* timeprops_ptr) {
Vec x, b, xlocal; /* approx solution, RHS */
Mat A; /* linear system matrix */
KSP ksp; /* KSP context */
PetscReal norm; //,val1,val2; /* norm of solution error */
PetscErrorCode ierr;
PetscInt xsize, *num_elem_proc, *to, *from, its;
PetscMPIInt rank, size;
KSPConvergedReason reason;
VecScatter vscat;
IS globalis, tois;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
MPI_Comm_size(PETSC_COMM_WORLD, &size);
/* -------------------------------------------------------------------
Compute the matrix and right-hand-side vector that define
the linear system, Ax = b.
------------------------------------------------------------------- */
ierr = PetscMalloc(size * sizeof(PetscInt), &num_elem_proc);
CHKERRQ(ierr);
num_elem_proc[rank] = num_nonzero_elem(El_Table);
if (rank > 0)
MPI_Send(&num_elem_proc[rank], 1, MPI_INT, 0, 22, PETSC_COMM_WORLD);
if (rank == 0)
for (int i = 1; i < size; i++)
MPI_Recv(&num_elem_proc[i], 1, MPI_INT, i, 22, PETSC_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Barrier(PETSC_COMM_WORLD);
MPI_Bcast(num_elem_proc, size, MPI_INT, 0, PETSC_COMM_WORLD);
//printf("Number of elements are (hi i am second)...........%d\n", num_nonzero_elem(Laplacian->El_Table));
int total_elem = 0, start_elem = 0;
//MPI_Allreduce(&num_elem, total_elem, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
ierr = PetscMalloc(num_elem_proc[rank] * sizeof(PetscInt), &to);
CHKERRQ(ierr);
ierr = PetscMalloc(num_elem_proc[rank] * sizeof(PetscInt), &from);
CHKERRQ(ierr);
for (int i = 0; i < size; i++)
total_elem += num_elem_proc[i];
for (int i = 0; i < rank; i++)
start_elem += num_elem_proc[i];
for (int i = 0; i < num_elem_proc[rank]; i++) {
from[i] = i;
to[i] = i + start_elem;
}
ierr = ISCreateGeneral(PETSC_COMM_WORLD, num_elem_proc[rank], from, PETSC_COPY_VALUES, &tois);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD, num_elem_proc[rank], to, PETSC_COPY_VALUES, &globalis);
CHKERRQ(ierr);
/*
Create parallel vectors
*/
ierr = VecCreate(PETSC_COMM_WORLD, &b);
CHKERRQ(ierr);
ierr = VecSetType(b, VECSTANDARD);
CHKERRQ(ierr);
ierr = VecSetSizes(b, num_elem_proc[rank], total_elem);
CHKERRQ(ierr);
ierr = VecSetFromOptions(b);
CHKERRQ(ierr);
ierr = VecGetSize(b, &xsize);
//cout<<"size b is "<<xsize<<endl;
ierr = VecCreateSeq(PETSC_COMM_SELF, num_elem_proc[rank], &xlocal);
CHKERRQ(ierr);
ierr = VecScatterCreate(xlocal, tois, b, globalis, &vscat);
CHKERRQ(ierr);
// we have to create a map between local and global vector
myctx.El_Table = El_Table;
myctx.Node_Table = NodeTable;
myctx.Scatter = vscat;
myctx.Total_elem = total_elem;
myctx.Num_elem_proc = num_elem_proc;
myctx.rank = rank;
myctx.size = size;
myctx.Timeptr = timeprops_ptr;
myctx.LapCoef = LapCoef;
myctx.delta_t = delta_t;
/*
right-hand-side vector.
*/
ierr = MakeRHS(&myctx, b);
CHKERRQ(ierr);
//ierr = VecView(b,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecDuplicate(b, &x);
CHKERRQ(ierr);
ierr = VecCopy(b, x);
CHKERRQ(ierr);
//ierr = VecView(x,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecGetSize(x, &xsize);
//cout<<"size x is "<<xsize<<endl;
/*
Create and assemble parallel matrix
*/
ierr = MatCreateShell(PETSC_COMM_WORLD, num_elem_proc[rank], num_elem_proc[rank], total_elem,
total_elem, &myctx, &A);
CHKERRQ(ierr);
ierr = MatShellSetOperation(A, MATOP_MULT, (void (*)(void))MatLaplacian2D_Mult);CHKERRQ
(ierr);
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);
CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp, A, A/*,DIFFERENT_NONZERO_PATTERN*/);
CHKERRQ(ierr);
ierr = KSPSetType(ksp, KSPFGMRES);
CHKERRQ(ierr);
/*
Set default preconditioner for this program to be block Jacobi.
This choice can be overridden at runtime with the option
-pc_type <type>
*/
// ierr = KSPSetTolerances(ksp,1.e-7,PETSC_DEFAULT,1.e9,3000);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT, 50000);
CHKERRQ(ierr);
/* -------------------------------------------------------------------
Solve the linear system
------------------------------------------------------------------- */
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
/*
Solve the linear system
*/
ierr = KSPSolve(ksp, b, x);
CHKERRQ(ierr);
/* -------------------------------------------------------------------
Check solution and clean up
------------------------------------------------------------------- */
/*
Check the error
*/
//ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
if (rank == 0) {
ierr = KSPGetIterationNumber(ksp, &its);
CHKERRQ(ierr);
ierr = KSPGetResidualNorm(ksp, &norm);
CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf( MPI_COMM_SELF, "Norm of error %g iterations %D\n",
(double) norm, its);
CHKERRQ(ierr);
//PetscSynchronizedFlush(PETSC_COMM_WORLD);
ierr = KSPGetConvergedReason(ksp, &reason);
ierr = PetscSynchronizedPrintf( MPI_COMM_SELF, "kind of divergence is: ...........%D \n",
reason);
//PetscSynchronizedFlush(PETSC_COMM_WORLD);
}
/*
*/
//ierr = VecGetArray(x,&xx);CHKERRQ(ierr);
update_phi(El_Table, x, &myctx);
//ierr = VecRestoreArray(x, &xx);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
MPI_Barrier(PETSC_COMM_WORLD);
ierr = KSPDestroy(&ksp);
CHKERRQ(ierr); //ierr = PetscFree(phin);CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr); //ierr = PCDestroy(&pc);CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = VecScatterDestroy(&vscat);
CHKERRQ(ierr);
ierr = ISDestroy(&globalis);
CHKERRQ(ierr);
ierr = ISDestroy(&tois);
CHKERRQ(ierr);
PetscFree(num_elem_proc);
CHKERRQ(ierr);
PetscFree(to);
CHKERRQ(ierr);
PetscFree(from);
CHKERRQ(ierr);
return 0;
}
PetscErrorCode _BlockSolve( void* solver, void* _stokesSLE ) {
Stokes_SLE* stokesSLE = (Stokes_SLE*)_stokesSLE;
StokesBlockKSPInterface* Solver = (StokesBlockKSPInterface*)solver;
/* Create shortcuts to stuff needed on sle */
Mat K;
Mat G;
Mat Gt;
Mat D;
Mat C;
Mat approxS;
Vec u;
Vec p;
Vec f;
Vec h;
Mat stokes_P;
Mat stokes_A;
Vec stokes_x;
Vec stokes_b;
Mat a[2][2];
Vec x[2];
Vec b[2];
KSP stokes_ksp;
PC stokes_pc;
PetscTruth sym,flg;
PetscErrorCode ierr;
PetscInt N,n;
SBKSP_GetStokesOperators( stokesSLE, &K,&G,&D,&C, &approxS, &f,&h, &u,&p );
/* create Gt */
if( !D ) {
ierr = MatTranspose( G, MAT_INITIAL_MATRIX, &Gt);CHKERRQ(ierr);
sym = PETSC_TRUE;
Solver->DIsSym = sym;
}
else {
Gt = D;
sym = PETSC_FALSE;
Solver->DIsSym = sym;
}
flg=PETSC_FALSE;
PetscOptionsHasName(PETSC_NULL,"-use_petsc_ksp",&flg);
if (flg) {
if( !C ) {
/* Everything in this bracket, dependent on !C, is to build
a matrix with diagonals of 0 for C the previous comment ways
need a 'zero' matrix to keep fieldsplit happy in petsc? */
MatType mtype;
Vec V;
//MatGetSize( G, &M, &N );
VecGetSize(p, &N);
VecGetLocalSize( p, &n );
MatCreate( PetscObjectComm((PetscObject) K), &C );
MatSetSizes( C, PETSC_DECIDE ,PETSC_DECIDE, N, N );
#if (((PETSC_VERSION_MAJOR==3) && (PETSC_VERSION_MINOR>=3)) || (PETSC_VERSION_MAJOR>3) )
MatSetUp(C);
#endif
MatGetType( G, &mtype );
MatSetType( C, mtype );
MatGetVecs( G, &V, PETSC_NULL );
VecSet(V, 0.0);
//VecSet(h, 1.0);
ierr = VecAssemblyBegin( V );CHKERRQ(ierr);
ierr = VecAssemblyEnd ( V );CHKERRQ(ierr);
ierr = MatDiagonalSet(C,V,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin( C, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
ierr = MatAssemblyEnd ( C, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
}
}
a[0][0]=K; a[0][1]=G;
a[1][0]=Gt; a[1][1]=C;
ierr = MatCreateNest(PetscObjectComm((PetscObject) K), 2, NULL, 2, NULL, (Mat *)a, &stokes_A);CHKERRQ(ierr);
ierr = MatAssemblyBegin( stokes_A, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
ierr = MatAssemblyEnd( stokes_A, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
x[0]=u;
x[1]=p;
ierr = VecCreateNest(PetscObjectComm((PetscObject) u), 2, NULL, x, &stokes_x);CHKERRQ(ierr);
ierr = VecAssemblyBegin( stokes_x );CHKERRQ(ierr);
ierr = VecAssemblyEnd( stokes_x);CHKERRQ(ierr);
b[0]=f;
b[1]=h;
ierr = VecCreateNest(PetscObjectComm((PetscObject) f), 2, NULL, b, &stokes_b);CHKERRQ(ierr);
ierr = VecAssemblyBegin( stokes_b );CHKERRQ(ierr);
ierr = VecAssemblyEnd( stokes_b);CHKERRQ(ierr);
/* if( approxS ) { */
/* a[0][0]=K; a[0][1]=G; */
/* a[1][0]=NULL; a[1][1]=approxS; */
/* ierr = MatCreateNest(PetscObjectComm((PetscObject) K), 2, NULL, 2, NULL, (Mat *)a, &stokes_P);CHKERRQ(ierr); */
/* ierr = MatAssemblyBegin( stokes_P, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr); */
/* ierr = MatAssemblyEnd( stokes_P, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr); */
/* } */
/* else { */
stokes_P = stokes_A;
/* } */
/* probably should make a Destroy function for these two */
/* Update options from file and/or string here so we can change things on the fly */
//PetscOptionsInsertFile(PETSC_COMM_WORLD, Solver->optionsFile, PETSC_FALSE);
//PetscOptionsInsertString(Solver->optionsString);
ierr = KSPCreate( PETSC_COMM_WORLD, &stokes_ksp );CHKERRQ(ierr);
Stg_KSPSetOperators( stokes_ksp, stokes_A, stokes_P, SAME_NONZERO_PATTERN );
ierr = KSPSetType( stokes_ksp, "bsscr" );/* i.e. making this the default solver : calls KSPCreate_XXX */CHKERRQ(ierr);
ierr = KSPGetPC( stokes_ksp, &stokes_pc );CHKERRQ(ierr);
ierr = PCSetType( stokes_pc, PCNONE );CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero( stokes_ksp, PETSC_TRUE );CHKERRQ(ierr);
ierr = KSPSetFromOptions( stokes_ksp );CHKERRQ(ierr);
/*
Doing this so the KSP Solver has access to the StgFEM Multigrid struct (PETScMGSolver).
As well as any custom stuff on the Stokes_SLE struct
*/
if( stokes_ksp->data ){/* then ksp->data has been created in a KSpSetUp_XXX function */
/* testing for our KSP types that need the data that is on Solver... */
/* for the moment then, this function not completely agnostic about our KSPs */
//if(!strcmp("bsscr",stokes_ksp->type_name)){/* if is bsscr then set up the data on the ksp */
flg=PETSC_FALSE;
PetscOptionsHasName(PETSC_NULL,"-use_petsc_ksp",&flg);
if (!flg) {
((KSP_COMMON*)(stokes_ksp->data))->st_sle = Solver->st_sle;
((KSP_COMMON*)(stokes_ksp->data))->mg = Solver->mg;
((KSP_COMMON*)(stokes_ksp->data))->DIsSym = Solver->DIsSym;
((KSP_COMMON*)(stokes_ksp->data))->preconditioner = Solver->preconditioner;
((KSP_COMMON*)(stokes_ksp->data))->solver = Solver;
}
}
ierr = KSPSolve( stokes_ksp, stokes_b, stokes_x );CHKERRQ(ierr);
Stg_KSPDestroy(&stokes_ksp );
//if( ((StokesBlockKSPInterface*)stokesSLE->solver)->preconditioner )
if(stokes_P != stokes_A) { Stg_MatDestroy(&stokes_P ); }
Stg_MatDestroy(&stokes_A );
Stg_VecDestroy(&stokes_x);
Stg_VecDestroy(&stokes_b);
if(!D){ Stg_MatDestroy(&Gt); }
if(C && (stokesSLE->cStiffMat->matrix != C) ){ Stg_MatDestroy(&C); }
PetscFunctionReturn(0);
}
int
main(int argc, char** argv) {
int levels = 4;
int mg_levels = 3;
PetscInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
construct_operator(&problem, levels);
KSP ksp;
PC pc;
KSPCreate(PETSC_COMM_WORLD, &ksp);
KSPSetOperators(ksp, problem.A, problem.A, SAME_PRECONDITIONER);
KSPSetType(ksp, KSPRICHARDSON);
if (1) {
KSPGetPC(ksp, &pc);
PCSetType(pc, PCMG);
PCMGSetLevels(pc, mg_levels, NULL);
PCMGSetGalerkin(pc);
PCMGSetType(pc, PC_MG_MULTIPLICATIVE);
PCMGSetCycleType(pc, PC_MG_CYCLE_V);
int ii;
for (ii=0; ii<mg_levels; ii++) {
if (ii == 0) {
KSP smooth_ksp;
PCMGGetSmoother(pc, ii, &smooth_ksp);
KSPSetType(smooth_ksp, KSPPREONLY);
PC smooth_pc;
KSPGetPC(smooth_ksp, &smooth_pc);
PCSetType(smooth_pc, PCLU);
} else {
// set up the smoother.
KSP smooth_ksp;
PC smooth_pc;
PCMGGetSmoother(pc, ii, &smooth_ksp);
KSPSetType(smooth_ksp, KSPRICHARDSON);
KSPRichardsonSetScale(smooth_ksp, 2./3.);
KSPGetPC(smooth_ksp, &smooth_pc);
PCSetType(smooth_pc, PCJACOBI);
KSPSetTolerances(smooth_ksp, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT, 2);
//set up the interpolation operator
Mat prolongation;
construct_prolongation_operator(ii+1+levels-mg_levels, &prolongation);
PCMGSetInterpolation(pc, ii, prolongation);
MatScale(prolongation, 1./2.);
Mat restriction;
MatTranspose(prolongation, &restriction);
PCMGSetRestriction(pc, ii, prolongation);
MatDestroy(prolongation);
MatDestroy(restriction);
}
}
} else {
KSPGetPC(ksp, &pc);
PCSetType(pc, PCJACOBI);
}
//*/
/*
if (0) {
KSPSetType(ksp, KSPRICHARDSON);
KSPRichardsonSetScale(ksp, 2./3.);
KSPGetPC(ksp, &pc);
PCSetType(pc, PCJACOBI);
} else {
PetscOptionsInsertString("-ksp_type richardson");
PetscOptionsInsertString("-ksp_richardson_scale 0.666666666666666666");
PetscOptionsInsertString("-pc_type jacobi");
}
//*/
KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
KSPSetFromOptions(ksp);
KSPSetUp(ksp);
//VecView(problem.x, PETSC_VIEWER_STDOUT_WORLD);
{
//CHKERR(PCApply(pc, problem.b, problem.x));
CHKERR(KSPSolve(ksp, problem.b, problem.x));
KSPConvergedReason reason;
CHKERR(KSPGetConvergedReason(ksp, &reason));
printf("KSPConvergedReason: %d\n", reason);
PetscInt its;
CHKERR(KSPGetIterationNumber(ksp, &its));
printf("Num iterations: %d\n", its);
}
//compute_residual_norm(&problem);
VecView(problem.x, PETSC_VIEWER_STDOUT_WORLD);
PetscFinalize();
return 0;
}
PetscErrorCode test_solve(void)
{
Mat A11, A12,A21,A22, A, tmp[2][2];
KSP ksp;
PC pc;
Vec b,x, f,h, diag, x1,x2;
Vec tmp_x[2],*_tmp_x;
int n, np, i,j;
PetscErrorCode ierr;
PetscFunctionBeginUser;
PetscPrintf(PETSC_COMM_WORLD, "%s \n", PETSC_FUNCTION_NAME);
n = 3;
np = 2;
/* Create matrices */
/* A11 */
ierr = VecCreate(PETSC_COMM_WORLD, &diag);
CHKERRQ(ierr);
ierr = VecSetSizes(diag, PETSC_DECIDE, n);
CHKERRQ(ierr);
ierr = VecSetFromOptions(diag);
CHKERRQ(ierr);
ierr = VecSet(diag, (1.0/10.0));
CHKERRQ(ierr); /* so inverse = diag(10) */
/* As a test, create a diagonal matrix for A11 */
ierr = MatCreate(PETSC_COMM_WORLD, &A11);
CHKERRQ(ierr);
ierr = MatSetSizes(A11, PETSC_DECIDE, PETSC_DECIDE, n, n);
CHKERRQ(ierr);
ierr = MatSetType(A11, MATAIJ);
CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A11, n, NULL);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A11, np, NULL,np, NULL);
CHKERRQ(ierr);
ierr = MatDiagonalSet(A11, diag, INSERT_VALUES);
CHKERRQ(ierr);
ierr = VecDestroy(&diag);
CHKERRQ(ierr);
/* A12 */
ierr = MatCreate(PETSC_COMM_WORLD, &A12);
CHKERRQ(ierr);
ierr = MatSetSizes(A12, PETSC_DECIDE, PETSC_DECIDE, n, np);
CHKERRQ(ierr);
ierr = MatSetType(A12, MATAIJ);
CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A12, np, NULL);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A12, np, NULL,np, NULL);
CHKERRQ(ierr);
for (i=0; i<n; i++) {
for (j=0; j<np; j++) {
ierr = MatSetValue(A12, i,j, (PetscScalar)(i+j*n), INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatSetValue(A12, 2,1, (PetscScalar)(4), INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(A12, MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(A12, MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* A21 */
ierr = MatTranspose(A12, MAT_INITIAL_MATRIX, &A21);
CHKERRQ(ierr);
A22 = NULL;
/* Create block matrix */
tmp[0][0] = A11;
tmp[0][1] = A12;
tmp[1][0] = A21;
tmp[1][1] = A22;
ierr = MatCreateNest(PETSC_COMM_WORLD,2,NULL,2,NULL,&tmp[0][0],&A);
CHKERRQ(ierr);
ierr = MatNestSetVecType(A,VECNEST);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* Create vectors */
ierr = MatCreateVecs(A12, &h, &f);
CHKERRQ(ierr);
ierr = VecSet(f, 1.0);
CHKERRQ(ierr);
ierr = VecSet(h, 0.0);
CHKERRQ(ierr);
/* Create block vector */
tmp_x[0] = f;
tmp_x[1] = h;
ierr = VecCreateNest(PETSC_COMM_WORLD,2,NULL,tmp_x,&b);
CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);
CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);
CHKERRQ(ierr);
ierr = VecDuplicate(b, &x);
CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetType(ksp, "gmres");
CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);
CHKERRQ(ierr);
ierr = PCSetType(pc, "none");
CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);
CHKERRQ(ierr);
ierr = KSPSolve(ksp, b, x);
CHKERRQ(ierr);
ierr = VecNestGetSubVecs(x,NULL,&_tmp_x);
CHKERRQ(ierr);
x1 = _tmp_x[0];
x2 = _tmp_x[1];
PetscPrintf(PETSC_COMM_WORLD, "x1 \n");
PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_INFO_DETAIL);
ierr = VecView(x1, PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD, "x2 \n");
PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_INFO_DETAIL);
ierr = VecView(x2, PETSC_VIEWER_STDOUT_WORLD);
CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
ierr = MatDestroy(&A11);
CHKERRQ(ierr);
ierr = MatDestroy(&A12);
CHKERRQ(ierr);
ierr = MatDestroy(&A21);
CHKERRQ(ierr);
ierr = VecDestroy(&f);
CHKERRQ(ierr);
ierr = VecDestroy(&h);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscSolver::PetscSolver(FlowField & flowField, Parameters & parameters):
LinearSolver(flowField, parameters), _ctx(parameters, flowField){
// Set the type of boundary nodes of the system
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,
by = DMDA_BOUNDARY_NONE,
bz = DMDA_BOUNDARY_NONE;
if (parameters.walls.typeLeft==PERIODIC){
bx = DMDA_BOUNDARY_PERIODIC;
}
if (parameters.walls.typeBottom==PERIODIC){
by = DMDA_BOUNDARY_PERIODIC;
}
if (parameters.walls.typeFront==PERIODIC){
bz = DMDA_BOUNDARY_PERIODIC;
}
KSPCreate(PETSC_COMM_WORLD,&_ksp);
PetscErrorCode (*computeMatrix)(KSP, Mat, Mat, MatStructure*, void*) = NULL;
if (_parameters.geometry.dim == 2){
computeMatrix = computeMatrix2D;
DMDACreate2d(PETSC_COMM_WORLD, bx, by, DMDA_STENCIL_STAR,
parameters.geometry.sizeX+2, parameters.geometry.sizeY+2,
parameters.parallel.numProcessors[0],
parameters.parallel.numProcessors[1],
1, 2,
_parameters.parallel.sizes[0],
_parameters.parallel.sizes[1],
&_da);
} else if (_parameters.geometry.dim == 3){
computeMatrix = computeMatrix3D;
DMDACreate3d(PETSC_COMM_WORLD, bx, by, bz, DMDA_STENCIL_STAR,
parameters.geometry.sizeX + 2, parameters.geometry.sizeY + 2,
parameters.geometry.sizeZ + 2,
parameters.parallel.numProcessors[0],
parameters.parallel.numProcessors[1],
parameters.parallel.numProcessors[2],
1, 2, // Degrees of freedom and stencil length
_parameters.parallel.sizes[0],
_parameters.parallel.sizes[1],
_parameters.parallel.sizes[2],
&_da);
}
// Find out what are the corners of the subdomain
DMDAGetCorners(_da, &_firstX, &_firstY, &_firstZ, &_lengthX, &_lengthY, &_lengthZ);
// Current function to assign the limits
createLimits(parameters, _da, _limitsX, _limitsY, _limitsZ);
// Set the rank in the context. Necessary since PETSc declares the rank of the neighbors under
// periodic conditions as the rank of the process itself. So the rank must be known to properly
// set matrices and RHS vectors
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
// Set offsets to fix where the results of the pressure will be written in the flow field
if (_firstX == 0){
_offsetX = 1;
} else {
_offsetX = 2;
}
if (_firstY == 0){
_offsetY = 1;
} else {
_offsetY = 2;
}
if (_firstZ == 0){
_offsetZ = 1;
} else {
_offsetZ = 2;
}
// Set a pointer to the limits in the context, so that they can be used by the function
_ctx.setLimits(_limitsX, _limitsY, _limitsZ);
// Determine whether a process writes a boundary on the system.
// Right now, it only depends on the position of the array. The identity of the neighbors will
// become significant only in the iterators, where it will apply a boundary condition only if
// the rank is invalid, so that there is no neighbor
_ctx.setAsBoundary = 0;
if (parameters.parallel.indices[0] == 0){
_ctx.setAsBoundary += LEFT_WALL_BIT;
}
if (parameters.parallel.indices[0] == parameters.parallel.numProcessors[0]-1){
_ctx.setAsBoundary += RIGHT_WALL_BIT;
}
if (parameters.parallel.indices[1] == 0){
_ctx.setAsBoundary += BOTTOM_WALL_BIT;
}
if (parameters.parallel.indices[1] == parameters.parallel.numProcessors[1]-1){
_ctx.setAsBoundary += TOP_WALL_BIT;
}
if (parameters.parallel.indices[2] == 0){
_ctx.setAsBoundary += FRONT_WALL_BIT;
}
if (parameters.parallel.indices[2] == parameters.parallel.numProcessors[2]-1){
_ctx.setAsBoundary += BACK_WALL_BIT;
}
// Set displacements to deal with periodic boundaries if necessary.
// If the boundary is periodic, it will take information from positions beyond the ghost cells,
// since they are used only for parallel communication. Otherwise, PETSc deals with
// communication of the pressure.
int Nx = parameters.geometry.sizeX + 2;
int Ny = parameters.geometry.sizeY + 2;
int Nz = parameters.geometry.sizeZ + 2;
if (parameters.walls.typeLeft==PERIODIC){
_ctx.displacement[0] = -2;
_ctx.displacement[1] = Nx+1;
} else {
_ctx.displacement[0] = 1;
_ctx.displacement[1] = Nx-2;
}
if (parameters.walls.typeBottom==PERIODIC){
_ctx.displacement[2] = -2;
_ctx.displacement[3] = Ny+1;
} else {
_ctx.displacement[2] = 1;
_ctx.displacement[3] = Ny-2;
}
if (parameters.walls.typeFront==PERIODIC){
_ctx.displacement[4] = -2;
_ctx.displacement[5] = Nz+1;
} else {
_ctx.displacement[4] = 1;
_ctx.displacement[5] = Nz-2;
}
DMCreateGlobalVector(_da, &_x);
KSPSetDM(_ksp, _da);
KSPSetComputeOperators(_ksp, computeMatrix, &_ctx);
KSPSetType(_ksp,KSPFGMRES);
KSPSetFromOptions(_ksp);
KSPSetInitialGuessNonzero(_ksp,PETSC_TRUE);
}
PetscErrorCode PCISSetUp(PC pc)
{
PC_IS *pcis = (PC_IS*)(pc->data);
Mat_IS *matis;
PetscErrorCode ierr;
PetscBool flg;
Vec counter;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)pc->mat,MATIS,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG,"Preconditioner type of Neumann Neumman requires matrix of type MATIS");
matis = (Mat_IS*)pc->mat->data;
pcis->pure_neumann = matis->pure_neumann;
/* get info on mapping */
ierr = ISLocalToGlobalMappingGetSize(matis->mapping,&pcis->n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetInfo(matis->mapping,&(pcis->n_neigh),&(pcis->neigh),&(pcis->n_shared),&(pcis->shared));CHKERRQ(ierr);
pcis->ISLocalToGlobalMappingGetInfoWasCalled = PETSC_TRUE;
/* Creating local and global index sets for interior and inteface nodes. */
{
PetscInt n_I;
PetscInt *idx_I_local,*idx_B_local,*idx_I_global,*idx_B_global;
PetscInt *array;
PetscInt i,j;
/* Identifying interior and interface nodes, in local numbering */
ierr = PetscMalloc(pcis->n*sizeof(PetscInt),&array);CHKERRQ(ierr);
ierr = PetscMemzero(array,pcis->n*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0;i<pcis->n_neigh;i++)
for (j=0;j<pcis->n_shared[i];j++)
array[pcis->shared[i][j]] += 1;
ierr = PetscMalloc(pcis->n*sizeof(PetscInt),&idx_I_local);CHKERRQ(ierr);
ierr = PetscMalloc(pcis->n*sizeof(PetscInt),&idx_B_local);CHKERRQ(ierr);
for (i=0, pcis->n_B=0, n_I=0; i<pcis->n; i++) {
if (!array[i]) {
idx_I_local[n_I] = i;
n_I++;
} else {
idx_B_local[pcis->n_B] = i;
pcis->n_B++;
}
}
/* Getting the global numbering */
idx_B_global = idx_I_local + n_I; /* Just avoiding allocating extra memory, since we have vacant space */
idx_I_global = idx_B_local + pcis->n_B;
ierr = ISLocalToGlobalMappingApply(matis->mapping,pcis->n_B,idx_B_local,idx_B_global);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(matis->mapping,n_I, idx_I_local,idx_I_global);CHKERRQ(ierr);
/* Creating the index sets. */
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_local,PETSC_COPY_VALUES, &pcis->is_B_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_global,PETSC_COPY_VALUES,&pcis->is_B_global);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_local,PETSC_COPY_VALUES, &pcis->is_I_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_global,PETSC_COPY_VALUES,&pcis->is_I_global);CHKERRQ(ierr);
/* Freeing memory and restoring arrays */
ierr = PetscFree(idx_B_local);CHKERRQ(ierr);
ierr = PetscFree(idx_I_local);CHKERRQ(ierr);
ierr = PetscFree(array);CHKERRQ(ierr);
}
/*
Extracting the blocks A_II, A_BI, A_IB and A_BB from A. If the numbering
is such that interior nodes come first than the interface ones, we have
[ | ]
[ A_II | A_IB ]
A = [ | ]
[-----------+------]
[ A_BI | A_BB ]
*/
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_II);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_IB);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_BI);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_BB);CHKERRQ(ierr);
/*
Creating work vectors and arrays
*/
ierr = VecDuplicate(matis->x,&pcis->vec1_N);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_N,&pcis->vec2_N);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n-pcis->n_B,&pcis->vec1_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec2_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec3_D);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n_B,&pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec2_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec3_B);CHKERRQ(ierr);
ierr = MatGetVecs(pc->pmat,&pcis->vec1_global,0);CHKERRQ(ierr);
ierr = PetscMalloc((pcis->n)*sizeof(PetscScalar),&pcis->work_N);CHKERRQ(ierr);
/* Creating the scatter contexts */
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_I_global,pcis->vec1_D,(IS)0,&pcis->global_to_D);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,pcis->is_B_local,pcis->vec1_B,(IS)0,&pcis->N_to_B);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_B_global,pcis->vec1_B,(IS)0,&pcis->global_to_B);CHKERRQ(ierr);
/* Creating scaling "matrix" D */
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_is_use_stiffness_scaling",&pcis->use_stiffness_scaling,NULL);CHKERRQ(ierr);
if (!pcis->D) {
ierr = VecDuplicate(pcis->vec1_B,&pcis->D);CHKERRQ(ierr);
if (!pcis->use_stiffness_scaling) {
ierr = VecSet(pcis->D,pcis->scaling_factor);CHKERRQ(ierr);
} else {
ierr = MatGetDiagonal(matis->A,pcis->vec1_N);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
}
ierr = VecCopy(pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = MatGetVecs(pc->pmat,&counter,0);CHKERRQ(ierr); /* temporary auxiliar vector */
ierr = VecSet(counter,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPointwiseDivide(pcis->D,pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDestroy(&counter);CHKERRQ(ierr);
/* See historical note 01, at the bottom of this file. */
/*
Creating the KSP contexts for the local Dirichlet and Neumann problems.
*/
if (pcis->computesolvers) {
PC pc_ctx;
/* Dirichlet */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_D);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_D,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_D,pcis->A_II,pcis->A_II,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_D,"is_localD_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_D,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_D,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_D);CHKERRQ(ierr);
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_D);CHKERRQ(ierr);
/* Neumann */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_N);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_N,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_N,matis->A,matis->A,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_N,"is_localN_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_N,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_N,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_N);CHKERRQ(ierr);
{
PetscBool damp_fixed = PETSC_FALSE,
remove_nullspace_fixed = PETSC_FALSE,
set_damping_factor_floating = PETSC_FALSE,
not_damp_floating = PETSC_FALSE,
not_remove_nullspace_floating = PETSC_FALSE;
PetscReal fixed_factor,
floating_factor;
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&fixed_factor,&damp_fixed);CHKERRQ(ierr);
if (!damp_fixed) fixed_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&damp_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_remove_nullspace_fixed",&remove_nullspace_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",
&floating_factor,&set_damping_factor_floating);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",&set_damping_factor_floating,NULL);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 1.e-12;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_damp_floating",¬_damp_floating,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_remove_nullspace_floating",¬_remove_nullspace_floating,NULL);CHKERRQ(ierr);
if (pcis->pure_neumann) { /* floating subdomain */
if (!(not_damp_floating)) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (!(not_remove_nullspace_floating)) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
} else { /* fixed subdomain */
if (damp_fixed) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (remove_nullspace_fixed) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
}
}
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_N);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_Redundant(PC pc)
{
PC_Redundant *red = (PC_Redundant*)pc->data;
PetscErrorCode ierr;
PetscInt mstart,mend,mlocal,M;
PetscMPIInt size;
MPI_Comm comm,subcomm;
Vec x;
const char *prefix;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
/* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size == 1) red->useparallelmat = PETSC_FALSE;
if (!pc->setupcalled) {
PetscInt mloc_sub;
if (!red->psubcomm) {
ierr = PetscSubcommCreate(comm,&red->psubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetNumber(red->psubcomm,red->nsubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetType(red->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr);
/* enable runtime switch of psubcomm type, e.g., '-psubcomm_type interlaced */
ierr = PetscSubcommSetFromOptions(red->psubcomm);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));CHKERRQ(ierr);
/* create a new PC that processors in each subcomm have copy of */
subcomm = red->psubcomm->comm;
ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)red->ksp);CHKERRQ(ierr);
ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr);
ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr);
} else {
subcomm = red->psubcomm->comm;
}
if (red->useparallelmat) {
/* grab the parallel matrix and put it into processors of a subcomminicator */
ierr = MatGetRedundantMatrix(pc->pmat,red->psubcomm->n,subcomm,MAT_INITIAL_MATRIX,&red->pmats);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
/* get working vectors xsub and ysub */
ierr = MatGetVecs(red->pmats,&red->xsub,&red->ysub);CHKERRQ(ierr);
/* create working vectors xdup and ydup.
xdup concatenates all xsub's contigously to form a mpi vector over dupcomm (see PetscSubcommCreate_interlaced())
ydup concatenates all ysub and has empty local arrays because ysub's arrays will be place into it.
Note: we use communicator dupcomm, not PetscObjectComm((PetscObject)pc)! */
ierr = MatGetLocalSize(red->pmats,&mloc_sub,NULL);CHKERRQ(ierr);
ierr = VecCreateMPI(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,&red->xdup);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(red->psubcomm->dupparent,1,mloc_sub,PETSC_DECIDE,NULL,&red->ydup);CHKERRQ(ierr);
/* create vecscatters */
if (!red->scatterin) { /* efficiency of scatterin is independent from psubcomm_type! */
IS is1,is2;
PetscInt *idx1,*idx2,i,j,k;
ierr = MatGetVecs(pc->pmat,&x,0);CHKERRQ(ierr);
ierr = VecGetSize(x,&M);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&mstart,&mend);CHKERRQ(ierr);
mlocal = mend - mstart;
ierr = PetscMalloc2(red->psubcomm->n*mlocal,&idx1,red->psubcomm->n*mlocal,&idx2);CHKERRQ(ierr);
j = 0;
for (k=0; k<red->psubcomm->n; k++) {
for (i=mstart; i<mend; i++) {
idx1[j] = i;
idx2[j++] = i + M*k;
}
}
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,red->xdup,is2,&red->scatterin);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
/* Impl below is good for PETSC_SUBCOMM_INTERLACED (no inter-process communication) and PETSC_SUBCOMM_CONTIGUOUS (communication within subcomm) */
ierr = ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*M,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart,1,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(red->xdup,is1,x,is2,&red->scatterout);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFree2(idx1,idx2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
} else { /* pc->setupcalled */
if (red->useparallelmat) {
MatReuse reuse;
/* grab the parallel matrix and put it into processors of a subcomminicator */
/*--------------------------------------------------------------------------*/
if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
/* destroy old matrices */
ierr = MatDestroy(&red->pmats);CHKERRQ(ierr);
reuse = MAT_INITIAL_MATRIX;
} else {
reuse = MAT_REUSE_MATRIX;
}
ierr = MatGetRedundantMatrix(pc->pmat,red->psubcomm->n,red->psubcomm->comm,reuse,&red->pmats);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
}
if (pc->setfromoptionscalled) {
ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
}
ierr = KSPSetUp(red->ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode SNESSetUp_NASM(SNES snes)
{
SNES_NASM *nasm = (SNES_NASM*)snes->data;
PetscErrorCode ierr;
DM dm,subdm;
DM *subdms;
PetscInt i;
const char *optionsprefix;
Vec F;
PetscMPIInt size;
KSP ksp;
PC pc;
PetscFunctionBegin;
if (!nasm->subsnes) {
ierr = SNESGetDM(snes,&dm);CHKERRQ(ierr);
if (dm) {
nasm->usesdm = PETSC_TRUE;
ierr = DMCreateDomainDecomposition(dm,&nasm->n,NULL,NULL,NULL,&subdms);CHKERRQ(ierr);
if (!subdms) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_ARG_WRONGSTATE,"DM has no default decomposition defined. Set subsolves manually with SNESNASMSetSubdomains().");
ierr = DMCreateDomainDecompositionScatters(dm,nasm->n,subdms,&nasm->iscatter,&nasm->oscatter,&nasm->gscatter);CHKERRQ(ierr);
ierr = SNESGetOptionsPrefix(snes, &optionsprefix);CHKERRQ(ierr);
ierr = PetscMalloc1(nasm->n,&nasm->subsnes);CHKERRQ(ierr);
for (i=0; i<nasm->n; i++) {
ierr = SNESCreate(PETSC_COMM_SELF,&nasm->subsnes[i]);CHKERRQ(ierr);
ierr = SNESAppendOptionsPrefix(nasm->subsnes[i],optionsprefix);CHKERRQ(ierr);
ierr = SNESAppendOptionsPrefix(nasm->subsnes[i],"sub_");CHKERRQ(ierr);
ierr = SNESSetDM(nasm->subsnes[i],subdms[i]);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)nasm->subsnes[i]),&size);CHKERRQ(ierr);
if (size == 1) {
ierr = SNESGetKSP(nasm->subsnes[i],&ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = PCSetType(pc,PCLU);CHKERRQ(ierr);
}
ierr = SNESSetFromOptions(nasm->subsnes[i]);CHKERRQ(ierr);
ierr = DMDestroy(&subdms[i]);CHKERRQ(ierr);
}
ierr = PetscFree(subdms);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)snes),PETSC_ERR_ARG_WRONGSTATE,"Cannot construct local problems automatically without a DM!");
} else SETERRQ(PetscObjectComm((PetscObject)snes),PETSC_ERR_ARG_WRONGSTATE,"Must set subproblems manually if there is no DM!");
/* allocate the global vectors */
if (!nasm->x) {
ierr = PetscCalloc1(nasm->n,&nasm->x);CHKERRQ(ierr);
}
if (!nasm->xl) {
ierr = PetscCalloc1(nasm->n,&nasm->xl);CHKERRQ(ierr);
}
if (!nasm->y) {
ierr = PetscCalloc1(nasm->n,&nasm->y);CHKERRQ(ierr);
}
if (!nasm->b) {
ierr = PetscCalloc1(nasm->n,&nasm->b);CHKERRQ(ierr);
}
for (i=0; i<nasm->n; i++) {
ierr = SNESGetFunction(nasm->subsnes[i],&F,NULL,NULL);CHKERRQ(ierr);
if (!nasm->x[i]) {ierr = VecDuplicate(F,&nasm->x[i]);CHKERRQ(ierr);}
if (!nasm->y[i]) {ierr = VecDuplicate(F,&nasm->y[i]);CHKERRQ(ierr);}
if (!nasm->b[i]) {ierr = VecDuplicate(F,&nasm->b[i]);CHKERRQ(ierr);}
if (!nasm->xl[i]) {
ierr = SNESGetDM(nasm->subsnes[i],&subdm);CHKERRQ(ierr);
ierr = DMCreateLocalVector(subdm,&nasm->xl[i]);CHKERRQ(ierr);
}
ierr = DMGlobalToLocalHookAdd(subdm,DMGlobalToLocalSubDomainDirichletHook_Private,NULL,nasm->xl[i]);CHKERRQ(ierr);
}
if (nasm->finaljacobian) {
ierr = SNESSetUpMatrices(snes);CHKERRQ(ierr);
if (nasm->fjtype == 2) {
ierr = VecDuplicate(snes->vec_sol,&nasm->xinit);CHKERRQ(ierr);
}
for (i=0; i<nasm->n;i++) {
ierr = SNESSetUpMatrices(nasm->subsnes[i]);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat mat; /* matrix */
Vec b,ustar,u; /* vectors (RHS, exact solution, approx solution) */
PC pc; /* PC context */
KSP ksp; /* KSP context */
PetscErrorCode ierr;
PetscInt n = 10,i,its,col[3];
PetscScalar value[3];
PCType pcname;
KSPType kspname;
PetscReal norm,tol=1000.*PETSC_MACHINE_EPSILON;
PetscInitialize(&argc,&args,(char*)0,help);
/* Create and initialize vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&b);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&ustar);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&u);CHKERRQ(ierr);
ierr = VecSet(ustar,1.0);CHKERRQ(ierr);
ierr = VecSet(u,0.0);CHKERRQ(ierr);
/* Create and assemble matrix */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,n,n,3,NULL,&mat);CHKERRQ(ierr);
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(mat,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0] = n - 2; col[1] = n - 1;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0; col[0] = 0; col[1] = 1; value[0] = 2.0; value[1] = -1.0;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Compute right-hand-side vector */
ierr = MatMult(mat,ustar,b);CHKERRQ(ierr);
/* Create PC context and set up data structures */
ierr = PCCreate(PETSC_COMM_WORLD,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
/* Create KSP context and set up data structures */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPRICHARDSON);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat);CHKERRQ(ierr);
ierr = KSPSetPC(ksp,pc);CHKERRQ(ierr);
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
/* Solve the problem */
ierr = KSPGetType(ksp,&kspname);CHKERRQ(ierr);
ierr = PCGetType(pc,&pcname);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Running %s with %s preconditioning\n",kspname,pcname);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,u);CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,ustar);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_2,&norm);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"2 norm of error %g Number of iterations %D\n",(double)norm,its);CHKERRQ(ierr);
}
/* Free data structures */
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&ustar);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PCDestroy(&pc);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PCBDDCNullSpaceAssembleCorrection(PC pc,IS local_dofs)
{
PC_BDDC *pcbddc = (PC_BDDC*)pc->data;
PC_IS *pcis = (PC_IS*)pc->data;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
KSP *local_ksp;
PC newpc;
NullSpaceCorrection_ctx shell_ctx;
Mat local_mat,local_pmat,small_mat,inv_small_mat;
MatStructure local_mat_struct;
Vec work1,work2;
const Vec *nullvecs;
VecScatter scatter_ctx;
IS is_aux;
MatFactorInfo matinfo;
PetscScalar *basis_mat,*Kbasis_mat,*array,*array_mat;
PetscScalar one = 1.0,zero = 0.0, m_one = -1.0;
PetscInt basis_dofs,basis_size,nnsp_size,i,k,n_I,n_R;
PetscBool nnsp_has_cnst;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Infer the local solver */
ierr = ISGetSize(local_dofs,&basis_dofs);CHKERRQ(ierr);
ierr = VecGetSize(pcis->vec1_D,&n_I);CHKERRQ(ierr);
ierr = VecGetSize(pcbddc->vec1_R,&n_R);CHKERRQ(ierr);
if (basis_dofs == n_I) {
/* Dirichlet solver */
local_ksp = &pcbddc->ksp_D;
} else if (basis_dofs == n_R) {
/* Neumann solver */
local_ksp = &pcbddc->ksp_R;
} else {
SETERRQ4(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Error in %s: unknown local IS size %d. n_I=%d, n_R=%d)\n",__FUNCT__,basis_dofs,n_I,n_R);
}
ierr = KSPGetOperators(*local_ksp,&local_mat,&local_pmat,&local_mat_struct);CHKERRQ(ierr);
/* Get null space vecs */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nnsp_has_cnst,&nnsp_size,&nullvecs);CHKERRQ(ierr);
basis_size = nnsp_size;
if (nnsp_has_cnst) {
basis_size++;
}
/* Create shell ctx */
ierr = PetscMalloc(sizeof(*shell_ctx),&shell_ctx);CHKERRQ(ierr);
/* Create work vectors in shell context */
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_small_1,basis_size,basis_size);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_small_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_small_1,&shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_full_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_full_1,basis_dofs,basis_dofs);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_full_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&shell_ctx->work_full_2);CHKERRQ(ierr);
/* Allocate workspace */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->basis_mat );CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->Kbasis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Restrict local null space on selected dofs (Dirichlet or Neumann)
and compute matrices N and K*N */
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,local_dofs,work1,(IS)0,&scatter_ctx);CHKERRQ(ierr);
for (k=0;k<nnsp_size;k++) {
ierr = VecScatterBegin(matis->ctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->ctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
if (nnsp_has_cnst) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecSet(work1,one);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter_ctx);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Assemble another Mat object in shell context */
ierr = MatTransposeMatMult(shell_ctx->basis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&small_mat);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&matinfo);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,basis_size,0,1,&is_aux);CHKERRQ(ierr);
ierr = MatLUFactor(small_mat,is_aux,is_aux,&matinfo);CHKERRQ(ierr);
ierr = ISDestroy(&is_aux);CHKERRQ(ierr);
ierr = PetscMalloc(basis_size*basis_size*sizeof(PetscScalar),&array_mat);CHKERRQ(ierr);
for (k=0;k<basis_size;k++) {
ierr = VecSet(shell_ctx->work_small_1,zero);CHKERRQ(ierr);
ierr = VecSetValue(shell_ctx->work_small_1,k,one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecAssemblyEnd(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = MatSolve(small_mat,shell_ctx->work_small_1,shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecGetArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
for (i=0;i<basis_size;i++) {
array_mat[i*basis_size+k]=array[i];
}
ierr = VecRestoreArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
}
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_size,basis_size,array_mat,&inv_small_mat);CHKERRQ(ierr);
ierr = MatMatMult(shell_ctx->basis_mat,inv_small_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&shell_ctx->Lbasis_mat);CHKERRQ(ierr);
ierr = PetscFree(array_mat);CHKERRQ(ierr);
ierr = MatDestroy(&inv_small_mat);CHKERRQ(ierr);
ierr = MatDestroy(&small_mat);CHKERRQ(ierr);
ierr = MatScale(shell_ctx->Kbasis_mat,m_one);CHKERRQ(ierr);
/* Rebuild local PC */
ierr = KSPGetPC(*local_ksp,&shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PCCreate(PETSC_COMM_SELF,&newpc);CHKERRQ(ierr);
ierr = PCSetOperators(newpc,local_mat,local_mat,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = PCSetType(newpc,PCSHELL);CHKERRQ(ierr);
ierr = PCShellSetContext(newpc,shell_ctx);CHKERRQ(ierr);
ierr = PCShellSetApply(newpc,PCBDDCApplyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCShellSetDestroy(newpc,PCBDDCDestroyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCSetUp(newpc);CHKERRQ(ierr);
ierr = KSPSetPC(*local_ksp,newpc);CHKERRQ(ierr);
ierr = PCDestroy(&newpc);CHKERRQ(ierr);
ierr = KSPSetUp(*local_ksp);CHKERRQ(ierr);
/* test */
/* TODO: this cause a deadlock when doing multilevel */
#if 0
if (pcbddc->dbg_flag) {
KSP check_ksp;
PC check_pc;
Mat test_mat;
Vec work3;
PetscViewer viewer=pcbddc->dbg_viewer;
PetscReal test_err,lambda_min,lambda_max;
PetscBool setsym,issym=PETSC_FALSE;
ierr = KSPGetPC(*local_ksp,&check_pc);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work3);CHKERRQ(ierr);
ierr = VecSetRandom(shell_ctx->work_small_1,NULL);CHKERRQ(ierr);
ierr = MatMult(shell_ctx->basis_mat,shell_ctx->work_small_1,work1);CHKERRQ(ierr);
ierr = VecCopy(work1,work2);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work3);CHKERRQ(ierr);
ierr = PCApply(check_pc,work3,work1);CHKERRQ(ierr);
ierr = VecAXPY(work1,m_one,work2);CHKERRQ(ierr);
ierr = VecNorm(work1,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d error for nullspace correction for ",PetscGlobalRank);
if (basis_dofs == n_I) {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Dirichlet ");
} else {
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Neumann ");
}
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"solver is :%1.14e\n",test_err);
ierr = MatTransposeMatMult(shell_ctx->Lbasis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&test_mat);CHKERRQ(ierr);
ierr = MatShift(test_mat,one);CHKERRQ(ierr);
ierr = MatNorm(test_mat,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = MatDestroy(&test_mat);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d error for nullspace matrices is :%1.14e\n",PetscGlobalRank,test_err);
/* Create ksp object suitable for extreme eigenvalues' estimation */
ierr = KSPCreate(PETSC_COMM_SELF,&check_ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(check_ksp,local_mat,local_mat,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetTolerances(check_ksp,1.e-8,1.e-8,PETSC_DEFAULT,basis_dofs);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(check_ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatIsSymmetricKnown(pc->pmat,&setsym,&issym);CHKERRQ(ierr);
if (issym) {
ierr = KSPSetType(check_ksp,KSPCG);CHKERRQ(ierr);
}
ierr = KSPSetPC(check_ksp,check_pc);CHKERRQ(ierr);
ierr = KSPSetUp(check_ksp);CHKERRQ(ierr);
ierr = VecSetRandom(work1,NULL);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = KSPSolve(check_ksp,work2,work2);CHKERRQ(ierr);
ierr = VecAXPY(work2,m_one,work1);CHKERRQ(ierr);
ierr = VecNorm(work2,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = KSPComputeExtremeSingularValues(check_ksp,&lambda_max,&lambda_min);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(check_ksp,&k);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Subdomain %04d error for adapted KSP %1.14e (it %d, eigs %1.6e %1.6e)\n",PetscGlobalRank,test_err,k,lambda_min,lambda_max);
ierr = KSPDestroy(&check_ksp);CHKERRQ(ierr);
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecDestroy(&work3);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
}
#endif
PetscFunctionReturn(0);
}
/*@
KSPSetFromOptions - Sets KSP options from the options database.
This routine must be called before KSPSetUp() if the user is to be
allowed to set the Krylov type.
Collective on KSP
Input Parameters:
. ksp - the Krylov space context
Options Database Keys:
+ -ksp_max_it - maximum number of linear iterations
. -ksp_rtol rtol - relative tolerance used in default determination of convergence, i.e.
if residual norm decreases by this factor than convergence is declared
. -ksp_atol abstol - absolute tolerance used in default convergence test, i.e. if residual
norm is less than this then convergence is declared
. -ksp_divtol tol - if residual norm increases by this factor than divergence is declared
. -ksp_converged_use_initial_residual_norm - see KSPConvergedDefaultSetUIRNorm()
. -ksp_converged_use_min_initial_residual_norm - see KSPConvergedDefaultSetUMIRNorm()
. -ksp_norm_type - none - skip norms used in convergence tests (useful only when not using
convergence test (say you always want to run with 5 iterations) to
save on communication overhead
preconditioned - default for left preconditioning
unpreconditioned - see KSPSetNormType()
natural - see KSPSetNormType()
. -ksp_check_norm_iteration it - do not compute residual norm until iteration number it (does compute at 0th iteration)
works only for PCBCGS, PCIBCGS and and PCCG
. -ksp_lag_norm - compute the norm of the residual for the ith iteration on the i+1 iteration; this means that one can use
the norm of the residual for convergence test WITHOUT an extra MPI_Allreduce() limiting global synchronizations.
This will require 1 more iteration of the solver than usual.
. -ksp_fischer_guess <model,size> - uses the Fischer initial guess generator for repeated linear solves
. -ksp_constant_null_space - assume the operator (matrix) has the constant vector in its null space
. -ksp_test_null_space - tests the null space set with KSPSetNullSpace() to see if it truly is a null space
. -ksp_knoll - compute initial guess by applying the preconditioner to the right hand side
. -ksp_monitor_cancel - cancel all previous convergene monitor routines set
. -ksp_monitor <optional filename> - print residual norm at each iteration
. -ksp_monitor_lg_residualnorm - plot residual norm at each iteration
. -ksp_monitor_solution - plot solution at each iteration
- -ksp_monitor_singular_value - monitor extremem singular values at each iteration
Notes:
To see all options, run your program with the -help option
or consult Users-Manual: ch_ksp
Level: beginner
.keywords: KSP, set, from, options, database
.seealso: KSPSetUseFischerGuess()
@*/
PetscErrorCode KSPSetFromOptions(KSP ksp)
{
PetscErrorCode ierr;
PetscInt indx;
const char *convtests[] = {"default","skip"};
char type[256], monfilename[PETSC_MAX_PATH_LEN];
PetscViewer monviewer;
PetscBool flg,flag,reuse;
PetscInt model[2]={0,0},nmax;
KSPNormType normtype;
PCSide pcside;
void *ctx;
PetscFunctionBegin;
PetscValidHeaderSpecific(ksp,KSP_CLASSID,1);
if (!ksp->pc) {ierr = KSPGetPC(ksp,&ksp->pc);CHKERRQ(ierr);}
ierr = PCSetFromOptions(ksp->pc);CHKERRQ(ierr);
if (!KSPRegisterAllCalled) {ierr = KSPRegisterAll();CHKERRQ(ierr);}
ierr = PetscObjectOptionsBegin((PetscObject)ksp);CHKERRQ(ierr);
ierr = PetscOptionsFList("-ksp_type","Krylov method","KSPSetType",KSPList,(char*)(((PetscObject)ksp)->type_name ? ((PetscObject)ksp)->type_name : KSPGMRES),type,256,&flg);CHKERRQ(ierr);
if (flg) {
ierr = KSPSetType(ksp,type);CHKERRQ(ierr);
}
/*
Set the type if it was never set.
*/
if (!((PetscObject)ksp)->type_name) {
ierr = KSPSetType(ksp,KSPGMRES);CHKERRQ(ierr);
}
ierr = PetscOptionsInt("-ksp_max_it","Maximum number of iterations","KSPSetTolerances",ksp->max_it,&ksp->max_it,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-ksp_rtol","Relative decrease in residual norm","KSPSetTolerances",ksp->rtol,&ksp->rtol,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-ksp_atol","Absolute value of residual norm","KSPSetTolerances",ksp->abstol,&ksp->abstol,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-ksp_divtol","Residual norm increase cause divergence","KSPSetTolerances",ksp->divtol,&ksp->divtol,NULL);CHKERRQ(ierr);
flag = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_converged_use_initial_residual_norm","Use initial residual residual norm for computing relative convergence","KSPConvergedDefaultSetUIRNorm",flag,&flag,NULL);CHKERRQ(ierr);
if (flag) {ierr = KSPConvergedDefaultSetUIRNorm(ksp);CHKERRQ(ierr);}
flag = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_converged_use_min_initial_residual_norm","Use minimum of initial residual norm and b for computing relative convergence","KSPConvergedDefaultSetUMIRNorm",flag,&flag,NULL);CHKERRQ(ierr);
if (flag) {ierr = KSPConvergedDefaultSetUMIRNorm(ksp);CHKERRQ(ierr);}
ierr = KSPGetInitialGuessNonzero(ksp,&flag);CHKERRQ(ierr);
ierr = PetscOptionsBool("-ksp_initial_guess_nonzero","Use the contents of the solution vector for initial guess","KSPSetInitialNonzero",flag,&flag,&flg);CHKERRQ(ierr);
if (flg) {
ierr = KSPSetInitialGuessNonzero(ksp,flag);CHKERRQ(ierr);
}
ierr = PCGetReusePreconditioner(ksp->pc,&reuse);CHKERRQ(ierr);
ierr = PetscOptionsBool("-ksp_reuse_preconditioner","Use initial preconditioner and don't ever compute a new one ","KSPReusePreconditioner",reuse,&reuse,NULL);CHKERRQ(ierr);
ierr = KSPSetReusePreconditioner(ksp,reuse);CHKERRQ(ierr);
ierr = PetscOptionsBool("-ksp_knoll","Use preconditioner applied to b for initial guess","KSPSetInitialGuessKnoll",ksp->guess_knoll,&ksp->guess_knoll,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-ksp_error_if_not_converged","Generate error if solver does not converge","KSPSetErrorIfNotConverged",ksp->errorifnotconverged,&ksp->errorifnotconverged,NULL);CHKERRQ(ierr);
nmax = 2;
ierr = PetscOptionsIntArray("-ksp_fischer_guess","Use Paul Fischer's algorithm for initial guess","KSPSetUseFischerGuess",model,&nmax,&flag);CHKERRQ(ierr);
if (flag) {
if (nmax != 2) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_ARG_OUTOFRANGE,"Must pass in model,size as arguments");
ierr = KSPSetUseFischerGuess(ksp,model[0],model[1]);CHKERRQ(ierr);
}
ierr = PetscOptionsEList("-ksp_convergence_test","Convergence test","KSPSetConvergenceTest",convtests,2,"default",&indx,&flg);CHKERRQ(ierr);
if (flg) {
switch (indx) {
case 0:
ierr = KSPConvergedDefaultCreate(&ctx);CHKERRQ(ierr);
ierr = KSPSetConvergenceTest(ksp,KSPConvergedDefault,ctx,KSPConvergedDefaultDestroy);CHKERRQ(ierr);
break;
case 1: ierr = KSPSetConvergenceTest(ksp,KSPConvergedSkip,NULL,NULL);CHKERRQ(ierr); break;
}
}
ierr = KSPSetUpNorms_Private(ksp,&normtype,&pcside);CHKERRQ(ierr);
ierr = PetscOptionsEnum("-ksp_norm_type","KSP Norm type","KSPSetNormType",KSPNormTypes,(PetscEnum)normtype,(PetscEnum*)&normtype,&flg);CHKERRQ(ierr);
if (flg) { ierr = KSPSetNormType(ksp,normtype);CHKERRQ(ierr); }
ierr = PetscOptionsInt("-ksp_check_norm_iteration","First iteration to compute residual norm","KSPSetCheckNormIteration",ksp->chknorm,&ksp->chknorm,NULL);CHKERRQ(ierr);
flag = ksp->lagnorm;
ierr = PetscOptionsBool("-ksp_lag_norm","Lag the calculation of the residual norm","KSPSetLagNorm",flag,&flag,&flg);CHKERRQ(ierr);
if (flg) {
ierr = KSPSetLagNorm(ksp,flag);CHKERRQ(ierr);
}
ierr = KSPGetDiagonalScale(ksp,&flag);CHKERRQ(ierr);
ierr = PetscOptionsBool("-ksp_diagonal_scale","Diagonal scale matrix before building preconditioner","KSPSetDiagonalScale",flag,&flag,&flg);CHKERRQ(ierr);
if (flg) {
ierr = KSPSetDiagonalScale(ksp,flag);CHKERRQ(ierr);
}
ierr = KSPGetDiagonalScaleFix(ksp,&flag);CHKERRQ(ierr);
ierr = PetscOptionsBool("-ksp_diagonal_scale_fix","Fix diagonally scaled matrix after solve","KSPSetDiagonalScaleFix",flag,&flag,&flg);CHKERRQ(ierr);
if (flg) {
ierr = KSPSetDiagonalScaleFix(ksp,flag);CHKERRQ(ierr);
}
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_constant_null_space","Add constant null space to Krylov solver","KSPSetNullSpace",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {
MatNullSpace nsp;
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)ksp),PETSC_TRUE,0,0,&nsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(ksp,nsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nsp);CHKERRQ(ierr);
}
/* option is actually checked in KSPSetUp(), just here so goes into help message */
if (ksp->nullsp) {
ierr = PetscOptionsName("-ksp_test_null_space","Is provided null space correct","None",&flg);CHKERRQ(ierr);
}
/*
Prints reason for convergence or divergence of each linear solve
*/
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_converged_reason","Print reason for converged or diverged","KSPSolve",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) ksp->printreason = PETSC_TRUE;
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_monitor_cancel","Remove any hardwired monitor routines","KSPMonitorCancel",flg,&flg,NULL);CHKERRQ(ierr);
/* -----------------------------------------------------------------------*/
/*
Cancels all monitors hardwired into code before call to KSPSetFromOptions()
*/
if (flg) {
ierr = KSPMonitorCancel(ksp);CHKERRQ(ierr);
}
/*
Prints preconditioned residual norm at each iteration
*/
ierr = PetscOptionsString("-ksp_monitor","Monitor preconditioned residual norm","KSPMonitorSet","stdout",monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscViewerASCIIOpen(PetscObjectComm((PetscObject)ksp),monfilename,&monviewer);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorDefault,monviewer,(PetscErrorCode (*)(void**))PetscViewerDestroy);CHKERRQ(ierr);
}
/*
Prints preconditioned residual norm at each iteration
*/
ierr = PetscOptionsString("-ksp_monitor_range","Monitor percent of residual entries more than 10 percent of max","KSPMonitorRange","stdout",monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscViewerASCIIOpen(PetscObjectComm((PetscObject)ksp),monfilename,&monviewer);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorRange,monviewer,(PetscErrorCode (*)(void**))PetscViewerDestroy);CHKERRQ(ierr);
}
ierr = PetscObjectTypeCompare((PetscObject)ksp->pc,PCKSP,&flg);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)ksp->pc,PCBJACOBI,&flag);CHKERRQ(ierr);
if (flg || flag) {
/* A hack for using dynamic tolerance in preconditioner */
ierr = PetscOptionsString("-sub_ksp_dynamic_tolerance","Use dynamic tolerance for PC if PC is a KSP","KSPMonitorDynamicTolerance","stdout",monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {
KSPDynTolCtx *scale = NULL;
PetscReal defaultv = 1.0;
ierr = PetscMalloc1(1,&scale);CHKERRQ(ierr);
scale->bnrm = -1.0;
scale->coef = defaultv;
ierr = PetscOptionsReal("-sub_ksp_dynamic_tolerance_param","Parameter of dynamic tolerance for inner PCKSP","KSPMonitorDynamicToleranceParam",defaultv,&(scale->coef),&flg);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorDynamicTolerance,scale,KSPMonitorDynamicToleranceDestroy);CHKERRQ(ierr);
}
}
/*
Plots the vector solution
*/
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_monitor_solution","Monitor solution graphically","KSPMonitorSet",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {
ierr = KSPMonitorSet(ksp,KSPMonitorSolution,NULL,NULL);CHKERRQ(ierr);
}
/*
Prints preconditioned and true residual norm at each iteration
*/
ierr = PetscOptionsString("-ksp_monitor_true_residual","Monitor true residual norm","KSPMonitorSet","stdout",monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscViewerASCIIOpen(PetscObjectComm((PetscObject)ksp),monfilename,&monviewer);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorTrueResidualNorm,monviewer,(PetscErrorCode (*)(void**))PetscViewerDestroy);CHKERRQ(ierr);
}
/*
Prints with max norm at each iteration
*/
ierr = PetscOptionsString("-ksp_monitor_max","Monitor true residual max norm","KSPMonitorSet","stdout",monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscViewerASCIIOpen(PetscObjectComm((PetscObject)ksp),monfilename,&monviewer);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorTrueResidualMaxNorm,monviewer,(PetscErrorCode (*)(void**))PetscViewerDestroy);CHKERRQ(ierr);
}
/*
Prints extreme eigenvalue estimates at each iteration
*/
ierr = PetscOptionsString("-ksp_monitor_singular_value","Monitor singular values","KSPMonitorSet","stdout",monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {
ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PetscObjectComm((PetscObject)ksp),monfilename,&monviewer);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorSingularValue,monviewer,(PetscErrorCode (*)(void**))PetscViewerDestroy);CHKERRQ(ierr);
}
/*
Prints preconditioned residual norm with fewer digits
*/
ierr = PetscOptionsString("-ksp_monitor_short","Monitor preconditioned residual norm with fewer digits","KSPMonitorSet","stdout",monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscViewerASCIIOpen(PetscObjectComm((PetscObject)ksp),monfilename,&monviewer);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorDefaultShort,monviewer,(PetscErrorCode (*)(void**))PetscViewerDestroy);CHKERRQ(ierr);
}
/*
Calls Python function
*/
ierr = PetscOptionsString("-ksp_monitor_python","Use Python function","KSPMonitorSet",0,monfilename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (flg) {ierr = PetscPythonMonitorSet((PetscObject)ksp,monfilename);CHKERRQ(ierr);}
/*
Graphically plots preconditioned residual norm
*/
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_monitor_lg_residualnorm","Monitor graphically preconditioned residual norm","KSPMonitorSet",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscDrawLG ctx;
ierr = KSPMonitorLGResidualNormCreate(0,0,PETSC_DECIDE,PETSC_DECIDE,300,300,&ctx);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorLGResidualNorm,ctx,(PetscErrorCode (*)(void**))KSPMonitorLGResidualNormDestroy);CHKERRQ(ierr);
}
/*
Graphically plots preconditioned and true residual norm
*/
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_monitor_lg_true_residualnorm","Monitor graphically true residual norm","KSPMonitorSet",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscDrawLG ctx;
ierr = KSPMonitorLGTrueResidualNormCreate(PetscObjectComm((PetscObject)ksp),0,0,PETSC_DECIDE,PETSC_DECIDE,300,300,&ctx);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorLGTrueResidualNorm,ctx,(PetscErrorCode (*)(void**))KSPMonitorLGTrueResidualNormDestroy);CHKERRQ(ierr);
}
/*
Graphically plots preconditioned residual norm and range of residual element values
*/
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_monitor_lg_range","Monitor graphically range of preconditioned residual norm","KSPMonitorSet",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer ctx;
ierr = PetscViewerDrawOpen(PetscObjectComm((PetscObject)ksp),0,0,PETSC_DECIDE,PETSC_DECIDE,300,300,&ctx);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorLGRange,ctx,(PetscErrorCode (*)(void**))PetscViewerDestroy);CHKERRQ(ierr);
}
#if defined(PETSC_HAVE_SAWS)
/*
Publish convergence information using AMS
*/
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_monitor_saws","Publish KSP progress using SAWs","KSPMonitorSet",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {
void *ctx;
ierr = KSPMonitorSAWsCreate(ksp,&ctx);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPMonitorSAWs,ctx,KSPMonitorSAWsDestroy);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr);
}
#endif
/* -----------------------------------------------------------------------*/
ierr = KSPSetUpNorms_Private(ksp,&normtype,&pcside);CHKERRQ(ierr);
ierr = PetscOptionsEnum("-ksp_pc_side","KSP preconditioner side","KSPSetPCSide",PCSides,(PetscEnum)pcside,(PetscEnum*)&pcside,&flg);CHKERRQ(ierr);
if (flg) {ierr = KSPSetPCSide(ksp,pcside);CHKERRQ(ierr);}
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_compute_singularvalues","Compute singular values of preconditioned operator","KSPSetComputeSingularValues",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) { ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr); }
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_compute_eigenvalues","Compute eigenvalues of preconditioned operator","KSPSetComputeSingularValues",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) { ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr); }
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_plot_eigenvalues","Scatter plot extreme eigenvalues","KSPSetComputeSingularValues",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) { ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE);CHKERRQ(ierr); }
#if defined(PETSC_HAVE_SAWS)
{
PetscBool set;
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_saws_block","Block for SAWs at end of KSPSolve","PetscObjectSAWsBlock",((PetscObject)ksp)->amspublishblock,&flg,&set);CHKERRQ(ierr);
if (set) {
ierr = PetscObjectSAWsSetBlock((PetscObject)ksp,flg);CHKERRQ(ierr);
}
}
#endif
if (ksp->ops->setfromoptions) {
ierr = (*ksp->ops->setfromoptions)(ksp);CHKERRQ(ierr);
}
/* process any options handlers added with PetscObjectAddOptionsHandler() */
ierr = PetscObjectProcessOptionsHandlers((PetscObject)ksp);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCNullSpaceAdaptGlobal(PC pc)
{
PC_IS* pcis = (PC_IS*) (pc->data);
PC_BDDC* pcbddc = (PC_BDDC*)(pc->data);
KSP inv_change;
PC pc_change;
const Vec *nsp_vecs;
Vec *new_nsp_vecs;
PetscInt i,nsp_size,new_nsp_size,start_new;
PetscBool nsp_has_cnst;
MatNullSpace new_nsp;
PetscErrorCode ierr;
MPI_Comm comm;
PetscFunctionBegin;
/* create KSP for change of basis */
ierr = KSPCreate(PETSC_COMM_SELF,&inv_change);CHKERRQ(ierr);
ierr = KSPSetOperators(inv_change,pcbddc->ChangeOfBasisMatrix,pcbddc->ChangeOfBasisMatrix,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetType(inv_change,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(inv_change,&pc_change);CHKERRQ(ierr);
ierr = PCSetType(pc_change,PCLU);CHKERRQ(ierr);
ierr = KSPSetUp(inv_change);CHKERRQ(ierr);
/* get nullspace and transform it */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nsp_has_cnst,&nsp_size,&nsp_vecs);CHKERRQ(ierr);
new_nsp_size = nsp_size;
if (nsp_has_cnst) {
new_nsp_size++;
}
ierr = PetscMalloc(new_nsp_size*sizeof(Vec),&new_nsp_vecs);CHKERRQ(ierr);
for (i=0;i<new_nsp_size;i++) {
ierr = VecDuplicate(pcis->vec1_global,&new_nsp_vecs[i]);CHKERRQ(ierr);
}
start_new = 0;
if (nsp_has_cnst) {
start_new = 1;
ierr = VecSet(new_nsp_vecs[0],1.0);CHKERRQ(ierr);
ierr = VecSet(pcis->vec1_B,1.0);CHKERRQ(ierr);
ierr = KSPSolve(inv_change,pcis->vec1_B,pcis->vec1_B);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[0],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[0],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
for (i=0;i<nsp_size;i++) {
ierr = VecCopy(nsp_vecs[i],new_nsp_vecs[i+start_new]);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,nsp_vecs[i],pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,nsp_vecs[i],pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = KSPSolve(inv_change,pcis->vec1_B,pcis->vec1_B);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[i+start_new],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,new_nsp_vecs[i+start_new],INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
ierr = PCBDDCOrthonormalizeVecs(new_nsp_size,new_nsp_vecs);CHKERRQ(ierr);
#if 0
PetscBool nsp_t=PETSC_FALSE;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("Original Null Space test: %d\n",nsp_t);
Mat temp_mat;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("Original Null Space, mat changed test: %d\n",nsp_t);
{
PetscReal test_norm;
for (i=0;i<new_nsp_size;i++) {
ierr = MatMult(pc->pmat,new_nsp_vecs[i],pcis->vec1_global);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_global,NORM_2,&test_norm);CHKERRQ(ierr);
if (test_norm > 1.e-12) {
printf("------------ERROR VEC %d------------------\n",i);
ierr = VecView(pcis->vec1_global,PETSC_VIEWER_STDOUT_WORLD);
printf("------------------------------------------\n");
}
}
}
#endif
ierr = KSPDestroy(&inv_change);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(comm,PETSC_FALSE,new_nsp_size,new_nsp_vecs,&new_nsp);CHKERRQ(ierr);
ierr = PCBDDCSetNullSpace(pc,new_nsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&new_nsp);CHKERRQ(ierr);
#if 0
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("New Null Space, mat changed: %d\n",nsp_t);
temp_mat = matis->A;
matis->A = pcbddc->local_mat;
pcbddc->local_mat = temp_mat;
ierr = MatNullSpaceTest(pcbddc->NullSpace,pc->pmat,&nsp_t);CHKERRQ(ierr);
printf("New Null Space, mat original: %d\n",nsp_t);
#endif
for (i=0;i<new_nsp_size;i++) {
ierr = VecDestroy(&new_nsp_vecs[i]);CHKERRQ(ierr);
}
ierr = PetscFree(new_nsp_vecs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode TaoCreate_NTR(Tao tao)
{
TAO_NTR *tr;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscNewLog(tao,&tr);CHKERRQ(ierr);
tao->ops->setup = TaoSetUp_NTR;
tao->ops->solve = TaoSolve_NTR;
tao->ops->view = TaoView_NTR;
tao->ops->setfromoptions = TaoSetFromOptions_NTR;
tao->ops->destroy = TaoDestroy_NTR;
/* Override default settings (unless already changed) */
if (!tao->max_it_changed) tao->max_it = 50;
if (!tao->trust0_changed) tao->trust0 = 100.0;
tao->data = (void*)tr;
/* Standard trust region update parameters */
tr->eta1 = 1.0e-4;
tr->eta2 = 0.25;
tr->eta3 = 0.50;
tr->eta4 = 0.90;
tr->alpha1 = 0.25;
tr->alpha2 = 0.50;
tr->alpha3 = 1.00;
tr->alpha4 = 2.00;
tr->alpha5 = 4.00;
/* Interpolation trust region update parameters */
tr->mu1 = 0.10;
tr->mu2 = 0.50;
tr->gamma1 = 0.25;
tr->gamma2 = 0.50;
tr->gamma3 = 2.00;
tr->gamma4 = 4.00;
tr->theta = 0.05;
/* Interpolation parameters for initialization */
tr->mu1_i = 0.35;
tr->mu2_i = 0.50;
tr->gamma1_i = 0.0625;
tr->gamma2_i = 0.50;
tr->gamma3_i = 2.00;
tr->gamma4_i = 5.00;
tr->theta_i = 0.25;
tr->min_radius = 1.0e-10;
tr->max_radius = 1.0e10;
tr->epsilon = 1.0e-6;
tr->pc_type = NTR_PC_BFGS;
tr->bfgs_scale_type = BFGS_SCALE_AHESS;
tr->init_type = NTR_INIT_INTERPOLATION;
tr->update_type = NTR_UPDATE_REDUCTION;
/* Set linear solver to default for trust region */
ierr = KSPCreate(((PetscObject)tao)->comm,&tao->ksp);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(tao->ksp,tao->hdr.prefix);CHKERRQ(ierr);
ierr = KSPSetType(tao->ksp,KSPCGSTCG);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FluidFieldSetup( FluidField f )
{
PetscLogDouble t1,t2;
PetscErrorCode ierr;
PetscFunctionBegin;
// Assemble viscous matricies
ierr = FluidFieldMatAssemble( f ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Lengths: %e %e %e\n", f->lens.x, f->lens.y, f->lens.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Size: %d %d %d\n", f->dims.x, f->dims.y, f->dims.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "dx: %e %e %e\n", f->dh.x, f->dh.y, f->dh.z ); CHKERRQ(ierr);
ierr = PetscTime(&t1); CHKERRQ(ierr);
// Create vectors
ierr = GACreate( f->daV, &f->ga); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daV,&f->rhs); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel0); CHKERRQ(ierr);
// ierr = DACreateGlobalVector(f->daE,&f->E); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daB,&f->buf); CHKERRQ(ierr);
// Set up the outer solver
ierr = KSPCreate(f->comm,&f->ksp); CHKERRQ(ierr);
ierr = KSPSetOperators(f->ksp,f->mat,f->mat, SAME_PRECONDITIONER); CHKERRQ(ierr);
ierr = KSPSetType(f->ksp,KSPFGMRES); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(f->ksp,PETSC_TRUE); CHKERRQ(ierr);
ierr = KSPSetTolerances(f->ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
ierr = KSPSetFromOptions(f->ksp);CHKERRQ(ierr);
// Split pressure from velocity [ u v w | p ]
PC pc;
ierr = KSPGetPC(f->ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
if( f->is3D ) {
const PetscInt ufields[] = {U_FACE,V_FACE,W_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,4); CHKERRQ(ierr); // [p u v w]
ierr = PCFieldSplitSetFields(pc,"v",3,ufields,ufields); CHKERRQ(ierr); // [u v w]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
} else {
const PetscInt ufields[] = {U_FACE,V_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr); // [p u v]
ierr = PCFieldSplitSetFields(pc,"v",2,ufields,ufields); CHKERRQ(ierr); // [u v]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
}
ierr = PCSetUp(pc); CHKERRQ(ierr);
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,4); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[1],KSPGMRES); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspVelP[1]);CHKERRQ(ierr);
// Split velocity [u v w] into component matricies [u], [v], [w]
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,f->is3D?3:2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
/* Set solver for each velocity component
* Split component velocity as parallel blocks along processors
* Use direct solver for each block
* TODO: use MG, w/FFT on coarse grid
*/
ierr = PetscTime(&t2); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Finished Solver Setup: %f sec\n",t2-t1); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv){
PetscErrorCode ierr;
int nx = 63, ny = 63;
DM dm;
PetscBool flg;
Mat A;
Vec u, b;
KSP solver;
PC pc;
double norm;
PetscInt stage;
ierr = PetscInitialize(&argc, &argv, NULL, NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL, "-nx", &nx, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL, "-ny", &ny, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL, "-assemble", &flg);CHKERRQ(ierr);
ierr = PetscLogStageRegister("preparing",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Domain creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = createDomain(&dm, nx, ny);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("matrix creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = createMat(dm, &A, flg);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Vector creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dm, &b);CHKERRQ(ierr);
ierr = VecDuplicate(b, &u);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Domain initialisation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = init2d(dm, b);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("solver creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD, &solver);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(solver, "poisson_");CHKERRQ(ierr);
ierr = KSPSetOperators(solver, A, A, DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPSetType(solver, KSPCG);
ierr = KSPGetPC(solver, &pc);CHKERRQ(ierr);
ierr = PCSetType(pc, PCNONE);CHKERRQ(ierr);
ierr = KSPSetFromOptions(solver);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Solving",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = KSPSolve(solver, b, u);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
//ierr = VecView(u, PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
//sleep(10);
VecDestroy(&u);
VecDestroy(&b);
MatDestroy(&A);
DMDestroy(&dm);
KSPDestroy(&solver);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode GetH(MPI_Comm comm, Mat *Hout, int mx, int my, int mz, double s, double nR, int dim, KSP *kspHout, PC *pcHout)
{
PetscErrorCode ierr;
Mat H;
int i,ns,ne;
int N=mx*my*mz;
MatCreate(comm, &H);
MatSetType(H,MATMPIAIJ);
MatSetSizes(H,PETSC_DECIDE, PETSC_DECIDE, N, N);
MatMPIAIJSetPreallocation(H, 3, PETSC_NULL, 3, PETSC_NULL);
if (dim==1){
double value[3];
int col[3];
ierr = MatGetOwnershipRange(H, &ns, &ne); CHKERRQ(ierr);
for (i = ns; i < ne; ++i) {
if (i==0){
col[0]= 0, value[0]= s*nR*nR*(-2.0)+1;
col[1]= 1, value[1]= s*nR*nR*2.0;
col[2]= 2, value[2]= 0;}
else if (i==N-1){
col[0]= N-3, value[0]= 0;
col[1]= N-2, value[1]= s*nR*nR*2.0;
col[2]= N-1, value[2]= s*nR*nR*(-2.0)+1;}
else{
col[0]= i-1, value[0]= s*nR*nR;
col[1]= i, value[1]= s*nR*nR*(-2.0)+1;
col[2]= i+1, value[2]= s*nR*nR;}
ierr = MatSetValues(H,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(H, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(H, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
}
if (dim==2) {
PetscPrintf(comm,"2D Helmholtz filter is not implemented yet!\n");
MatShift(H,1.0);}
if (dim==3) {
PetscPrintf(comm,"3D Helmholtz filter is not implemented yet!\n");
MatShift(H,1.0);}
ierr = PetscObjectSetName((PetscObject) H,"Hop"); CHKERRQ(ierr);
KSP ksp;
PC pc;
ierr = KSPCreate(comm,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp, KSPGMRES);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCLU);CHKERRQ(ierr);
ierr = PCFactorSetMatSolverPackage(pc,MATSOLVERMUMPS);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp,1e-14,PETSC_DEFAULT,PETSC_DEFAULT,maxit);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
*Hout=H;
*kspHout=ksp;
*pcHout=pc;
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat C;
PetscScalar v,none = -1.0;
PetscInt i,j,Ii,J,Istart,Iend,N,m = 4,n = 4,its,k;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscReal err_norm,res_norm,err_tol=1.e-7,res_tol=1.e-6;
Vec x,b,u,u_tmp;
PetscRandom r;
PC pc;
KSP ksp;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
N = m*n;
/* Generate matrix */
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,N,N);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(C,&Istart,&Iend);CHKERRQ(ierr);
for (Ii=Istart; Ii<Iend; Ii++) {
v = -1.0; i = Ii/n; j = Ii - i*n;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* a shift can make C indefinite. Preconditioners LU, ILU (for BAIJ format) and ICC may fail */
/* ierr = MatShift(C,alpha);CHKERRQ(ierr); */
/* ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Setup and solve for system */
/* Create vectors. */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,N);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);CHKERRQ(ierr);
ierr = VecDuplicate(x,&u_tmp);CHKERRQ(ierr);
/* Set exact solution u; then compute right-hand-side vector b. */
ierr = PetscRandomCreate(PETSC_COMM_SELF,&r);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(r);CHKERRQ(ierr);
ierr = VecSetRandom(u,r);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&r);CHKERRQ(ierr);
ierr = MatMult(C,u,b);CHKERRQ(ierr);
for (k=0; k<3; k++) {
if (k == 0) { /* CG */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,C,C);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n CG: \n");CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPCG);CHKERRQ(ierr);
} else if (k == 1) { /* MINRES */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,C,C);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n MINRES: \n");CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPMINRES);CHKERRQ(ierr);
} else { /* SYMMLQ */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,C,C);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n SYMMLQ: \n");CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPSYMMLQ);CHKERRQ(ierr);
}
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
/* ierr = PCSetType(pc,PCICC);CHKERRQ(ierr); */
ierr = PCSetType(pc,PCJACOBI);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp,1.e-7,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);CHKERRQ(ierr);
/*
Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
These options will override those specified above as long as
KSPSetFromOptions() is called _after_ any other customization
routines.
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* Solve linear system; */
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
/* Check error */
ierr = VecCopy(u,u_tmp);CHKERRQ(ierr);
ierr = VecAXPY(u_tmp,none,x);CHKERRQ(ierr);
ierr = VecNorm(u_tmp,NORM_2,&err_norm);CHKERRQ(ierr);
ierr = MatMult(C,x,u_tmp);CHKERRQ(ierr);
ierr = VecAXPY(u_tmp,none,b);CHKERRQ(ierr);
ierr = VecNorm(u_tmp,NORM_2,&res_norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Number of iterations = %3D\n",its);CHKERRQ(ierr);
if (res_norm > res_tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Residual norm %g;",(double)res_norm);CHKERRQ(ierr);
}
if (err_norm > err_tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD," Error norm %g.\n",(double)err_norm);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&u_tmp);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv) {
PetscErrorCode ierr;
PetscBool view = PETSC_FALSE,
viewsoln = PETSC_FALSE,
noprealloc = PETSC_FALSE;
char root[256] = "", nodesname[256], issname[256], solnname[256];
UM mesh;
unfemCtx user;
SNES snes;
KSP ksp;
PC pc;
Mat A;
Vec r, u, uexact;
double err, h_max;
PetscInitialize(&argc,&argv,NULL,help);
ierr = PetscLogStageRegister("Read mesh ", &user.readstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Set-up ", &user.setupstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Solver ", &user.solverstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Residual eval ", &user.resstage); CHKERRQ(ierr); //STRIP
ierr = PetscLogStageRegister("Jacobian eval ", &user.jacstage); CHKERRQ(ierr); //STRIP
user.quaddeg = 1;
user.solncase = 0;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, "un_", "options for unfem", ""); CHKERRQ(ierr);
ierr = PetscOptionsInt("-case",
"exact solution cases: 0=linear, 1=nonlinear, 2=nonhomoNeumann, 3=chapter3, 4=koch",
"unfem.c",user.solncase,&(user.solncase),NULL); CHKERRQ(ierr);
ierr = PetscOptionsString("-mesh",
"file name root of mesh stored in PETSc binary with .vec,.is extensions",
"unfem.c",root,root,sizeof(root),NULL); CHKERRQ(ierr);
ierr = PetscOptionsInt("-quaddeg",
"quadrature degree (1,2,3)",
"unfem.c",user.quaddeg,&(user.quaddeg),NULL); CHKERRQ(ierr);
ierr = PetscOptionsBool("-view",
"view loaded nodes and elements at stdout",
"unfem.c",view,&view,NULL); CHKERRQ(ierr);
ierr = PetscOptionsBool("-view_solution",
"view solution u(x,y) to binary file; uses root name of mesh plus .soln\nsee petsc2tricontour.py to view graphically",
"unfem.c",viewsoln,&viewsoln,NULL); CHKERRQ(ierr);
ierr = PetscOptionsBool("-noprealloc",
"do not perform preallocation before matrix assembly",
"unfem.c",noprealloc,&noprealloc,NULL); CHKERRQ(ierr);
ierr = PetscOptionsEnd(); CHKERRQ(ierr);
// set parameters and exact solution
user.a_fcn = &a_lin;
user.f_fcn = &f_lin;
user.uexact_fcn = &uexact_lin;
user.gD_fcn = &gD_lin;
user.gN_fcn = &gN_lin;
switch (user.solncase) {
case 0 :
break;
case 1 :
user.a_fcn = &a_nonlin;
user.f_fcn = &f_nonlin;
break;
case 2 :
user.gN_fcn = &gN_linneu;
break;
case 3 :
user.a_fcn = &a_square;
user.f_fcn = &f_square;
user.uexact_fcn = &uexact_square;
user.gD_fcn = &gD_square;
user.gN_fcn = NULL; // seg fault if ever called
break;
case 4 :
user.a_fcn = &a_koch;
user.f_fcn = &f_koch;
user.uexact_fcn = NULL;
user.gD_fcn = &gD_koch;
user.gN_fcn = NULL; // seg fault if ever called
break;
default :
SETERRQ(PETSC_COMM_WORLD,1,"other solution cases not implemented");
}
// determine filenames
strcpy(nodesname, root);
strncat(nodesname, ".vec", 4);
strcpy(issname, root);
strncat(issname, ".is", 3);
//STARTMAININITIAL
PetscLogStagePush(user.readstage); //STRIP
// read mesh object of type UM
ierr = UMInitialize(&mesh); CHKERRQ(ierr);
ierr = UMReadNodes(&mesh,nodesname); CHKERRQ(ierr);
ierr = UMReadISs(&mesh,issname); CHKERRQ(ierr);
ierr = UMStats(&mesh, &h_max, NULL, NULL, NULL); CHKERRQ(ierr);
if (view) { //STRIP
PetscViewer stdoutviewer; //STRIP
ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&stdoutviewer); CHKERRQ(ierr); //STRIP
ierr = UMViewASCII(&mesh,stdoutviewer); CHKERRQ(ierr); //STRIP
} //STRIP
user.mesh = &mesh;
PetscLogStagePop(); //STRIP
// configure Vecs and SNES
PetscLogStagePush(user.setupstage); //STRIP
ierr = VecCreate(PETSC_COMM_WORLD,&r); CHKERRQ(ierr);
ierr = VecSetSizes(r,PETSC_DECIDE,mesh.N); CHKERRQ(ierr);
ierr = VecSetFromOptions(r); CHKERRQ(ierr);
ierr = VecDuplicate(r,&u); CHKERRQ(ierr);
ierr = VecSet(u,0.0); CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes); CHKERRQ(ierr);
ierr = SNESSetFunction(snes,r,FormFunction,&user); CHKERRQ(ierr);
// reset default KSP and PC
ierr = SNESGetKSP(snes,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPCG); CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCICC); CHKERRQ(ierr);
// setup matrix for Picard iteration, including preallocation
ierr = MatCreate(PETSC_COMM_WORLD,&A); CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,mesh.N,mesh.N); CHKERRQ(ierr);
ierr = MatSetFromOptions(A); CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE); CHKERRQ(ierr);
if (noprealloc) {
ierr = MatSetUp(A); CHKERRQ(ierr);
} else {
ierr = Preallocation(A,&user); CHKERRQ(ierr);
}
ierr = SNESSetJacobian(snes,A,A,FormPicard,&user); CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes); CHKERRQ(ierr);
PetscLogStagePop(); //STRIP
// solve
PetscLogStagePush(user.solverstage); //STRIP
ierr = SNESSolve(snes,NULL,u);CHKERRQ(ierr);
PetscLogStagePop(); //STRIP
//ENDMAININITIAL
if (viewsoln) {
strcpy(solnname, root);
strncat(solnname, ".soln", 5);
ierr = UMViewSolutionBinary(&mesh,solnname,u); CHKERRQ(ierr);
}
if (user.uexact_fcn) {
// measure error relative to exact solution
ierr = VecDuplicate(r,&uexact); CHKERRQ(ierr);
ierr = FillExact(uexact,&user); CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,uexact); CHKERRQ(ierr); // u <- u + (-1.0) uexact
ierr = VecNorm(u,NORM_INFINITY,&err); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"case %d result for N=%d nodes with h = %.3e : |u-u_ex|_inf = %g\n",
user.solncase,mesh.N,h_max,err); CHKERRQ(ierr);
VecDestroy(&uexact);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,
"case %d completed for N=%d nodes with h = %.3e (no exact solution)\n",
user.solncase,mesh.N,h_max); CHKERRQ(ierr);
}
// clean-up
SNESDestroy(&snes);
MatDestroy(&A);
VecDestroy(&u); VecDestroy(&r);
UMDestroy(&mesh);
PetscFinalize();
return 0;
}
PetscErrorCode EPSSetFromOptions_JD(EPS eps)
{
PetscErrorCode ierr;
PetscBool flg,op;
PetscInt opi,opi0;
PetscReal opf;
KSP ksp;
PetscBool orth;
const char *orth_list[2] = {"I","B"};
PetscFunctionBegin;
ierr = PetscOptionsHead("EPS Jacobi-Davidson (JD) Options");CHKERRQ(ierr);
ierr = EPSJDGetKrylovStart(eps,&op);CHKERRQ(ierr);
ierr = PetscOptionsBool("-eps_jd_krylov_start","Start the searching subspace with a krylov basis","EPSJDSetKrylovStart",op,&op,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetKrylovStart(eps,op);CHKERRQ(ierr); }
ierr = EPSJDGetBlockSize(eps,&opi);CHKERRQ(ierr);
ierr = PetscOptionsInt("-eps_jd_blocksize","Number vectors add to the searching subspace","EPSJDSetBlockSize",opi,&opi,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetBlockSize(eps,opi);CHKERRQ(ierr); }
ierr = EPSJDGetRestart(eps,&opi,&opi0);CHKERRQ(ierr);
ierr = PetscOptionsInt("-eps_jd_minv","Set the size of the searching subspace after restarting","EPSJDSetRestart",opi,&opi,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetRestart(eps,opi,opi0);CHKERRQ(ierr); }
ierr = PetscOptionsInt("-eps_jd_plusk","Set the number of saved eigenvectors from the previous iteration when restarting","EPSJDSetRestart",opi0,&opi0,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetRestart(eps,opi,opi0);CHKERRQ(ierr); }
ierr = EPSJDGetInitialSize(eps,&opi);CHKERRQ(ierr);
ierr = PetscOptionsInt("-eps_jd_initial_size","Set the initial size of the searching subspace","EPSJDSetInitialSize",opi,&opi,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetInitialSize(eps,opi);CHKERRQ(ierr); }
ierr = EPSJDGetFix(eps,&opf);CHKERRQ(ierr);
ierr = PetscOptionsReal("-eps_jd_fix","Set the tolerance for changing the target in the correction equation","EPSJDSetFix",opf,&opf,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetFix(eps,opf);CHKERRQ(ierr); }
ierr = EPSJDGetBOrth(eps,&orth);CHKERRQ(ierr);
ierr = PetscOptionsEList("-eps_jd_borth","orthogonalization used in the search subspace","EPSJDSetBOrth",orth_list,2,orth_list[orth?1:0],&opi,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetBOrth(eps,opi==1?PETSC_TRUE:PETSC_FALSE);CHKERRQ(ierr); }
ierr = EPSJDGetConstCorrectionTol(eps,&op);CHKERRQ(ierr);
ierr = PetscOptionsBool("-eps_jd_const_correction_tol","Disable the dynamic stopping criterion when solving the correction equation","EPSJDSetConstCorrectionTol",op,&op,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetConstCorrectionTol(eps,op);CHKERRQ(ierr); }
ierr = EPSJDGetWindowSizes(eps,&opi,&opi0);CHKERRQ(ierr);
ierr = PetscOptionsInt("-eps_jd_pwindow","(Experimental!) Set the number of converged vectors in the projector","EPSJDSetWindowSizes",opi,&opi,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetWindowSizes(eps,opi,opi0);CHKERRQ(ierr); }
ierr = PetscOptionsInt("-eps_jd_qwindow","(Experimental!) Set the number of converged vectors in the projected problem","EPSJDSetWindowSizes",opi0,&opi0,&flg);CHKERRQ(ierr);
if (flg) { ierr = EPSJDSetWindowSizes(eps,opi,opi0);CHKERRQ(ierr); }
/* Set STPrecond as the default ST */
if (!((PetscObject)eps->st)->type_name) {
ierr = STSetType(eps->st,STPRECOND);CHKERRQ(ierr);
}
ierr = STPrecondSetKSPHasMat(eps->st,PETSC_FALSE);CHKERRQ(ierr);
/* Set the default options of the KSP */
ierr = STGetKSP(eps->st,&ksp);CHKERRQ(ierr);
if (!((PetscObject)ksp)->type_name) {
ierr = KSPSetType(ksp,KSPBCGSL);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp,1e-4,PETSC_DEFAULT,PETSC_DEFAULT,90);CHKERRQ(ierr);
}
ierr = PetscOptionsTail();CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCSetUp_MG(PC pc)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_MG_Levels **mglevels = mg->levels;
PetscErrorCode ierr;
PetscInt i,n = mglevels[0]->levels;
PC cpc;
PetscBool preonly,lu,redundant,cholesky,svd,dump = PETSC_FALSE,opsset,use_amat;
Mat dA,dB;
MatStructure uflag;
Vec tvec;
DM *dms;
PetscViewer viewer = 0;
PetscFunctionBegin;
/* FIX: Move this to PCSetFromOptions_MG? */
if (mg->usedmfornumberoflevels) {
PetscInt levels;
ierr = DMGetRefineLevel(pc->dm,&levels);
CHKERRQ(ierr);
levels++;
if (levels > n) { /* the problem is now being solved on a finer grid */
ierr = PCMGSetLevels(pc,levels,NULL);
CHKERRQ(ierr);
n = levels;
ierr = PCSetFromOptions(pc);
CHKERRQ(ierr); /* it is bad to call this here, but otherwise will never be called for the new hierarchy */
mglevels = mg->levels;
}
}
ierr = KSPGetPC(mglevels[0]->smoothd,&cpc);
CHKERRQ(ierr);
/* If user did not provide fine grid operators OR operator was not updated since last global KSPSetOperators() */
/* so use those from global PC */
/* Is this what we always want? What if user wants to keep old one? */
ierr = KSPGetOperatorsSet(mglevels[n-1]->smoothd,NULL,&opsset);
CHKERRQ(ierr);
if (opsset) {
Mat mmat;
ierr = KSPGetOperators(mglevels[n-1]->smoothd,NULL,&mmat,NULL);
CHKERRQ(ierr);
if (mmat == pc->pmat) opsset = PETSC_FALSE;
}
if (!opsset) {
ierr = PCGetUseAmat(pc,&use_amat);
CHKERRQ(ierr);
if(use_amat) {
ierr = PetscInfo(pc,"Using outer operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");
CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->mat,pc->pmat,pc->flag);
CHKERRQ(ierr);
}
else {
ierr = PetscInfo(pc,"Using matrix (pmat) operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");
CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->pmat,pc->pmat,pc->flag);
CHKERRQ(ierr);
}
}
/* Skipping this for galerkin==2 (externally managed hierarchy such as ML and GAMG). Cleaner logic here would be great. Wrap ML/GAMG as DMs? */
if (pc->dm && mg->galerkin != 2 && !pc->setupcalled) {
/* construct the interpolation from the DMs */
Mat p;
Vec rscale;
ierr = PetscMalloc(n*sizeof(DM),&dms);
CHKERRQ(ierr);
dms[n-1] = pc->dm;
for (i=n-2; i>-1; i--) {
DMKSP kdm;
ierr = DMCoarsen(dms[i+1],MPI_COMM_NULL,&dms[i]);
CHKERRQ(ierr);
ierr = KSPSetDM(mglevels[i]->smoothd,dms[i]);
CHKERRQ(ierr);
if (mg->galerkin) {
ierr = KSPSetDMActive(mglevels[i]->smoothd,PETSC_FALSE);
CHKERRQ(ierr);
}
ierr = DMGetDMKSPWrite(dms[i],&kdm);
CHKERRQ(ierr);
/* Ugly hack so that the next KSPSetUp() will use the RHS that we set. A better fix is to change dmActive to take
* a bitwise OR of computing the matrix, RHS, and initial iterate. */
kdm->ops->computerhs = NULL;
kdm->rhsctx = NULL;
if (!mglevels[i+1]->interpolate) {
ierr = DMCreateInterpolation(dms[i],dms[i+1],&p,&rscale);
CHKERRQ(ierr);
ierr = PCMGSetInterpolation(pc,i+1,p);
CHKERRQ(ierr);
if (rscale) {
ierr = PCMGSetRScale(pc,i+1,rscale);
CHKERRQ(ierr);
}
ierr = VecDestroy(&rscale);
CHKERRQ(ierr);
ierr = MatDestroy(&p);
CHKERRQ(ierr);
}
}
for (i=n-2; i>-1; i--) {
ierr = DMDestroy(&dms[i]);
CHKERRQ(ierr);
}
ierr = PetscFree(dms);
CHKERRQ(ierr);
}
if (pc->dm && !pc->setupcalled) {
/* finest smoother also gets DM but it is not active, independent of whether galerkin==2 */
ierr = KSPSetDM(mglevels[n-1]->smoothd,pc->dm);
CHKERRQ(ierr);
ierr = KSPSetDMActive(mglevels[n-1]->smoothd,PETSC_FALSE);
CHKERRQ(ierr);
}
if (mg->galerkin == 1) {
Mat B;
/* currently only handle case where mat and pmat are the same on coarser levels */
ierr = KSPGetOperators(mglevels[n-1]->smoothd,&dA,&dB,&uflag);
CHKERRQ(ierr);
if (!pc->setupcalled) {
for (i=n-2; i>-1; i--) {
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);
CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B,uflag);
CHKERRQ(ierr);
if (i != n-2) {
ierr = PetscObjectDereference((PetscObject)dB);
CHKERRQ(ierr);
}
dB = B;
}
if (n > 1) {
ierr = PetscObjectDereference((PetscObject)dB);
CHKERRQ(ierr);
}
} else {
for (i=n-2; i>-1; i--) {
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&B,NULL);
CHKERRQ(ierr);
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);
CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B,uflag);
CHKERRQ(ierr);
dB = B;
}
}
} else if (!mg->galerkin && pc->dm && pc->dm->x) {
/* need to restrict Jacobian location to coarser meshes for evaluation */
for (i=n-2; i>-1; i--) {
Mat R;
Vec rscale;
if (!mglevels[i]->smoothd->dm->x) {
Vec *vecs;
ierr = KSPGetVecs(mglevels[i]->smoothd,1,&vecs,0,NULL);
CHKERRQ(ierr);
mglevels[i]->smoothd->dm->x = vecs[0];
ierr = PetscFree(vecs);
CHKERRQ(ierr);
}
ierr = PCMGGetRestriction(pc,i+1,&R);
CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);
CHKERRQ(ierr);
ierr = MatRestrict(R,mglevels[i+1]->smoothd->dm->x,mglevels[i]->smoothd->dm->x);
CHKERRQ(ierr);
ierr = VecPointwiseMult(mglevels[i]->smoothd->dm->x,mglevels[i]->smoothd->dm->x,rscale);
CHKERRQ(ierr);
}
}
if (!mg->galerkin && pc->dm) {
for (i=n-2; i>=0; i--) {
DM dmfine,dmcoarse;
Mat Restrict,Inject;
Vec rscale;
ierr = KSPGetDM(mglevels[i+1]->smoothd,&dmfine);
CHKERRQ(ierr);
ierr = KSPGetDM(mglevels[i]->smoothd,&dmcoarse);
CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i+1,&Restrict);
CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);
CHKERRQ(ierr);
Inject = NULL; /* Callback should create it if it needs Injection */
ierr = DMRestrict(dmfine,Restrict,rscale,Inject,dmcoarse);
CHKERRQ(ierr);
}
}
if (!pc->setupcalled) {
for (i=0; i<n; i++) {
ierr = KSPSetFromOptions(mglevels[i]->smoothd);
CHKERRQ(ierr);
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && (mglevels[i]->smoothu != mglevels[i]->smoothd)) {
ierr = KSPSetFromOptions(mglevels[i]->smoothu);
CHKERRQ(ierr);
}
}
for (i=1; i<n; i++) {
ierr = PCMGGetInterpolation(pc,i,&mglevels[i]->interpolate);
CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i,&mglevels[i]->restrct);
CHKERRQ(ierr);
}
for (i=0; i<n-1; i++) {
if (!mglevels[i]->b) {
Vec *vec;
ierr = KSPGetVecs(mglevels[i]->smoothd,1,&vec,0,NULL);
CHKERRQ(ierr);
ierr = PCMGSetRhs(pc,i,*vec);
CHKERRQ(ierr);
ierr = VecDestroy(vec);
CHKERRQ(ierr);
ierr = PetscFree(vec);
CHKERRQ(ierr);
}
if (!mglevels[i]->r && i) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);
CHKERRQ(ierr);
ierr = PCMGSetR(pc,i,tvec);
CHKERRQ(ierr);
ierr = VecDestroy(&tvec);
CHKERRQ(ierr);
}
if (!mglevels[i]->x) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);
CHKERRQ(ierr);
ierr = PCMGSetX(pc,i,tvec);
CHKERRQ(ierr);
ierr = VecDestroy(&tvec);
CHKERRQ(ierr);
}
}
if (n != 1 && !mglevels[n-1]->r) {
/* PCMGSetR() on the finest level if user did not supply it */
Vec *vec;
ierr = KSPGetVecs(mglevels[n-1]->smoothd,1,&vec,0,NULL);
CHKERRQ(ierr);
ierr = PCMGSetR(pc,n-1,*vec);
CHKERRQ(ierr);
ierr = VecDestroy(vec);
CHKERRQ(ierr);
ierr = PetscFree(vec);
CHKERRQ(ierr);
}
}
if (pc->dm) {
/* need to tell all the coarser levels to rebuild the matrix using the DM for that level */
for (i=0; i<n-1; i++) {
if (mglevels[i]->smoothd->setupstage != KSP_SETUP_NEW) mglevels[i]->smoothd->setupstage = KSP_SETUP_NEWMATRIX;
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu == mglevels[i]->smoothd) {
/* if doing only down then initial guess is zero */
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothd,PETSC_TRUE);
CHKERRQ(ierr);
}
if (mglevels[i]->eventsmoothsetup) {
ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);
CHKERRQ(ierr);
}
ierr = KSPSetUp(mglevels[i]->smoothd);
CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {
ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);
CHKERRQ(ierr);
}
if (!mglevels[i]->residual) {
Mat mat;
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&mat,NULL);
CHKERRQ(ierr);
ierr = PCMGSetResidual(pc,i,PCMGResidual_Default,mat);
CHKERRQ(ierr);
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && mglevels[i]->smoothu != mglevels[i]->smoothd) {
Mat downmat,downpmat;
MatStructure matflag;
/* check if operators have been set for up, if not use down operators to set them */
ierr = KSPGetOperatorsSet(mglevels[i]->smoothu,&opsset,NULL);
CHKERRQ(ierr);
if (!opsset) {
ierr = KSPGetOperators(mglevels[i]->smoothd,&downmat,&downpmat,&matflag);
CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[i]->smoothu,downmat,downpmat,matflag);
CHKERRQ(ierr);
}
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothu,PETSC_TRUE);
CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {
ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);
CHKERRQ(ierr);
}
ierr = KSPSetUp(mglevels[i]->smoothu);
CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {
ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);
CHKERRQ(ierr);
}
}
}
/*
If coarse solver is not direct method then DO NOT USE preonly
*/
ierr = PetscObjectTypeCompare((PetscObject)mglevels[0]->smoothd,KSPPREONLY,&preonly);
CHKERRQ(ierr);
if (preonly) {
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCLU,&lu);
CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCREDUNDANT,&redundant);
CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCCHOLESKY,&cholesky);
CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)cpc,PCSVD,&svd);
CHKERRQ(ierr);
if (!lu && !redundant && !cholesky && !svd) {
ierr = KSPSetType(mglevels[0]->smoothd,KSPGMRES);
CHKERRQ(ierr);
}
}
if (!pc->setupcalled) {
ierr = KSPSetFromOptions(mglevels[0]->smoothd);
CHKERRQ(ierr);
}
if (mglevels[0]->eventsmoothsetup) {
ierr = PetscLogEventBegin(mglevels[0]->eventsmoothsetup,0,0,0,0);
CHKERRQ(ierr);
}
ierr = KSPSetUp(mglevels[0]->smoothd);
CHKERRQ(ierr);
if (mglevels[0]->eventsmoothsetup) {
ierr = PetscLogEventEnd(mglevels[0]->eventsmoothsetup,0,0,0,0);
CHKERRQ(ierr);
}
/*
Dump the interpolation/restriction matrices plus the
Jacobian/stiffness on each level. This allows MATLAB users to
easily check if the Galerkin condition A_c = R A_f R^T is satisfied.
Only support one or the other at the same time.
*/
#if defined(PETSC_USE_SOCKET_VIEWER)
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_mg_dump_matlab",&dump,NULL);
CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_SOCKET_(PetscObjectComm((PetscObject)pc));
dump = PETSC_FALSE;
#endif
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_mg_dump_binary",&dump,NULL);
CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_BINARY_(PetscObjectComm((PetscObject)pc));
if (viewer) {
for (i=1; i<n; i++) {
ierr = MatView(mglevels[i]->restrct,viewer);
CHKERRQ(ierr);
}
for (i=0; i<n; i++) {
ierr = KSPGetPC(mglevels[i]->smoothd,&pc);
CHKERRQ(ierr);
ierr = MatView(pc->mat,viewer);
CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
/*@C
PCMGSetLevels - Sets the number of levels to use with MG.
Must be called before any other MG routine.
Logically Collective on PC
Input Parameters:
+ pc - the preconditioner context
. levels - the number of levels
- comms - optional communicators for each level; this is to allow solving the coarser problems
on smaller sets of processors. Use NULL_OBJECT for default in Fortran
Level: intermediate
Notes:
If the number of levels is one then the multigrid uses the -mg_levels prefix
for setting the level options rather than the -mg_coarse prefix.
.keywords: MG, set, levels, multigrid
.seealso: PCMGSetType(), PCMGGetLevels()
@*/
PetscErrorCode PCMGSetLevels(PC pc,PetscInt levels,MPI_Comm *comms)
{
PetscErrorCode ierr;
PC_MG *mg = (PC_MG*)pc->data;
MPI_Comm comm;
PC_MG_Levels **mglevels = mg->levels;
PCMGType mgtype = mg->am;
PetscInt mgctype = (PetscInt) PC_MG_CYCLE_V;
PetscInt i;
PetscMPIInt size;
const char *prefix;
PC ipc;
PetscInt n;
PetscFunctionBegin;
PetscValidHeaderSpecific(pc,PC_CLASSID,1);
PetscValidLogicalCollectiveInt(pc,levels,2);
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
if (mg->nlevels == levels) PetscFunctionReturn(0);
if (mglevels) {
mgctype = mglevels[0]->cycles;
/* changing the number of levels so free up the previous stuff */
ierr = PCReset_MG(pc);CHKERRQ(ierr);
n = mglevels[0]->levels;
for (i=0; i<n; i++) {
if (mglevels[i]->smoothd != mglevels[i]->smoothu) {
ierr = KSPDestroy(&mglevels[i]->smoothd);CHKERRQ(ierr);
}
ierr = KSPDestroy(&mglevels[i]->smoothu);CHKERRQ(ierr);
ierr = PetscFree(mglevels[i]);CHKERRQ(ierr);
}
ierr = PetscFree(mg->levels);CHKERRQ(ierr);
}
mg->nlevels = levels;
ierr = PetscMalloc1(levels,&mglevels);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)pc,levels*(sizeof(PC_MG*)));CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
mg->stageApply = 0;
for (i=0; i<levels; i++) {
ierr = PetscNewLog(pc,&mglevels[i]);CHKERRQ(ierr);
mglevels[i]->level = i;
mglevels[i]->levels = levels;
mglevels[i]->cycles = mgctype;
mg->default_smoothu = 2;
mg->default_smoothd = 2;
mglevels[i]->eventsmoothsetup = 0;
mglevels[i]->eventsmoothsolve = 0;
mglevels[i]->eventresidual = 0;
mglevels[i]->eventinterprestrict = 0;
if (comms) comm = comms[i];
ierr = KSPCreate(comm,&mglevels[i]->smoothd);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(mglevels[i]->smoothd,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)mglevels[i]->smoothd,(PetscObject)pc,levels-i);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(mglevels[i]->smoothd,prefix);CHKERRQ(ierr);
ierr = PetscObjectComposedDataSetInt((PetscObject) mglevels[i]->smoothd, PetscMGLevelId, mglevels[i]->level);CHKERRQ(ierr);
if (i || levels == 1) {
char tprefix[128];
ierr = KSPSetType(mglevels[i]->smoothd,KSPCHEBYSHEV);CHKERRQ(ierr);
ierr = KSPSetConvergenceTest(mglevels[i]->smoothd,KSPConvergedSkip,NULL,NULL);CHKERRQ(ierr);
ierr = KSPSetNormType(mglevels[i]->smoothd,KSP_NORM_NONE);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[i]->smoothd,&ipc);CHKERRQ(ierr);
ierr = PCSetType(ipc,PCSOR);CHKERRQ(ierr);
ierr = KSPSetTolerances(mglevels[i]->smoothd,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT, mg->default_smoothd);CHKERRQ(ierr);
sprintf(tprefix,"mg_levels_%d_",(int)i);
ierr = KSPAppendOptionsPrefix(mglevels[i]->smoothd,tprefix);CHKERRQ(ierr);
} else {
ierr = KSPAppendOptionsPrefix(mglevels[0]->smoothd,"mg_coarse_");CHKERRQ(ierr);
/* coarse solve is (redundant) LU by default; set shifttype NONZERO to avoid annoying zero-pivot in LU preconditioner */
ierr = KSPSetType(mglevels[0]->smoothd,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[0]->smoothd,&ipc);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size > 1) {
ierr = PCSetType(ipc,PCREDUNDANT);CHKERRQ(ierr);
} else {
ierr = PCSetType(ipc,PCLU);CHKERRQ(ierr);
}
ierr = PCFactorSetShiftType(ipc,MAT_SHIFT_INBLOCKS);CHKERRQ(ierr);
}
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)mglevels[i]->smoothd);CHKERRQ(ierr);
mglevels[i]->smoothu = mglevels[i]->smoothd;
mg->rtol = 0.0;
mg->abstol = 0.0;
mg->dtol = 0.0;
mg->ttol = 0.0;
mg->cyclesperpcapply = 1;
}
mg->levels = mglevels;
ierr = PCMGSetType(pc,mgtype);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_Redundant(PC pc)
{
PC_Redundant *red = (PC_Redundant*)pc->data;
PetscErrorCode ierr;
PetscInt mstart,mend,mlocal,M;
PetscMPIInt size;
MPI_Comm comm,subcomm;
Vec x;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
/* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size == 1) red->useparallelmat = PETSC_FALSE;
if (!pc->setupcalled) {
PetscInt mloc_sub;
if (!red->psubcomm) { /* create red->psubcomm, new ksp and pc over subcomm */
KSP ksp;
ierr = PCRedundantGetKSP(pc,&ksp);CHKERRQ(ierr);
}
subcomm = PetscSubcommChild(red->psubcomm);
if (red->useparallelmat) {
/* grab the parallel matrix and put it into processors of a subcomminicator */
ierr = MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,subcomm,MAT_INITIAL_MATRIX,&red->pmats);CHKERRQ(ierr);
ierr = MPI_Comm_size(subcomm,&size);CHKERRQ(ierr);
if (size > 1) {
PetscBool foundpack;
ierr = MatGetFactorAvailable(red->pmats,NULL,MAT_FACTOR_LU,&foundpack);CHKERRQ(ierr);
if (!foundpack) { /* reset default ksp and pc */
ierr = KSPSetType(red->ksp,KSPGMRES);CHKERRQ(ierr);
ierr = PCSetType(red->pc,PCBJACOBI);CHKERRQ(ierr);
} else {
ierr = PCFactorSetMatSolverType(red->pc,NULL);CHKERRQ(ierr);
}
}
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
/* get working vectors xsub and ysub */
ierr = MatCreateVecs(red->pmats,&red->xsub,&red->ysub);CHKERRQ(ierr);
/* create working vectors xdup and ydup.
xdup concatenates all xsub's contigously to form a mpi vector over dupcomm (see PetscSubcommCreate_interlaced())
ydup concatenates all ysub and has empty local arrays because ysub's arrays will be place into it.
Note: we use communicator dupcomm, not PetscObjectComm((PetscObject)pc)! */
ierr = MatGetLocalSize(red->pmats,&mloc_sub,NULL);CHKERRQ(ierr);
ierr = VecCreateMPI(PetscSubcommContiguousParent(red->psubcomm),mloc_sub,PETSC_DECIDE,&red->xdup);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PetscSubcommContiguousParent(red->psubcomm),1,mloc_sub,PETSC_DECIDE,NULL,&red->ydup);CHKERRQ(ierr);
/* create vecscatters */
if (!red->scatterin) { /* efficiency of scatterin is independent from psubcomm_type! */
IS is1,is2;
PetscInt *idx1,*idx2,i,j,k;
ierr = MatCreateVecs(pc->pmat,&x,0);CHKERRQ(ierr);
ierr = VecGetSize(x,&M);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&mstart,&mend);CHKERRQ(ierr);
mlocal = mend - mstart;
ierr = PetscMalloc2(red->psubcomm->n*mlocal,&idx1,red->psubcomm->n*mlocal,&idx2);CHKERRQ(ierr);
j = 0;
for (k=0; k<red->psubcomm->n; k++) {
for (i=mstart; i<mend; i++) {
idx1[j] = i;
idx2[j++] = i + M*k;
}
}
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = VecScatterCreateWithData(x,is1,red->xdup,is2,&red->scatterin);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
/* Impl below is good for PETSC_SUBCOMM_INTERLACED (no inter-process communication) and PETSC_SUBCOMM_CONTIGUOUS (communication within subcomm) */
ierr = ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*M,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart,1,&is2);CHKERRQ(ierr);
ierr = VecScatterCreateWithData(red->xdup,is1,x,is2,&red->scatterout);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFree2(idx1,idx2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
} else { /* pc->setupcalled */
if (red->useparallelmat) {
MatReuse reuse;
/* grab the parallel matrix and put it into processors of a subcomminicator */
/*--------------------------------------------------------------------------*/
if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
/* destroy old matrices */
ierr = MatDestroy(&red->pmats);CHKERRQ(ierr);
reuse = MAT_INITIAL_MATRIX;
} else {
reuse = MAT_REUSE_MATRIX;
}
ierr = MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,PetscSubcommChild(red->psubcomm),reuse,&red->pmats);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
}
if (pc->setfromoptionscalled) {
ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
}
ierr = KSPSetUp(red->ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETScMGSolver_UpdateSolvers( PETScMGSolver* self ) {
PETScMGSolver_Level* level;
PC pc;
KSP levelKSP;
PC levelPC;
PetscErrorCode ec;
unsigned l_i;
PetscTruth smoothers_differ, flag;
PetscMPIInt size;
MPI_Comm comm;
assert( self && Stg_CheckType( self, PETScMGSolver ) );
ec = KSPGetPC( self->mgData->ksp, &pc );
CheckPETScError( ec );
ec = PCMGSetLevels( pc, self->nLevels, PETSC_NULL );
CheckPETScError( ec );
ec = PCMGSetType( pc, PC_MG_MULTIPLICATIVE );
CheckPETScError( ec );
ec=PetscOptionsGetTruth( PETSC_NULL, "-pc_mg_different_smoothers", &smoothers_differ, &flag ); CheckPETScError(ec);
ec=PetscObjectGetComm( (PetscObject)pc, &comm ); CheckPETScError(ec);
MPI_Comm_size( comm, &size );
for( l_i = 1; l_i < self->nLevels; l_i++ ) {
level = self->levels + l_i;
printf("Configuring MG level %d \n", l_i );
ec = PCMGGetSmootherDown( pc, l_i, &levelKSP );
CheckPETScError( ec );
if(smoothers_differ==PETSC_TRUE) { ec=KSPAppendOptionsPrefix( levelKSP, "down_" ); CheckPETScError(ec); }
ec = KSPSetType( levelKSP, KSPRICHARDSON ); CheckPETScError( ec );
ec = KSPGetPC( levelKSP, &levelPC ); CheckPETScError( ec );
if(size==1) {
ec = PCSetType( levelPC, PCSOR ); CheckPETScError( ec );
}
/* This does not work - bug with the order the operators are created I guess */
/* For parallel jobs you best bet is to use the command line args and let petsc work it out */
/*
else {
KSP *sub_ksp;
PetscInt k, n_local, first_local;
PC sub_pc;
PCSetType( levelPC, PCBJACOBI );
KSPSetUp( levelKSP );
PCBJacobiGetSubKSP( levelPC, &n_local,&first_local,&sub_ksp);
for(k=0;k<n_local;k++ ) {
KSPSetType( sub_ksp[k], KSPFGMRES );
KSPGetPC( sub_ksp[k], &sub_pc );
PCSetType( sub_pc, PCSOR );
}
}
*/
ec = KSPSetTolerances( levelKSP, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT, level->nDownIts ); CheckPETScError( ec );
if( l_i == self->nLevels - 1 ) {
ec = KSPSetInitialGuessNonzero( levelKSP, PETSC_TRUE ); CheckPETScError( ec );
}
else { ec = KSPSetInitialGuessNonzero( levelKSP, PETSC_FALSE ); CheckPETScError( ec ); }
ec = PCMGGetSmootherUp( pc, l_i, &levelKSP ); CheckPETScError( ec );
if(smoothers_differ==PETSC_TRUE) { ec=KSPAppendOptionsPrefix( levelKSP, "up_" ); CheckPETScError(ec); }
ec = KSPSetType( levelKSP, KSPRICHARDSON ); CheckPETScError( ec );
ec = KSPGetPC( levelKSP, &levelPC ); CheckPETScError( ec );
if(size==1) {
ec = PCSetType( levelPC, PCSOR ); CheckPETScError( ec );
}
ec = KSPSetTolerances( levelKSP, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT, level->nUpIts ); CheckPETScError( ec );
ec = KSPSetInitialGuessNonzero( levelKSP, PETSC_TRUE ); CheckPETScError( ec );
ec = PCMGSetCyclesOnLevel( pc, l_i, level->nCycles ); CheckPETScError( ec );
}
}
void
SolverLinearPetsc<T>::setPetscSolverType()
{
int ierr = 0;
DVLOG(2) << "[SolverLinearPetsc] solver type: " << this->solverType() << "\n";
switch ( this->solverType() )
{
case CG:
ierr = KSPSetType ( M_ksp, ( char* ) KSPCG );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case CR:
ierr = KSPSetType ( M_ksp, ( char* ) KSPCR );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case CGS:
ierr = KSPSetType ( M_ksp, ( char* ) KSPCGS );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case BICG:
ierr = KSPSetType ( M_ksp, ( char* ) KSPBICG );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case TCQMR:
ierr = KSPSetType ( M_ksp, ( char* ) KSPTCQMR );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case TFQMR:
ierr = KSPSetType ( M_ksp, ( char* ) KSPTFQMR );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case LSQR:
ierr = KSPSetType ( M_ksp, ( char* ) KSPLSQR );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case BICGSTAB:
ierr = KSPSetType ( M_ksp, ( char* ) KSPBCGS );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case MINRES:
ierr = KSPSetType ( M_ksp, ( char* ) KSPMINRES );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case GMRES:
ierr = KSPSetType ( M_ksp, ( char* ) KSPGMRES );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case FGMRES:
ierr = KSPSetType ( M_ksp, ( char* ) KSPFGMRES );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case RICHARDSON:
ierr = KSPSetType ( M_ksp, ( char* ) KSPRICHARDSON );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case CHEBYSHEV:
#if PETSC_VERSION_LESS_THAN(3,3,0)
ierr = KSPSetType ( M_ksp, ( char* ) KSPCHEBYCHEV );
#else
ierr = KSPSetType ( M_ksp, ( char* ) KSPCHEBYSHEV );
#endif
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case PREONLY :
ierr = KSPSetType ( M_ksp, ( char* ) KSPPREONLY );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
default:
std::cerr << "ERROR: Unsupported PETSC Solver: "
<< this->solverType() << std::endl
<< "Continuing with PETSC defaults" << std::endl;
}
}
