void PETScKrylovLinearSolver::initializeSolverState(const SAMRAIVectorReal<NDIM, double>& x, const SAMRAIVectorReal<NDIM, double>& b) { IBTK_TIMER_START(t_initialize_solver_state); int ierr; // Rudimentary error checking. #if !defined(NDEBUG) if (x.getNumberOfComponents() != b.getNumberOfComponents()) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " vectors must have the same number of components" << std::endl); } const Pointer<PatchHierarchy<NDIM> >& patch_hierarchy = x.getPatchHierarchy(); if (patch_hierarchy != b.getPatchHierarchy()) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " vectors must have the same hierarchy" << std::endl); } const int coarsest_ln = x.getCoarsestLevelNumber(); if (coarsest_ln < 0) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " coarsest level number must not be negative" << std::endl); } if (coarsest_ln != b.getCoarsestLevelNumber()) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " vectors must have same coarsest level number" << std::endl); } const int finest_ln = x.getFinestLevelNumber(); if (finest_ln < coarsest_ln) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " finest level number must be >= coarsest level number" << std::endl); } if (finest_ln != b.getFinestLevelNumber()) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " vectors must have same finest level number" << std::endl); } for (int ln = coarsest_ln; ln <= finest_ln; ++ln) { if (!patch_hierarchy->getPatchLevel(ln)) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " hierarchy level " << ln << " does not exist" << std::endl); } } #endif // Deallocate the solver state if the solver is already initialized. if (d_is_initialized) { d_reinitializing_solver = true; deallocateSolverState(); } // Create the KSP solver. if (d_managing_petsc_ksp) { ierr = KSPCreate(d_petsc_comm, &d_petsc_ksp); IBTK_CHKERRQ(ierr); resetKSPOptions(); } else if (!d_petsc_ksp) { TBOX_ERROR(d_object_name << "::initializeSolverState()\n" << " cannot initialize solver state for wrapped PETSc KSP object " "if the wrapped object is NULL" << std::endl); } // Setup solution and rhs vectors. d_x = x.cloneVector(x.getName()); d_petsc_x = PETScSAMRAIVectorReal::createPETScVector(d_x, d_petsc_comm); d_b = b.cloneVector(b.getName()); d_petsc_b = PETScSAMRAIVectorReal::createPETScVector(d_b, d_petsc_comm); // Initialize the linear operator and preconditioner objects. if (d_A) d_A->initializeOperatorState(*d_x, *d_b); if (d_managing_petsc_ksp || d_user_provided_mat) resetKSPOperators(); if (d_pc_solver) d_pc_solver->initializeSolverState(*d_x, *d_b); if (d_managing_petsc_ksp || d_user_provided_pc) resetKSPPC(); // Set the KSP options from the PETSc options database. if (d_options_prefix != "") { ierr = KSPSetOptionsPrefix(d_petsc_ksp, d_options_prefix.c_str()); IBTK_CHKERRQ(ierr); } ierr = KSPSetFromOptions(d_petsc_ksp); IBTK_CHKERRQ(ierr); // Reset the member state variables to correspond to the values used by the // KSP object. (Command-line options always take precedence.) const char* ksp_type; ierr = KSPGetType(d_petsc_ksp, &ksp_type); IBTK_CHKERRQ(ierr); d_ksp_type = ksp_type; PetscBool initial_guess_nonzero; ierr = KSPGetInitialGuessNonzero(d_petsc_ksp, &initial_guess_nonzero); IBTK_CHKERRQ(ierr); d_initial_guess_nonzero = (initial_guess_nonzero == PETSC_TRUE); ierr = KSPGetTolerances(d_petsc_ksp, &d_rel_residual_tol, &d_abs_residual_tol, NULL, &d_max_iterations); IBTK_CHKERRQ(ierr); // Configure the nullspace object. resetMatNullspace(); // Indicate that the solver is initialized. d_reinitializing_solver = false; d_is_initialized = true; IBTK_TIMER_STOP(t_initialize_solver_state); return; } // initializeSolverState
PetscErrorCode petscConverged(KSP ksp, PetscInt n, PetscReal rnorm, KSPConvergedReason * reason, void * ctx) { // Cast the context pointer coming from PETSc to an FEProblem& and // get a reference to the System from it. FEProblem & problem = *static_cast<FEProblem *>(ctx); // Let's be nice and always check PETSc error codes. PetscErrorCode ierr = 0; // We want the default behavior of the KSPDefaultConverged test, but // we don't want PETSc to die in that function with a CHKERRQ // call... that is probably extremely unlikely/impossible, but just // to be on the safe side, we push a different error handler before // calling KSPDefaultConverged(). ierr = PetscPushErrorHandler(PetscReturnErrorHandler, /*void* ctx=*/ PETSC_NULL); CHKERRABORT(problem.comm().get(),ierr); #if PETSC_VERSION_LESS_THAN(3,0,0) // Prior to PETSc 3.0.0, you could call KSPDefaultConverged with a NULL context // pointer, as it was unused. KSPDefaultConverged(ksp, n, rnorm, reason, PETSC_NULL); #elif PETSC_RELEASE_LESS_THAN(3,5,0) // As of PETSc 3.0.0, you must call KSPDefaultConverged with a // non-NULL context pointer which must be created with // KSPDefaultConvergedCreate(), and destroyed with // KSPDefaultConvergedDestroy(). void* default_ctx = NULL; KSPDefaultConvergedCreate(&default_ctx); KSPDefaultConverged(ksp, n, rnorm, reason, default_ctx); KSPDefaultConvergedDestroy(default_ctx); #else // As of PETSc 3.5.0, use KSPConvergedDefaultXXX void* default_ctx = NULL; KSPConvergedDefaultCreate(&default_ctx); KSPConvergedDefault(ksp, n, rnorm, reason, default_ctx); KSPConvergedDefaultDestroy(default_ctx); #endif // Pop the Error handler we pushed on the stack to go back // to default PETSc error handling behavior. ierr = PetscPopErrorHandler(); CHKERRABORT(problem.comm().get(),ierr); // Get tolerances from the KSP object PetscReal rtol = 0.; PetscReal atol = 0.; PetscReal dtol = 0.; PetscInt maxits = 0; ierr = KSPGetTolerances(ksp, &rtol, &atol, &dtol, &maxits); CHKERRABORT(problem.comm().get(),ierr); // Now do some additional MOOSE-specific tests... std::string msg; MooseLinearConvergenceReason moose_reason = problem.checkLinearConvergence(msg, n, rnorm, rtol, atol, dtol, maxits); switch (moose_reason) { case MOOSE_CONVERGED_RTOL: *reason = KSP_CONVERGED_RTOL; break; case MOOSE_CONVERGED_ITS: *reason = KSP_CONVERGED_ITS; break; case MOOSE_DIVERGED_NANORINF: #if PETSC_VERSION_LESS_THAN(3,4,0) // Report divergence due to exceeding the divergence tolerance. *reason = KSP_DIVERGED_DTOL; #else // KSP_DIVERGED_NANORINF was added in PETSc 3.4.0. *reason = KSP_DIVERGED_NANORINF; #endif break; default: { // If it's not either of the two specific cases we handle, just go // with whatever PETSc decided in KSPDefaultConverged. break; } } return 0; }
static PetscErrorCode KSPSolve_BCGSL(KSP ksp) { KSP_BCGSL *bcgsl = (KSP_BCGSL *) ksp->data; PetscScalar alpha, beta, omega, sigma; PetscScalar rho0, rho1; PetscReal kappa0, kappaA, kappa1; PetscReal ghat, epsilon, abstol; PetscReal zeta, zeta0, rnmax_computed, rnmax_true, nrm0; PetscTruth bUpdateX; PetscTruth bBombed = PETSC_FALSE; PetscInt maxit; PetscInt h, i, j, k, vi, ell; PetscBLASInt ldMZ,bierr; PetscErrorCode ierr; PetscFunctionBegin; if (ksp->normtype == KSP_NORM_NATURAL) SETERRQ(PETSC_ERR_SUP,"Cannot use natural norm with KSPBCGSL"); if (ksp->normtype == KSP_NORM_PRECONDITIONED && ksp->pc_side != PC_LEFT) SETERRQ(PETSC_ERR_SUP,"Use -ksp_norm_type unpreconditioned for right preconditioning and KSPBCGSL"); if (ksp->normtype == KSP_NORM_UNPRECONDITIONED && ksp->pc_side != PC_RIGHT) SETERRQ(PETSC_ERR_SUP,"Use -ksp_norm_type preconditioned for left preconditioning and KSPBCGSL"); /* set up temporary vectors */ vi = 0; ell = bcgsl->ell; bcgsl->vB = ksp->work[vi]; vi++; bcgsl->vRt = ksp->work[vi]; vi++; bcgsl->vTm = ksp->work[vi]; vi++; bcgsl->vvR = ksp->work+vi; vi += ell+1; bcgsl->vvU = ksp->work+vi; vi += ell+1; bcgsl->vXr = ksp->work[vi]; vi++; ldMZ = PetscBLASIntCast(ell+1); /* Prime the iterative solver */ ierr = KSPInitialResidual(ksp, VX, VTM, VB, VVR[0], ksp->vec_rhs); CHKERRQ(ierr); ierr = VecNorm(VVR[0], NORM_2, &zeta0); CHKERRQ(ierr); rnmax_computed = zeta0; rnmax_true = zeta0; ierr = (*ksp->converged)(ksp, 0, zeta0, &ksp->reason, ksp->cnvP); CHKERRQ(ierr); if (ksp->reason) { ierr = PetscObjectTakeAccess(ksp); CHKERRQ(ierr); ksp->its = 0; ksp->rnorm = zeta0; ierr = PetscObjectGrantAccess(ksp); CHKERRQ(ierr); PetscFunctionReturn(0); } ierr = VecSet(VVU[0],0.0); CHKERRQ(ierr); alpha = 0.; rho0 = omega = 1; if (bcgsl->delta>0.0) { ierr = VecCopy(VX, VXR); CHKERRQ(ierr); ierr = VecSet(VX,0.0); CHKERRQ(ierr); ierr = VecCopy(VVR[0], VB); CHKERRQ(ierr); } else { ierr = VecCopy(ksp->vec_rhs, VB); CHKERRQ(ierr); } /* Life goes on */ ierr = VecCopy(VVR[0], VRT); CHKERRQ(ierr); zeta = zeta0; ierr = KSPGetTolerances(ksp, &epsilon, &abstol, PETSC_NULL, &maxit); CHKERRQ(ierr); for (k=0; k<maxit; k += bcgsl->ell) { ksp->its = k; ksp->rnorm = zeta; KSPLogResidualHistory(ksp, zeta); KSPMonitor(ksp, ksp->its, zeta); ierr = (*ksp->converged)(ksp, k, zeta, &ksp->reason, ksp->cnvP); CHKERRQ(ierr); if (ksp->reason) break; /* BiCG part */ rho0 = -omega*rho0; nrm0 = zeta; for (j=0; j<bcgsl->ell; j++) { /* rho1 <- r_j' * r_tilde */ ierr = VecDot(VVR[j], VRT, &rho1); CHKERRQ(ierr); if (rho1 == 0.0) { ksp->reason = KSP_DIVERGED_BREAKDOWN_BICG; bBombed = PETSC_TRUE; break; } beta = alpha*(rho1/rho0); rho0 = rho1; for (i=0; i<=j; i++) { /* u_i <- r_i - beta*u_i */ ierr = VecAYPX(VVU[i], -beta, VVR[i]); CHKERRQ(ierr); } /* u_{j+1} <- inv(K)*A*u_j */ ierr = KSP_PCApplyBAorAB(ksp, VVU[j], VVU[j+1], VTM); CHKERRQ(ierr); ierr = VecDot(VVU[j+1], VRT, &sigma); CHKERRQ(ierr); if (sigma == 0.0) { ksp->reason = KSP_DIVERGED_BREAKDOWN_BICG; bBombed = PETSC_TRUE; break; } alpha = rho1/sigma; /* x <- x + alpha*u_0 */ ierr = VecAXPY(VX, alpha, VVU[0]); CHKERRQ(ierr); for (i=0; i<=j; i++) { /* r_i <- r_i - alpha*u_{i+1} */ ierr = VecAXPY(VVR[i], -alpha, VVU[i+1]); CHKERRQ(ierr); } /* r_{j+1} <- inv(K)*A*r_j */ ierr = KSP_PCApplyBAorAB(ksp, VVR[j], VVR[j+1], VTM); CHKERRQ(ierr); ierr = VecNorm(VVR[0], NORM_2, &nrm0); CHKERRQ(ierr); if (bcgsl->delta>0.0) { if (rnmax_computed<nrm0) rnmax_computed = nrm0; if (rnmax_true<nrm0) rnmax_true = nrm0; } /* NEW: check for early exit */ ierr = (*ksp->converged)(ksp, k+j, nrm0, &ksp->reason, ksp->cnvP); CHKERRQ(ierr); if (ksp->reason) { ierr = PetscObjectTakeAccess(ksp); CHKERRQ(ierr); ksp->its = k+j; ksp->rnorm = nrm0; ierr = PetscObjectGrantAccess(ksp); CHKERRQ(ierr); break; } } if (bBombed==PETSC_TRUE) break; /* Polynomial part */ for(i = 0; i <= bcgsl->ell; ++i) { ierr = VecMDot(VVR[i], i+1, VVR, &MZa[i*ldMZ]); CHKERRQ(ierr); } /* Symmetrize MZa */ for(i = 0; i <= bcgsl->ell; ++i) { for(j = i+1; j <= bcgsl->ell; ++j) { MZa[i*ldMZ+j] = MZa[j*ldMZ+i] = PetscConj(MZa[j*ldMZ+i]); } } /* Copy MZa to MZb */ ierr = PetscMemcpy(MZb,MZa,ldMZ*ldMZ*sizeof(PetscScalar)); CHKERRQ(ierr); if (!bcgsl->bConvex || bcgsl->ell==1) { PetscBLASInt ione = 1,bell = PetscBLASIntCast(bcgsl->ell); AY0c[0] = -1; LAPACKpotrf_("Lower", &bell, &MZa[1+ldMZ], &ldMZ, &bierr); if (ierr!=0) { ksp->reason = KSP_DIVERGED_BREAKDOWN; bBombed = PETSC_TRUE; break; } ierr = PetscMemcpy(&AY0c[1],&MZb[1],bcgsl->ell*sizeof(PetscScalar)); CHKERRQ(ierr); LAPACKpotrs_("Lower", &bell, &ione, &MZa[1+ldMZ], &ldMZ, &AY0c[1], &ldMZ, &bierr); } else { PetscBLASInt ione = 1; PetscScalar aone = 1.0, azero = 0.0; PetscBLASInt neqs = PetscBLASIntCast(bcgsl->ell-1); LAPACKpotrf_("Lower", &neqs, &MZa[1+ldMZ], &ldMZ, &bierr); if (ierr!=0) { ksp->reason = KSP_DIVERGED_BREAKDOWN; bBombed = PETSC_TRUE; break; } ierr = PetscMemcpy(&AY0c[1],&MZb[1],(bcgsl->ell-1)*sizeof(PetscScalar)); CHKERRQ(ierr); LAPACKpotrs_("Lower", &neqs, &ione, &MZa[1+ldMZ], &ldMZ, &AY0c[1], &ldMZ, &bierr); AY0c[0] = -1; AY0c[bcgsl->ell] = 0.; ierr = PetscMemcpy(&AYlc[1],&MZb[1+ldMZ*(bcgsl->ell)],(bcgsl->ell-1)*sizeof(PetscScalar)); CHKERRQ(ierr); LAPACKpotrs_("Lower", &neqs, &ione, &MZa[1+ldMZ], &ldMZ, &AYlc[1], &ldMZ, &bierr); AYlc[0] = 0.; AYlc[bcgsl->ell] = -1; BLASgemv_("NoTr", &ldMZ, &ldMZ, &aone, MZb, &ldMZ, AY0c, &ione, &azero, AYtc, &ione); kappa0 = BLASdot_(&ldMZ, AY0c, &ione, AYtc, &ione); /* round-off can cause negative kappa's */ if (kappa0<0) kappa0 = -kappa0; kappa0 = sqrt(kappa0); kappaA = BLASdot_(&ldMZ, AYlc, &ione, AYtc, &ione); BLASgemv_("noTr", &ldMZ, &ldMZ, &aone, MZb, &ldMZ, AYlc, &ione, &azero, AYtc, &ione); kappa1 = BLASdot_(&ldMZ, AYlc, &ione, AYtc, &ione); if (kappa1<0) kappa1 = -kappa1; kappa1 = sqrt(kappa1); if (kappa0!=0.0 && kappa1!=0.0) { if (kappaA<0.7*kappa0*kappa1) { ghat = (kappaA<0.0) ? -0.7*kappa0/kappa1 : 0.7*kappa0/kappa1; } else { ghat = kappaA/(kappa1*kappa1); } for (i=0; i<=bcgsl->ell; i++) { AY0c[i] = AY0c[i] - ghat* AYlc[i]; } } } omega = AY0c[bcgsl->ell]; for (h=bcgsl->ell; h>0 && omega==0.0; h--) { omega = AY0c[h]; } if (omega==0.0) { ksp->reason = KSP_DIVERGED_BREAKDOWN; break; } ierr = VecMAXPY(VX, bcgsl->ell,AY0c+1, VVR); CHKERRQ(ierr); for (i=1; i<=bcgsl->ell; i++) { AY0c[i] *= -1.0; } ierr = VecMAXPY(VVU[0], bcgsl->ell,AY0c+1, VVU+1); CHKERRQ(ierr); ierr = VecMAXPY(VVR[0], bcgsl->ell,AY0c+1, VVR+1); CHKERRQ(ierr); for (i=1; i<=bcgsl->ell; i++) { AY0c[i] *= -1.0; } ierr = VecNorm(VVR[0], NORM_2, &zeta); CHKERRQ(ierr); /* Accurate Update */ if (bcgsl->delta>0.0) { if (rnmax_computed<zeta) rnmax_computed = zeta; if (rnmax_true<zeta) rnmax_true = zeta; bUpdateX = (PetscTruth) (zeta<bcgsl->delta*zeta0 && zeta0<=rnmax_computed); if ((zeta<bcgsl->delta*rnmax_true && zeta0<=rnmax_true) || bUpdateX) { /* r0 <- b-inv(K)*A*X */ ierr = KSP_PCApplyBAorAB(ksp, VX, VVR[0], VTM); CHKERRQ(ierr); ierr = VecAYPX(VVR[0], -1.0, VB); CHKERRQ(ierr); rnmax_true = zeta; if (bUpdateX) { ierr = VecAXPY(VXR,1.0,VX); CHKERRQ(ierr); ierr = VecSet(VX,0.0); CHKERRQ(ierr); ierr = VecCopy(VVR[0], VB); CHKERRQ(ierr); rnmax_computed = zeta; } } } } if (bcgsl->delta>0.0) { ierr = VecAXPY(VX,1.0,VXR); CHKERRQ(ierr); } ierr = (*ksp->converged)(ksp, k, zeta, &ksp->reason, ksp->cnvP); CHKERRQ(ierr); if (!ksp->reason) ksp->reason = KSP_DIVERGED_ITS; PetscFunctionReturn(0); }