/**
My handrolled SNES monitor. Gives some information about KSP convergence as well
as looking pretty.
@param snes Petsc nonlinear context
@param its number of iterations
@param norm norm of nonlinear residual
@param dctx application context
*/
PetscErrorCode MonitorFunction(SNES snes, PetscInt its, double norm, void *dctx)
{
PetscErrorCode ierr;
PetscInt lits;
PetscMPIInt rank;
KSP ksp;
KSPConvergedReason kspreason;
PetscReal kspnorm;
PetscFunctionBegin;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = SNESGetKSP(snes, &ksp);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(ksp, &kspreason);CHKERRQ(ierr);
ierr = KSPGetResidualNorm(ksp, &kspnorm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp, &lits);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, " %d SNES norm %e, %d KSP its last norm %e",
its, norm, lits, kspnorm);CHKERRQ(ierr);
if (kspreason < 0) {
ierr = PetscPrintf(PETSC_COMM_WORLD, ", KSP failed: %s", KSPConvergedReasons[kspreason]);CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_WORLD, ".\n");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
double _GetResidualNorm( MGSolver_PETScData* mgData ) {
PC pc;
const KSPType kspType;
const PCType pcType;
PetscScalar rnorm;
PetscErrorCode ec;
ec = KSPGetType( mgData->ksp, &kspType );
CheckPETScError( ec );
ec = KSPGetPC( mgData->ksp, &pc );
CheckPETScError( ec );
ec = PCGetType( pc, &pcType );
CheckPETScError( ec );
if( !strcmp( kspType, KSPRICHARDSON ) && !strcmp( pcType, PCSOR ) ) {
Vec residual;
//residual = MatrixSolver_GetResidual( mgData );
//rnorm = (PetscScalar)Vector_L2Norm( residual );
residual = _GetResidual( mgData );
VecNorm( residual, NORM_2, &rnorm );
}
else {
ec = KSPGetResidualNorm( mgData->ksp, &rnorm );
CheckPETScError( ec );
}
return (double)rnorm;
}
/// Solve again w/ the specified RHS, put result in specified vector (specialized)
void p_resolveImpl(const VectorType& b, VectorType& x) const
{
PetscErrorCode ierr(0);
int me(this->processor_rank());
try {
const Vec *bvec(PETScVector(b));
Vec *xvec(PETScVector(x));
ierr = KSPSolve(p_KSP, *bvec, *xvec); CHKERRXX(ierr);
int its;
KSPConvergedReason reason;
PetscReal rnorm;
ierr = KSPGetIterationNumber(p_KSP, &its); CHKERRXX(ierr);
ierr = KSPGetConvergedReason(p_KSP, &reason); CHKERRXX(ierr);
ierr = KSPGetResidualNorm(p_KSP, &rnorm); CHKERRXX(ierr);
std::string msg;
if (reason < 0) {
msg =
boost::str(boost::format("%d: PETSc KSP diverged after %d iterations, reason: %d") %
me % its % reason);
throw Exception(msg);
} else if (me == 0) {
msg =
boost::str(boost::format("%d: PETSc KSP converged after %d iterations, reason: %d") %
me % its % reason);
std::cerr << msg << std::endl;
}
} catch (const PETSC_EXCEPTION_TYPE& e) {
throw PETScException(ierr, e);
} catch (const Exception& e) {
throw e;
}
}
bool
PETScKrylovLinearSolver::solveSystem(SAMRAIVectorReal<NDIM, double>& x, SAMRAIVectorReal<NDIM, double>& b)
{
IBTK_TIMER_START(t_solve_system);
#if !defined(NDEBUG)
TBOX_ASSERT(d_A);
#endif
int ierr;
// Initialize the solver, when necessary.
const bool deallocate_after_solve = !d_is_initialized;
if (deallocate_after_solve) initializeSolverState(x, b);
#if !defined(NDEBUG)
TBOX_ASSERT(d_petsc_ksp);
#endif
resetKSPOptions();
// Allocate scratch data.
d_b->allocateVectorData();
// Solve the system using a PETSc KSP object.
d_b->copyVector(Pointer<SAMRAIVectorReal<NDIM, double> >(&b, false));
d_A->setHomogeneousBc(d_homogeneous_bc);
d_A->modifyRhsForBcs(*d_b);
d_A->setHomogeneousBc(true);
PETScSAMRAIVectorReal::replaceSAMRAIVector(d_petsc_x, Pointer<SAMRAIVectorReal<NDIM, double> >(&x, false));
PETScSAMRAIVectorReal::replaceSAMRAIVector(d_petsc_b, d_b);
ierr = KSPSolve(d_petsc_ksp, d_petsc_b, d_petsc_x);
IBTK_CHKERRQ(ierr);
d_A->setHomogeneousBc(d_homogeneous_bc);
d_A->imposeSolBcs(x);
// Get iterations count and residual norm.
ierr = KSPGetIterationNumber(d_petsc_ksp, &d_current_iterations);
IBTK_CHKERRQ(ierr);
ierr = KSPGetResidualNorm(d_petsc_ksp, &d_current_residual_norm);
IBTK_CHKERRQ(ierr);
d_A->setHomogeneousBc(d_homogeneous_bc);
// Determine the convergence reason.
KSPConvergedReason reason;
ierr = KSPGetConvergedReason(d_petsc_ksp, &reason);
IBTK_CHKERRQ(ierr);
const bool converged = (static_cast<int>(reason) > 0);
if (d_enable_logging) reportKSPConvergedReason(reason, plog);
// Dealocate scratch data.
d_b->deallocateVectorData();
// Deallocate the solver, when necessary.
if (deallocate_after_solve) deallocateSolverState();
IBTK_TIMER_STOP(t_solve_system);
return converged;
} // solveSystem
PetscErrorCode BSSCR_PCBFBTSubKSPMonitor( KSP ksp, PetscInt index, PetscLogDouble time )
{
PetscInt max_it;
PetscReal rnorm;
KSPConvergedReason reason;
KSPGetIterationNumber( ksp, &max_it );
KSPGetResidualNorm( ksp, &rnorm );
KSPGetConvergedReason( ksp, &reason );
PetscPrintf(((PetscObject)ksp)->comm," PCBFBTSubKSP (%d): %D Residual norm; r0 %12.12e, r %12.12e: Reason %s: Time %5.5e \n",
index, max_it, ksp->rnorm0, rnorm, KSPConvergedReasons[reason], time );
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_Lp_monitor( KSP ksp, PetscInt index )
{
PetscInt max_it;
PetscReal rnorm;
KSPConvergedReason reason;
KSPGetIterationNumber( ksp, &max_it );
KSPGetResidualNorm( ksp, &rnorm );
KSPGetConvergedReason( ksp, &reason );
if (ksp->reason > 0) {
PetscPrintf(((PetscObject)ksp)->comm,"\t<Lp(%d)>: Linear solve converged. its.=%.4d ; |r|=%5.5e ; Reason=%s\n",
index, max_it, rnorm, KSPConvergedReasons[reason] );
} else {
PetscPrintf(((PetscObject)ksp)->comm,"\t<Lp(%d)>: Linear solve did not converge. its.=%.4d ; |r|=%5.5e ; Reason=%s\n",
index, max_it, rnorm, KSPConvergedReasons[reason]);
}
PetscFunctionReturn(0);
}
void PETSc::GetFinalRelativeResidualNorm(double *rel_resid_norm)
{
KSPGetResidualNorm(ksp,rel_resid_norm);
}
PetscErrorCode KSPSolve_TSIRM(KSP ksp)
{
PetscErrorCode ierr;
KSP_TSIRM *tsirm = (KSP_TSIRM*)ksp->data;
KSP sub_ksp;
PC pc;
Mat AS;
Vec x,b;
PetscScalar *array;
PetscReal norm = 20;
PetscInt i,*ind_row,first_iteration = 1,its = 0,total = 0,col = 0;
PetscInt restart = 30;
KSP ksp_min; /* KSP for minimization */
PC pc_min; /* PC for minimization */
PetscFunctionBegin;
x = ksp->vec_sol; /* Solution vector */
b = ksp->vec_rhs; /* Right-hand side vector */
/* Row indexes (these indexes are global) */
ierr = PetscMalloc1(tsirm->Iend-tsirm->Istart,&ind_row);
CHKERRQ(ierr);
for (i=0; i<tsirm->Iend-tsirm->Istart; i++) ind_row[i] = i+tsirm->Istart;
/* Inner solver */
ierr = KSPGetPC(ksp,&pc);
CHKERRQ(ierr);
ierr = PCKSPGetKSP(pc,&sub_ksp);
CHKERRQ(ierr);
ierr = KSPSetTolerances(sub_ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,restart);
CHKERRQ(ierr);
/* previously it seemed good but with SNES it seems not good... */
ierr = KSP_MatMult(sub_ksp,tsirm->A,x,tsirm->r);
CHKERRQ(ierr);
ierr = VecAXPY(tsirm->r,-1,b);
CHKERRQ(ierr);
ierr = VecNorm(tsirm->r,NORM_2,&norm);
CHKERRQ(ierr);
ksp->its = 0;
ierr = KSPConvergedDefault(ksp,ksp->its,norm,&ksp->reason,ksp->cnvP);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(sub_ksp,PETSC_TRUE);
CHKERRQ(ierr);
do {
for (col=0; col<tsirm->size_ls && ksp->reason==0; col++) {
/* Solve (inner iteration) */
ierr = KSPSolve(sub_ksp,b,x);
CHKERRQ(ierr);
ierr = KSPGetIterationNumber(sub_ksp,&its);
CHKERRQ(ierr);
total += its;
/* Build S^T */
ierr = VecGetArray(x,&array);
CHKERRQ(ierr);
ierr = MatSetValues(tsirm->S,tsirm->Iend-tsirm->Istart,ind_row,1,&col,array,INSERT_VALUES);
CHKERRQ(ierr);
ierr = VecRestoreArray(x,&array);
CHKERRQ(ierr);
ierr = KSPGetResidualNorm(sub_ksp,&norm);
CHKERRQ(ierr);
ksp->rnorm = norm;
ksp->its ++;
ierr = KSPConvergedDefault(ksp,ksp->its,norm,&ksp->reason,ksp->cnvP);
CHKERRQ(ierr);
ierr = KSPMonitor(ksp,ksp->its,norm);
CHKERRQ(ierr);
}
/* Minimization step */
if (!ksp->reason) {
ierr = MatAssemblyBegin(tsirm->S,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(tsirm->S,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
if (first_iteration) {
ierr = MatMatMult(tsirm->A,tsirm->S,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&AS);
CHKERRQ(ierr);
first_iteration = 0;
} else {
ierr = MatMatMult(tsirm->A,tsirm->S,MAT_REUSE_MATRIX,PETSC_DEFAULT,&AS);
CHKERRQ(ierr);
}
/* CGLS or LSQR method to minimize the residuals*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp_min);
CHKERRQ(ierr);
if (tsirm->cgls) {
ierr = KSPSetType(ksp_min,KSPCGLS);
CHKERRQ(ierr);
} else {
ierr = KSPSetType(ksp_min,KSPLSQR);
CHKERRQ(ierr);
}
ierr = KSPSetOperators(ksp_min,AS,AS);
CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp_min,tsirm->tol_ls,PETSC_DEFAULT,PETSC_DEFAULT,tsirm->maxiter_ls);
CHKERRQ(ierr);
ierr = KSPGetPC(ksp_min,&pc_min);
CHKERRQ(ierr);
ierr = PCSetType(pc_min,PCNONE);
CHKERRQ(ierr);
ierr = KSPSolve(ksp_min,b,tsirm->Alpha);
CHKERRQ(ierr); /* Find Alpha such that ||AS Alpha = b|| */
ierr = KSPDestroy(&ksp_min);
CHKERRQ(ierr);
/* Apply minimization */
ierr = MatMult(tsirm->S,tsirm->Alpha,x);
CHKERRQ(ierr); /* x = S * Alpha */
}
} while (ksp->its<ksp->max_it && !ksp->reason);
ierr = MatDestroy(&AS);
CHKERRQ(ierr);
ierr = PetscFree(ind_row);
CHKERRQ(ierr);
ksp->its = total;
PetscFunctionReturn(0);
}
krylov_petsc_info_t
krylov_petsc_solve
(
p4est_t* p4est,
problem_data_t* vecs,
weakeqn_ptrs_t* fcns,
p4est_ghost_t** ghost,
element_data_t** ghost_data,
dgmath_jit_dbase_t* dgmath_jit_dbase,
krylov_petsc_params_t* krylov_params
)
{
krylov_petsc_info_t info;
KSP ksp;
Vec x,b;
PC pc;
/* double* u_temp; */
/* double* rhs_temp; */
KSPCreate(PETSC_COMM_WORLD,&ksp);
VecCreate(PETSC_COMM_WORLD,&x);//CHKERRQ(ierr);
VecSetSizes(x, vecs->local_nodes, PETSC_DECIDE);//CHKERRQ(ierr);
VecSetFromOptions(x);//CHKERRQ(ierr);
VecDuplicate(x,&b);//CHKERRQ(ierr);
/* VecGetArray(x,&u_temp); */
/* VecGetArray(b,&rhs_temp); */
krylov_pc_ctx_t kct;
kct.p4est = p4est;
kct.vecs = vecs;
kct.fcns = fcns;
kct.ghost = ghost;
kct.ghost_data = ghost_data;
kct.dgmath_jit_dbase = dgmath_jit_dbase;
kct.pc_data = krylov_params->pc_data;
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_monitor","");
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_view","");
/* KSPMonitorSet(ksp, KSPMonitorDefault, NULL, NULL); */
/* PetscOptionsSetValue(NULL,"-ksp_converged_reason",""); */
PetscOptionsSetValue(NULL,"-ksp_atol","1e-20");
/* PetscOptionsSetValue(NULL,"-with-debugging","1"); */
PetscOptionsSetValue(NULL,"-ksp_rtol","1e-20");
PetscOptionsSetValue(NULL,"-ksp_max_it","1000000");
KSPGetPC(ksp,&pc);
krylov_pc_t* kp = NULL;
if (krylov_params != NULL && krylov_params->user_defined_pc) {
PCSetType(pc,PCSHELL);//CHKERRQ(ierr);
kp = krylov_params->pc_create(&kct);
PCShellSetApply(pc, krylov_petsc_pc_apply);//CHKERRQ(ierr);
PCShellSetSetUp(pc, krylov_petsc_pc_setup);
PCShellSetContext(pc, kp);//CHKERRQ(ierr);
}
else {
PCSetType(pc,PCNONE);//CHKERRQ(ierr);
}
KSPSetType(ksp, krylov_params->krylov_type);
KSPSetFromOptions(ksp);
/* Create matrix-free shell for Aij */
Mat A;
MatCreateShell
(
PETSC_COMM_WORLD,
vecs->local_nodes,
vecs->local_nodes,
PETSC_DETERMINE,
PETSC_DETERMINE,
(void*)&kct,
&A
);
MatShellSetOperation(A,MATOP_MULT,(void(*)())krylov_petsc_apply_aij);
/* Set Amat and Pmat, where Pmat is the matrix the Preconditioner needs */
KSPSetOperators(ksp,A,A);
/* linalg_copy_1st_to_2nd(vecs->u, u_temp, vecs->local_nodes); */
/* linalg_copy_1st_to_2nd(vecs->rhs, rhs_temp, vecs->local_nodes); */
VecPlaceArray(b, vecs->rhs);
VecPlaceArray(x, vecs->u);
KSPSolve(ksp,b,x);
if (krylov_params != NULL && krylov_params->user_defined_pc) {
krylov_params->pc_destroy(kp);
}
KSPGetIterationNumber(ksp, &(info.iterations));
KSPGetResidualNorm(ksp, &(info.residual_norm));
/* linalg_copy_1st_to_2nd(u_temp, vecs->u, vecs->local_nodes); */
/* VecRestoreArray(x,&u_temp); */
/* VecRestoreArray(b,&rhs_temp); */
VecResetArray(b);
VecResetArray(x);
VecDestroy(&x);
VecDestroy(&b);
KSPDestroy(&ksp);
return info;
}
int implicit_solver(HashTable* El_Table, HashTable* NodeTable, double delta_t, double LapCoef,
TimeProps* timeprops_ptr) {
Vec x, b, xlocal; /* approx solution, RHS */
Mat A; /* linear system matrix */
KSP ksp; /* KSP context */
PetscReal norm; //,val1,val2; /* norm of solution error */
PetscErrorCode ierr;
PetscInt xsize, *num_elem_proc, *to, *from, its;
PetscMPIInt rank, size;
KSPConvergedReason reason;
VecScatter vscat;
IS globalis, tois;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
MPI_Comm_size(PETSC_COMM_WORLD, &size);
/* -------------------------------------------------------------------
Compute the matrix and right-hand-side vector that define
the linear system, Ax = b.
------------------------------------------------------------------- */
ierr = PetscMalloc(size * sizeof(PetscInt), &num_elem_proc);
CHKERRQ(ierr);
num_elem_proc[rank] = num_nonzero_elem(El_Table);
if (rank > 0)
MPI_Send(&num_elem_proc[rank], 1, MPI_INT, 0, 22, PETSC_COMM_WORLD);
if (rank == 0)
for (int i = 1; i < size; i++)
MPI_Recv(&num_elem_proc[i], 1, MPI_INT, i, 22, PETSC_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Barrier(PETSC_COMM_WORLD);
MPI_Bcast(num_elem_proc, size, MPI_INT, 0, PETSC_COMM_WORLD);
//printf("Number of elements are (hi i am second)...........%d\n", num_nonzero_elem(Laplacian->El_Table));
int total_elem = 0, start_elem = 0;
//MPI_Allreduce(&num_elem, total_elem, 1, MPI_INT, MPI_SUM, PETSC_COMM_WORLD);
ierr = PetscMalloc(num_elem_proc[rank] * sizeof(PetscInt), &to);
CHKERRQ(ierr);
ierr = PetscMalloc(num_elem_proc[rank] * sizeof(PetscInt), &from);
CHKERRQ(ierr);
for (int i = 0; i < size; i++)
total_elem += num_elem_proc[i];
for (int i = 0; i < rank; i++)
start_elem += num_elem_proc[i];
for (int i = 0; i < num_elem_proc[rank]; i++) {
from[i] = i;
to[i] = i + start_elem;
}
ierr = ISCreateGeneral(PETSC_COMM_WORLD, num_elem_proc[rank], from, PETSC_COPY_VALUES, &tois);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD, num_elem_proc[rank], to, PETSC_COPY_VALUES, &globalis);
CHKERRQ(ierr);
/*
Create parallel vectors
*/
ierr = VecCreate(PETSC_COMM_WORLD, &b);
CHKERRQ(ierr);
ierr = VecSetType(b, VECSTANDARD);
CHKERRQ(ierr);
ierr = VecSetSizes(b, num_elem_proc[rank], total_elem);
CHKERRQ(ierr);
ierr = VecSetFromOptions(b);
CHKERRQ(ierr);
ierr = VecGetSize(b, &xsize);
//cout<<"size b is "<<xsize<<endl;
ierr = VecCreateSeq(PETSC_COMM_SELF, num_elem_proc[rank], &xlocal);
CHKERRQ(ierr);
ierr = VecScatterCreate(xlocal, tois, b, globalis, &vscat);
CHKERRQ(ierr);
// we have to create a map between local and global vector
myctx.El_Table = El_Table;
myctx.Node_Table = NodeTable;
myctx.Scatter = vscat;
myctx.Total_elem = total_elem;
myctx.Num_elem_proc = num_elem_proc;
myctx.rank = rank;
myctx.size = size;
myctx.Timeptr = timeprops_ptr;
myctx.LapCoef = LapCoef;
myctx.delta_t = delta_t;
/*
right-hand-side vector.
*/
ierr = MakeRHS(&myctx, b);
CHKERRQ(ierr);
//ierr = VecView(b,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecDuplicate(b, &x);
CHKERRQ(ierr);
ierr = VecCopy(b, x);
CHKERRQ(ierr);
//ierr = VecView(x,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecGetSize(x, &xsize);
//cout<<"size x is "<<xsize<<endl;
/*
Create and assemble parallel matrix
*/
ierr = MatCreateShell(PETSC_COMM_WORLD, num_elem_proc[rank], num_elem_proc[rank], total_elem,
total_elem, &myctx, &A);
CHKERRQ(ierr);
ierr = MatShellSetOperation(A, MATOP_MULT, (void (*)(void))MatLaplacian2D_Mult);CHKERRQ
(ierr);
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);
CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp, A, A/*,DIFFERENT_NONZERO_PATTERN*/);
CHKERRQ(ierr);
ierr = KSPSetType(ksp, KSPFGMRES);
CHKERRQ(ierr);
/*
Set default preconditioner for this program to be block Jacobi.
This choice can be overridden at runtime with the option
-pc_type <type>
*/
// ierr = KSPSetTolerances(ksp,1.e-7,PETSC_DEFAULT,1.e9,3000);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT, 50000);
CHKERRQ(ierr);
/* -------------------------------------------------------------------
Solve the linear system
------------------------------------------------------------------- */
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
/*
Solve the linear system
*/
ierr = KSPSolve(ksp, b, x);
CHKERRQ(ierr);
/* -------------------------------------------------------------------
Check solution and clean up
------------------------------------------------------------------- */
/*
Check the error
*/
//ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
if (rank == 0) {
ierr = KSPGetIterationNumber(ksp, &its);
CHKERRQ(ierr);
ierr = KSPGetResidualNorm(ksp, &norm);
CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf( MPI_COMM_SELF, "Norm of error %g iterations %D\n",
(double) norm, its);
CHKERRQ(ierr);
//PetscSynchronizedFlush(PETSC_COMM_WORLD);
ierr = KSPGetConvergedReason(ksp, &reason);
ierr = PetscSynchronizedPrintf( MPI_COMM_SELF, "kind of divergence is: ...........%D \n",
reason);
//PetscSynchronizedFlush(PETSC_COMM_WORLD);
}
/*
*/
//ierr = VecGetArray(x,&xx);CHKERRQ(ierr);
update_phi(El_Table, x, &myctx);
//ierr = VecRestoreArray(x, &xx);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
MPI_Barrier(PETSC_COMM_WORLD);
ierr = KSPDestroy(&ksp);
CHKERRQ(ierr); //ierr = PetscFree(phin);CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr); //ierr = PCDestroy(&pc);CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = VecScatterDestroy(&vscat);
CHKERRQ(ierr);
ierr = ISDestroy(&globalis);
CHKERRQ(ierr);
ierr = ISDestroy(&tois);
CHKERRQ(ierr);
PetscFree(num_elem_proc);
CHKERRQ(ierr);
PetscFree(to);
CHKERRQ(ierr);
PetscFree(from);
CHKERRQ(ierr);
return 0;
}
typename SolverLinearPetsc<T>::solve_return_type
SolverLinearPetsc<T>::solve ( MatrixSparse<T> const& matrix_in,
MatrixSparse<T> const& precond_in,
Vector<T> & solution_in,
Vector<T> const& rhs_in,
const double tol,
const unsigned int m_its,
bool transpose )
{
this->setWorldComm( matrix_in.comm() );
this->init ();
MatrixPetsc<T> * matrix = const_cast<MatrixPetsc<T> *>( dynamic_cast<MatrixPetsc<T> const*>( &matrix_in ) );
MatrixPetsc<T> * precond = const_cast<MatrixPetsc<T> *>( dynamic_cast<MatrixPetsc<T> const*>( &precond_in ) );
VectorPetsc<T> * solution = dynamic_cast<VectorPetsc<T>*>( &solution_in );
VectorPetsc<T> * rhs = const_cast<VectorPetsc<T> *>( dynamic_cast<VectorPetsc<T> const*>( &rhs_in ) );
// We cast to pointers so we can be sure that they succeeded
// by comparing the result against NULL.
FEELPP_ASSERT( matrix != NULL ).error( "non petsc matrix structure" );
FEELPP_ASSERT( precond != NULL ).error( "non petsc matrix structure" );
FEELPP_ASSERT( solution != NULL ).error( "non petsc vector structure" );
FEELPP_ASSERT( rhs != NULL ).error( "non petsc vector structure" );
int ierr=0;
int its=0;
PetscReal final_resid=0.;
// Close the matrices and vectors in case this wasn't already done.
matrix->close ();
precond->close ();
solution->close ();
rhs->close ();
if ( !this->M_preconditioner && this->preconditionerType() == FIELDSPLIT_PRECOND )
matrix->updatePCFieldSplit( M_pc );
// // If matrix != precond, then this means we have specified a
// // special preconditioner, so reset preconditioner type to PCMAT.
// if (matrix != precond)
// {
// this->_preconditioner_type = USER_PRECOND;
// this->set_petsc_preconditioner_type ();
// }
// 2.1.x & earlier style
#if (PETSC_VERSION_MAJOR == 2) && (PETSC_VERSION_MINOR <= 1)
// Set operators. The input matrix works as the preconditioning matrix
ierr = SLESSetOperators( M_sles, matrix->mat(), precond->mat(),
SAME_NONZERO_PATTERN );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set the tolerances for the iterative solver. Use the user-supplied
// tolerance for the relative residual & leave the others at default values.
ierr = KSPSetTolerances ( M_ksp,
this->rTolerance(),
this->aTolerance(),
this->dTolerance(),
this->maxIterations() );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// makes the default convergence test use || B*(b - A*(initial guess))||
// instead of || B*b ||. In the case of right preconditioner or if
// KSPSetNormType(ksp,KSP_NORM_UNPRECONDIITONED) is used there is no B in
// the above formula. UIRNorm is short for Use Initial Residual Norm.
#if PETSC_VERSION_GREATER_OR_EQUAL_THAN(3,4,4)
KSPConvergedDefaultSetUIRNorm( M_ksp );
#else
KSPDefaultConvergedSetUIRNorm( M_ksp );
#endif
// Solve the linear system
ierr = SLESSolve ( M_sles, rhs->vec(), solution->vec(), &its );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Get the norm of the final residual to return to the user.
ierr = KSPGetResidualNorm ( M_ksp, &final_resid );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// 2.2.0
#elif (PETSC_VERSION_MAJOR == 2) && (PETSC_VERSION_MINOR == 2) && (PETSC_VERSION_SUBMINOR == 0)
// Set operators. The input matrix works as the preconditioning matrix
ierr = KSPSetOperators( M_ksp, matrix->mat(), precond->mat(),
MatStructure::SAME_NONZERO_PATTERN );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set the tolerances for the iterative solver. Use the user-supplied
// tolerance for the relative residual & leave the others at default values.
// Convergence is detected at iteration k if
// ||r_k||_2 < max(rtol*||b||_2 , abstol)
// where r_k is the residual vector and b is the right-hand side. Note that
// it is the *maximum* of the two values, the larger of which will almost
// always be rtol*||b||_2.
ierr = KSPSetTolerances ( M_ksp,
this->rTolerance(),
this->aTolerance(),
this->dTolerance(),
this->maxIterations() );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set the solution vector to use
ierr = KSPSetSolution ( M_ksp, solution->vec() );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set the RHS vector to use
ierr = KSPSetRhs ( M_ksp, rhs->vec() );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// makes the default convergence test use || B*(b - A*(initial guess))||
// instead of || B*b ||. In the case of right preconditioner or if
// KSPSetNormType(ksp,KSP_NORM_UNPRECONDIITONED) is used there is no B in
// the above formula. UIRNorm is short for Use Initial Residual Norm.
#if PETSC_VERSION_GREATER_OR_EQUAL_THAN(3,4,4)
KSPConvergedDefaultSetUIRNorm( M_ksp );
#else
KSPDefaultConvergedSetUIRNorm( M_ksp );
#endif
// Solve the linear system
if ( transpose )
ierr = KSPSolveTranspose ( M_ksp );
else
ierr = KSPSolve ( M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Get the number of iterations required for convergence
ierr = KSPGetIterationNumber ( M_ksp, &its );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Get the norm of the final residual to return to the user.
ierr = KSPGetResidualNorm ( M_ksp, &final_resid );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// 2.2.1 & newer style
#else
//std::cout << "sles: " << this->precMatrixStructure() << "\n";
// Set operators. The input matrix works as the preconditioning matrix
#if PETSC_VERSION_LESS_THAN(3,5,0)
ierr = KSPSetOperators( M_ksp, matrix->mat(), precond->mat(),
PetscGetMatStructureEnum(this->precMatrixStructure()) );
#else
ierr = KSPSetReusePreconditioner( M_ksp, (this->precMatrixStructure() == Feel::SAME_PRECONDITIONER)? PETSC_TRUE : PETSC_FALSE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = KSPSetOperators( M_ksp, matrix->mat(), precond->mat() );
#endif
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set the tolerances for the iterative solver. Use the user-supplied
// tolerance for the relative residual & leave the others at default values.
ierr = KSPSetTolerances ( M_ksp,
this->rTolerance(),
//1e-15,
this->aTolerance(),
this->dTolerance(),
this->maxIterations() );
CHKERRABORT( this->worldComm().globalComm(),ierr );
//PreconditionerPetsc<T>::setPetscPreconditionerType( this->preconditionerType(),this->matSolverPackageType(),M_pc, this->worldComm() );
// makes the default convergence test use || B*(b - A*(initial guess))||
// instead of || B*b ||. In the case of right preconditioner or if
// KSPSetNormType(ksp,KSP_NORM_UNPRECONDIITONED) is used there is no B in
// the above formula. UIRNorm is short for Use Initial Residual Norm.
#if PETSC_VERSION_LESS_THAN(3,5,0)
KSPDefaultConvergedSetUIRNorm( M_ksp );
#else
KSPConvergedDefaultSetUIRNorm( M_ksp );
#endif
// Solve the linear system
if ( transpose )
ierr = KSPSolveTranspose ( M_ksp, rhs->vec(), solution->vec() );
else
ierr = KSPSolve ( M_ksp, rhs->vec(), solution->vec() );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Get the number of iterations required for convergence
ierr = KSPGetIterationNumber ( M_ksp, &its );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Get the norm of the final residual to return to the user.
ierr = KSPGetResidualNorm ( M_ksp, &final_resid );
//std::cout << "final residual = " << final_resid << "\n";
CHKERRABORT( this->worldComm().globalComm(),ierr );
KSPConvergedReason reason;
KSPGetConvergedReason( M_ksp,&reason );
if ( option( _prefix=this->prefix(), _name="ksp-view" ).template as<bool>() )
check( KSPView( M_ksp, PETSC_VIEWER_STDOUT_WORLD ) );
if ( reason==KSP_DIVERGED_INDEFINITE_PC )
{
LOG(INFO) << "[solverlinearpetsc] Divergence because of indefinite preconditioner;\n";
LOG(INFO) << "[solverlinearpetsc] Run the executable again but with '-pc_factor_shift_type POSITIVE_DEFINITE' option.\n";
}
else if ( reason<0 )
{
LOG(INFO) <<"[solverlinearpetsc] Other kind of divergence: this should not happen.\n";
}
bool hasConverged;
if ( reason> 0 )
{
hasConverged=true;
if (this->showKSPConvergedReason() && this->worldComm().globalRank() == this->worldComm().masterRank() )
std::cout<< "Linear solve converged due to " << PetscConvertKSPReasonToString(reason)
<< " iterations " << its << std::endl;
}
else
{
hasConverged=false;
if (this->showKSPConvergedReason() && this->worldComm().globalRank() == this->worldComm().masterRank() )
std::cout<< "Linear solve did not converge due to " << PetscConvertKSPReasonToString(reason)
<< " iterations " << its << std::endl;
}
#endif
// return the # of its. and the final residual norm.
//return std::make_pair(its, final_resid);
return solve_return_type( boost::make_tuple( hasConverged, its, final_resid ) );
}
void PETSC_STDCALL kspgetresidualnorm_(KSP ksp,PetscReal *rnorm, int *__ierr ){
*__ierr = KSPGetResidualNorm(
(KSP)PetscToPointer((ksp) ),rnorm);
}
