PetscErrorCode KSPSetFromOptions_GMRES(KSP ksp)
{
PetscErrorCode ierr;
PetscInt restart;
PetscReal haptol;
KSP_GMRES *gmres = (KSP_GMRES*)ksp->data;
PetscBool flg;
PetscFunctionBegin;
ierr = PetscOptionsHead("KSP GMRES Options");CHKERRQ(ierr);
ierr = PetscOptionsInt("-ksp_gmres_restart","Number of Krylov search directions","KSPGMRESSetRestart",gmres->max_k,&restart,&flg);CHKERRQ(ierr);
if (flg) { ierr = KSPGMRESSetRestart(ksp,restart);CHKERRQ(ierr); }
ierr = PetscOptionsReal("-ksp_gmres_haptol","Tolerance for exact convergence (happy ending)","KSPGMRESSetHapTol",gmres->haptol,&haptol,&flg);CHKERRQ(ierr);
if (flg) { ierr = KSPGMRESSetHapTol(ksp,haptol);CHKERRQ(ierr); }
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_gmres_preallocate","Preallocate Krylov vectors","KSPGMRESSetPreAllocateVectors",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {ierr = KSPGMRESSetPreAllocateVectors(ksp);CHKERRQ(ierr);}
ierr = PetscOptionsBoolGroupBegin("-ksp_gmres_classicalgramschmidt","Classical (unmodified) Gram-Schmidt (fast)","KSPGMRESSetOrthogonalization",&flg);CHKERRQ(ierr);
if (flg) {ierr = KSPGMRESSetOrthogonalization(ksp,KSPGMRESClassicalGramSchmidtOrthogonalization);CHKERRQ(ierr);}
ierr = PetscOptionsBoolGroupEnd("-ksp_gmres_modifiedgramschmidt","Modified Gram-Schmidt (slow,more stable)","KSPGMRESSetOrthogonalization",&flg);CHKERRQ(ierr);
if (flg) {ierr = KSPGMRESSetOrthogonalization(ksp,KSPGMRESModifiedGramSchmidtOrthogonalization);CHKERRQ(ierr);}
ierr = PetscOptionsEnum("-ksp_gmres_cgs_refinement_type","Type of iterative refinement for classical (unmodified) Gram-Schmidt","KSPGMRESSetCGSRefinementType",
KSPGMRESCGSRefinementTypes,(PetscEnum)gmres->cgstype,(PetscEnum*)&gmres->cgstype,&flg);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsBool("-ksp_gmres_krylov_monitor","Plot the Krylov directions","KSPMonitorSet",flg,&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewers viewers;
ierr = PetscViewersCreate(PetscObjectComm((PetscObject)ksp),&viewers);CHKERRQ(ierr);
ierr = KSPMonitorSet(ksp,KSPGMRESMonitorKrylov,viewers,(PetscErrorCode (*)(void**))PetscViewersDestroy);CHKERRQ(ierr);
}
ierr = PetscOptionsTail();CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
TSMonitorSPEigCtxCreate - Creates a context for use with TS to monitor the eigenvalues of the linearized operator
Collective on TS
Input Parameters:
+ host - the X display to open, or null for the local machine
. label - the title to put in the title bar
. x, y - the screen coordinates of the upper left coordinate of the window
. m, n - the screen width and height in pixels
- howoften - if positive then determines the frequency of the plotting, if -1 then only at the final time
Output Parameter:
. ctx - the context
Options Database Key:
. -ts_monitor_sp_eig - plot egienvalues of linearized right hand side
Notes:
Use TSMonitorSPEigCtxDestroy() to destroy.
Currently only works if the Jacobian is provided explicitly.
Currently only works for ODEs u_t - F(t,u) = 0; that is with no mass matrix.
Level: intermediate
.keywords: TS, monitor, line graph, residual, seealso
.seealso: TSMonitorSPEigTimeStep(), TSMonitorSet(), TSMonitorLGSolution(), TSMonitorLGError()
@*/
PetscErrorCode TSMonitorSPEigCtxCreate(MPI_Comm comm,const char host[],const char label[],int x,int y,int m,int n,PetscInt howoften,TSMonitorSPEigCtx *ctx)
{
PetscDraw win;
PetscErrorCode ierr;
PC pc;
PetscFunctionBegin;
ierr = PetscNew(ctx);CHKERRQ(ierr);
ierr = PetscRandomCreate(comm,&(*ctx)->rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions((*ctx)->rand);CHKERRQ(ierr);
ierr = PetscDrawCreate(comm,host,label,x,y,m,n,&win);CHKERRQ(ierr);
ierr = PetscDrawSetFromOptions(win);CHKERRQ(ierr);
ierr = PetscDrawSPCreate(win,1,&(*ctx)->drawsp);CHKERRQ(ierr);
ierr = KSPCreate(comm,&(*ctx)->ksp);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix((*ctx)->ksp,"ts_monitor_sp_eig_");CHKERRQ(ierr); /* this is wrong, used use also prefix from the TS */
ierr = KSPSetType((*ctx)->ksp,KSPGMRES);CHKERRQ(ierr);
ierr = KSPGMRESSetRestart((*ctx)->ksp,200);CHKERRQ(ierr);
ierr = KSPSetTolerances((*ctx)->ksp,1.e-10,PETSC_DEFAULT,PETSC_DEFAULT,200);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues((*ctx)->ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = KSPSetFromOptions((*ctx)->ksp);CHKERRQ(ierr);
ierr = KSPGetPC((*ctx)->ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);CHKERRQ(ierr);
(*ctx)->howoften = howoften;
(*ctx)->computeexplicitly = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-ts_monitor_sp_eig_explicitly",&(*ctx)->computeexplicitly,NULL);CHKERRQ(ierr);
(*ctx)->comm = comm;
(*ctx)->xmin = -2.1;
(*ctx)->xmax = 1.1;
(*ctx)->ymin = -1.1;
(*ctx)->ymax = 1.1;
PetscFunctionReturn(0);
}
static PetscErrorCode KSPComputeShifts_GMRES(KSP ksp)
{
PetscErrorCode ierr;
KSP_AGMRES *agmres = (KSP_AGMRES*)(ksp->data);
KSP kspgmres;
Mat Amat, Pmat;
MatStructure flag;
PetscInt max_k = agmres->max_k;
PC pc;
PetscInt m;
PetscScalar *Rshift, *Ishift;
PetscFunctionBegin;
/* Perform one cycle of classical GMRES (with the Arnoldi process) to get the Hessenberg matrix
We assume here that the ksp is AGMRES and that the operators for the
linear system have been set in this ksp */
ierr = KSPCreate(PetscObjectComm((PetscObject)ksp), &kspgmres);CHKERRQ(ierr);
if (!ksp->pc) { ierr = KSPGetPC(ksp,&ksp->pc);CHKERRQ(ierr); }
ierr = PCGetOperators(ksp->pc, &Amat, &Pmat);CHKERRQ(ierr);
ierr = KSPSetOperators(kspgmres, Amat, Pmat);CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspgmres);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-ksp_view", &flg);CHKERRQ(ierr);
if (flag) { ierr = PetscOptionsClearValue("-ksp_view");CHKERRQ(ierr); }
ierr = KSPSetType(kspgmres, KSPGMRES);CHKERRQ(ierr);
ierr = KSPGMRESSetRestart(kspgmres, max_k);CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = KSPSetPC(kspgmres, pc);CHKERRQ(ierr);
/* Copy common options */
kspgmres->pc_side = ksp->pc_side;
/* Setup KSP context */
ierr = KSPSetComputeEigenvalues(kspgmres, PETSC_TRUE);CHKERRQ(ierr);
ierr = KSPSetUp(kspgmres);CHKERRQ(ierr);
kspgmres->max_it = max_k; /* Restrict the maximum number of iterations to one cycle of GMRES */
kspgmres->rtol = ksp->rtol;
ierr = KSPSolve(kspgmres, ksp->vec_rhs, ksp->vec_sol);CHKERRQ(ierr);
ksp->guess_zero = PETSC_FALSE;
ksp->rnorm = kspgmres->rnorm;
ksp->its = kspgmres->its;
if (kspgmres->reason == KSP_CONVERGED_RTOL) {
ksp->reason = KSP_CONVERGED_RTOL;
PetscFunctionReturn(0);
} else ksp->reason = KSP_CONVERGED_ITERATING;
/* Now, compute the Shifts values */
ierr = PetscMalloc2(max_k,&Rshift,max_k,&Ishift);CHKERRQ(ierr);
ierr = KSPComputeEigenvalues(kspgmres, max_k, Rshift, Ishift, &m);CHKERRQ(ierr);
if (m < max_k) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_PLIB, "Unable to compute the Shifts for the Newton basis");
else {
ierr = KSPAGMRESLejaOrdering(Rshift, Ishift, agmres->Rshift, agmres->Ishift, max_k);CHKERRQ(ierr);
agmres->HasShifts = PETSC_TRUE;
}
/* Restore KSP view options */
if (flg) { ierr = PetscOptionsSetValue("-ksp_view", "");CHKERRQ(ierr); }
PetscFunctionReturn(0);
}
void PotentialSolve::SetupOperator(PotentialOp optype, double fval, vector<double> origin) {
//Set the matrix
switch (optype) {
case REALPBC :
_BuildCARTPBC();
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &mnull);
break;
case CARTPBC :
_BuildCARTPBC(fval);
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &mnull);
break;
case RADIAL :
_BuildRADIAL(fval, origin);
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &mnull);
break;
default :
RAISE_ERR(99, "Unknown operator type");
}
// Sanity check
PetscTruth isNull;
MatNullSpaceTest(mnull, a, &isNull);
if (isNull != PETSC_TRUE) {
PetscPrintf(PETSC_COMM_WORLD, "Warning -- Null space is not truly a null space!\n");
}
PetscPrintf(PETSC_COMM_WORLD, "Completed operator initialization\n");
// Initialize the solver
KSPCreate(PETSC_COMM_WORLD,&solver);
KSPSetOperators(solver, a, a,SAME_PRECONDITIONER);
KSPSetTolerances(solver, rtol, atol, PETSC_DEFAULT, maxit);
// Set some defaults
KSPSetType(solver,KSPLGMRES);
KSPGMRESSetRestart(solver, 10);
// Possible for user to override
KSPSetFromOptions(solver);
// Set up the null space
KSPSetNullSpace(solver, mnull);
PetscPrintf(PETSC_COMM_WORLD, "Solver setup\n");
}
/*
* steady_state solves for the steady_state of the system
* that was previously setup using the add_to_ham and add_lin
* routines. Solver selection and parameterscan be controlled via PETSc
* command line options.
*/
void steady_state(Vec x){
PetscViewer mat_view;
PC pc;
Vec b;
KSP ksp; /* linear solver context */
PetscInt row,col,its,j,i,Istart,Iend;
PetscScalar mat_tmp;
long dim;
int num_pop;
double *populations;
Mat solve_A;
if (_lindblad_terms) {
dim = total_levels*total_levels;
solve_A = full_A;
if (nid==0) {
printf("Lindblad terms found, using Lindblad solver.");
}
} else {
if (nid==0) {
printf("Warning! Steady state not supported for Schrodinger.\n");
printf(" Defaulting to (less efficient) Lindblad Solver\n");
exit(0);
}
dim = total_levels*total_levels;
solve_A = ham_A;
}
if (!stab_added){
if (nid==0) printf("Adding stabilization...\n");
/*
* Add elements to the matrix to make the normalization work
* I have no idea why this works, I am copying it from qutip
* We add 1.0 in the 0th spot and every n+1 after
*/
if (nid==0) {
row = 0;
for (i=0;i<total_levels;i++){
col = i*(total_levels+1);
mat_tmp = 1.0 + 0.*PETSC_i;
MatSetValue(full_A,row,col,mat_tmp,ADD_VALUES);
}
/* Print dense ham, if it was asked for */
if (_print_dense_ham){
FILE *fp_ham;
fp_ham = fopen("ham","w");
if (nid==0){
for (i=0;i<total_levels;i++){
for (j=0;j<total_levels;j++){
fprintf(fp_ham,"%e %e ",PetscRealPart(_hamiltonian[i][j]),PetscImaginaryPart(_hamiltonian[i][j]));
}
fprintf(fp_ham,"\n");
}
}
fclose(fp_ham);
for (i=0;i<total_levels;i++){
free(_hamiltonian[i]);
}
free(_hamiltonian);
_print_dense_ham = 0;
}
}
stab_added = 1;
}
// if (!matrix_assembled) {
MatGetOwnershipRange(full_A,&Istart,&Iend);
/*
* Explicitly add 0.0 to all diagonal elements;
* this fixes a 'matrix in wrong state' message that PETSc
* gives if the diagonal was never initialized.
*/
if (nid==0) printf("Adding 0 to diagonal elements...\n");
for (i=Istart;i<Iend;i++){
mat_tmp = 0 + 0.*PETSC_i;
MatSetValue(full_A,i,i,mat_tmp,ADD_VALUES);
}
/* Tell PETSc to assemble the matrix */
MatAssemblyBegin(full_A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(full_A,MAT_FINAL_ASSEMBLY);
if (nid==0) printf("Matrix Assembled.\n");
matrix_assembled = 1;
// }
/* Print information about the matrix. */
PetscViewerASCIIOpen(PETSC_COMM_WORLD,NULL,&mat_view);
PetscViewerPushFormat(mat_view,PETSC_VIEWER_ASCII_INFO);
MatView(full_A,mat_view);
PetscViewerPopFormat(mat_view);
PetscViewerDestroy(&mat_view);
/*
* Create parallel vectors.
* - When using VecCreate(), VecSetSizes() and VecSetFromOptions(),
* we specify only the vector's global
* dimension; the parallel partitioning is determined at runtime.
* - Note: We form 1 vector from scratch and then duplicate as needed.
*/
VecCreate(PETSC_COMM_WORLD,&b);
VecSetSizes(b,PETSC_DECIDE,dim);
VecSetFromOptions(b);
// VecDuplicate(b,&x); Assume x is passed in
/*
* Set rhs, b, and solution, x to 1.0 in the first
* element, 0.0 elsewhere.
*/
VecSet(b,0.0);
VecSet(x,0.0);
if(nid==0) {
row = 0;
mat_tmp = 1.0 + 0.0*PETSC_i;
VecSetValue(x,row,mat_tmp,INSERT_VALUES);
VecSetValue(b,row,mat_tmp,INSERT_VALUES);
}
/* Assemble x and b */
VecAssemblyBegin(x);
VecAssemblyEnd(x);
VecAssemblyBegin(b);
VecAssemblyEnd(b);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*
* Create the linear solver and set various options *
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
* Create linear solver context
*/
KSPCreate(PETSC_COMM_WORLD,&ksp);
/*
* Set operators. Here the matrix that defines the linear system
* also serves as the preconditioning matrix.
*/
KSPSetOperators(ksp,full_A,full_A);
/*
* Set good default options for solver
*/
/* relative tolerance */
KSPSetTolerances(ksp,default_rtol,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
/* bjacobi preconditioner */
KSPGetPC(ksp,&pc);
PCSetType(pc,PCASM);
/* gmres solver with 100 restart*/
KSPSetType(ksp,KSPGMRES);
KSPGMRESSetRestart(ksp,default_restart);
/*
* Set runtime options, e.g.,
* -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
*/
KSPSetFromOptions(ksp);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
if (nid==0) printf("KSP set. Solving for steady state...\n");
KSPSolve(ksp,b,x);
num_pop = get_num_populations();
populations = malloc(num_pop*sizeof(double));
get_populations(x,&populations);
if(nid==0){
printf("Final populations: ");
for(i=0;i<num_pop;i++){
printf(" %e ",populations[i]);
}
printf("\n");
}
KSPGetIterationNumber(ksp,&its);
PetscPrintf(PETSC_COMM_WORLD,"Iterations %D\n",its);
/* Free work space */
KSPDestroy(&ksp);
// VecDestroy(&x);
VecDestroy(&b);
return;
}
void SolverLinearPetsc<T>::init ()
{
// Initialize the data structures if not done so already.
if ( !this->initialized() )
{
this->setInitialized( true );
int ierr=0;
// 2.1.x & earlier style
#if (PETSC_VERSION_MAJOR == 2) && (PETSC_VERSION_MINOR <= 1)
// Create the linear solver context
ierr = SLESCreate ( this->worldComm().globalComm(), &M_sles );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Create the Krylov subspace & preconditioner contexts
ierr = SLESGetKSP ( M_sles, &M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = SLESGetPC ( M_sles, &M_pc );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Have the Krylov subspace method use our good initial guess rather than 0
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_TRUE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set user-specified solver and preconditioner types
this->setPetscSolverType();
this->setPetscPreconditionerType();
this->setPetscConstantNullSpace();
// Set the options from user-input
// Set runtime options, e.g.,
// -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
// These options will override those specified above as long as
// SLESSetFromOptions() is called _after_ any other customization
// routines.
ierr = SLESSetFromOptions ( M_sles );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// 2.2.0 & newer style
#else
// Create the linear solver context
ierr = KSPCreate ( this->worldComm().globalComm(), &M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Create the preconditioner context
ierr = KSPGetPC ( M_ksp, &M_pc );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Have the Krylov subspace method use our good initial guess rather than 0
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_TRUE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set user-specified solver and preconditioner types
this->setPetscSolverType();
this->setPetscConstantNullSpace();
// Set the options from user-input
// Set runtime options, e.g.,
// -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
// These options will override those specified above as long as
// KSPSetFromOptions() is called _after_ any other customization
// routines.
//ierr = PCSetFromOptions ( M_pc );
//CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = KSPSetFromOptions ( M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
#endif
// Have the Krylov subspace method use our good initial guess
// rather than 0, unless the user requested a KSPType of
// preonly, which complains if asked to use initial guesses.
#if PETSC_VERSION_LESS_THAN(3,0,0)
KSPType ksp_type;
#else
#if PETSC_VERSION_LESS_THAN(3,4,0)
const KSPType ksp_type;
#else
KSPType ksp_type;
#endif
#endif
ierr = KSPGetType ( M_ksp, &ksp_type );
CHKERRABORT( this->worldComm().globalComm(),ierr );
if ( std::string((char*)ksp_type) == std::string( ( char* )KSPPREONLY ) )
{
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_FALSE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
}
if ( std::string((char*)ksp_type) == std::string( ( char* )KSPGMRES ) )
{
int nRestartGMRES = option(_name="gmres-restart", _prefix=this->prefix() ).template as<int>();
ierr = KSPGMRESSetRestart( M_ksp, nRestartGMRES );
CHKERRABORT( this->worldComm().globalComm(),ierr );
}
// Notify PETSc of location to store residual history.
// This needs to be called before any solves, since
// it sets the residual history length to zero. The default
// behavior is for PETSc to allocate (internally) an array
// of size 1000 to hold the residual norm history.
ierr = KSPSetResidualHistory( M_ksp,
PETSC_NULL, // pointer to the array which holds the history
PETSC_DECIDE, // size of the array holding the history
PETSC_TRUE ); // Whether or not to reset the history for each solve.
CHKERRABORT( this->worldComm().globalComm(),ierr );
//If there is a preconditioner object we need to set the internal setup and apply routines
if ( this->M_preconditioner )
{
VLOG(2) << "preconditioner: " << this->M_preconditioner << "\n";
PCSetType(M_pc, PCSHELL);
PCShellSetName( M_pc, this->M_preconditioner->name().c_str() );
PCShellSetContext( M_pc,( void* )this->M_preconditioner.get() );
PCShellSetSetUp( M_pc,__feel_petsc_preconditioner_setup );
PCShellSetApply( M_pc,__feel_petsc_preconditioner_apply );
PCShellSetView( M_pc,__feel_petsc_preconditioner_view );
#if PETSC_VERSION_LESS_THAN(3,4,0)
const PCType pc_type;
#else
PCType pc_type;
#endif
ierr = PCGetType ( M_pc, &pc_type );
CHKERRABORT( this->worldComm().globalComm(),ierr );
VLOG(2) << "preconditioner set as " << pc_type << "\n";
}
else
{
this->setPetscPreconditionerType();
// sets the software that is used to perform the factorization
PetscPCFactorSetMatSolverPackage( M_pc,this->matSolverPackageType() );
}
if ( Environment::vm(_name="ksp-monitor",_prefix=this->prefix()).template as<bool>() )
{
KSPMonitorSet( M_ksp,KSPMonitorDefault,PETSC_NULL,PETSC_NULL );
}
}
}
int main(int argc,char **args){
PetscErrorCode ierr;
PetscInitialize(&argc,&args,0,help);
MPI_Comm comm=MPI_COMM_WORLD;
PetscMPIInt rank,size;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
// -------------------------------------------------------------------
PetscOptionsGetInt (NULL, "-vtk_order",&VTK_ORDER ,NULL);
PetscOptionsGetInt (NULL, "-dof",&INPUT_DOF ,NULL);
PetscOptionsGetReal(NULL, "-scal",& SCAL_EXP ,NULL);
PetscOptionsGetBool(NULL, "-periodic",& PERIODIC ,NULL);
PetscOptionsGetBool(NULL, "-tree",& TREE_ONLY ,NULL);
PetscOptionsGetInt (NULL, "-max_depth" ,&MAXDEPTH ,NULL);
PetscOptionsGetInt (NULL, "-min_depth" ,&MINDEPTH ,NULL);
PetscOptionsGetReal(NULL, "-ref_tol" ,& TOL ,NULL);
PetscOptionsGetReal(NULL, "-gmres_tol" ,&GMRES_TOL ,NULL);
PetscOptionsGetInt (NULL, "-fmm_q" ,& CHEB_DEG ,NULL);
PetscOptionsGetInt (NULL, "-fmm_m" ,&MUL_ORDER ,NULL);
PetscOptionsGetInt (NULL, "-gmres_iter",& MAX_ITER ,NULL);
PetscOptionsGetReal(NULL, "-eta" ,& eta_ ,NULL);
// -------------------------------------------------------------------
{
/* -------------------------------------------------------------------
Compute the matrix and right-hand-side vector that define
the linear system, Ax = b.
------------------------------------------------------------------- */
// Initialize FMM
FMMData fmm_data;
FMM_Init(comm, &fmm_data);
std::cout << "Can you HERE me now" << std::endl;
if(TREE_ONLY){
pvfmm::Profile::print(&comm);
ierr = PetscFinalize();
return 0;
}
eval_function_at_nodes(&fmm_data, eta, fmm_data.eta);
PetscInt m,n, M,N,l,L;
m=fmm_data.m; // local rows
n=fmm_data.n; // local columns
M=fmm_data.M; // global rows
N=fmm_data.N; // global columns
l=fmm_data.l; // local values at cheb nodes in cubes (the above are coeff numbers)
L=fmm_data.L; // global values at cheb nodes in cubes
Vec pt_sources,eta_comp,phi_0,phi_0_val,eta_val,phi;
{ // Create vectors
VecCreateMPI(comm,n,PETSC_DETERMINE,&pt_sources);
VecCreateMPI(comm,l,PETSC_DETERMINE,&phi_0_val); // vec of values at each cube
VecCreateMPI(comm,l,PETSC_DETERMINE,&eta_val); // vec of values at each cube
VecCreateMPI(comm,m,PETSC_DETERMINE,&phi_0); // b=G[f] // vec of the cheb coeffs
VecCreateMPI(comm,n,PETSC_DETERMINE,&eta_comp); // Ax=b
VecCreateMPI(comm,m,PETSC_DETERMINE,&phi);
}
std::vector<PetscInt> idxs;
std::vector<PetscScalar> eta_std_vec((int)L);
idxs.reserve((int)L);
//std::iota(idxs.begin(), idxs.end(), 0);
for(int i=0;i<(int)L;i++){
idxs.push_back(i);
eta_std_vec[i] = (PetscScalar)fmm_data.eta[i];
}
ierr = VecSetValues(eta_val,L,idxs.data(),eta_std_vec.data(),INSERT_VALUES);
CHKERRQ(ierr);
// Seeing if the copy constructor works??
//FMM_Tree_t* eta_tree = fmm_data.tree;
//std::cout << "here" << std::endl;
//fmm_data.eta_tree->Write2File("results/eta_other",0);
//PtWiseTreeMult(fmm_data,*fmm_data.eta_tree);
//fmm_data.tree->Write2File("results/etatimespt_srces",0);
VecAssemblyBegin(eta_val);
VecAssemblyEnd(eta_val);
//VecView(eta_val, PETSC_VIEWER_STDOUT_WORLD);
fmm_data.eta_val = eta_val;
pvfmm::Profile::Tic("Input_Vector_pt_sources",&comm,true);
{ // Create Input Vector. f
tree2vec(fmm_data,pt_sources);
fmm_data.tree->Write2File("results/pt_source",0);
}
pvfmm::Profile::Toc();
pvfmm::Profile::Tic("Input_Vector_phi_0",&comm,true);
{ // Compute phi_0(x) = \int G(x,y)fn_input(y)dy
fmm_data.tree->SetupFMM(fmm_data.fmm_mat);
fmm_data.tree->RunFMM();
fmm_data.tree->Copy_FMMOutput();
fmm_data.tree->Write2File("results/phi_0",0);
tree2vec(fmm_data,phi_0);
fmm_data.phi_0 = phi_0;
eval_cheb_at_nodes(&fmm_data,phi_0_val);
fmm_data.phi_0_vec = phi_0_val;
}
pvfmm::Profile::Toc();
pvfmm::Profile::Tic("FMMCreateShell",&comm,true);
Mat A;
{ // Create Matrix. A
FMMCreateShell(&fmm_data, &A);
MatShellSetOperation(A,MATOP_MULT,(void(*)(void))mult);
}
pvfmm::Profile::Toc();
pvfmm::Profile::Tic("Phi",&comm,true);
{ // Compute phi(x) = phi_0(x) -\int G(x,y)eta(y)phi_0(y)dy
CompPhiUsingBorn(phi, fmm_data);
fmm_data.tree->Write2File("results/phi",0);
// tree2vec(fmm_data,b);
}
pvfmm::Profile::Toc();
pvfmm::Profile::Tic("Right_hand_side",&comm,true);
{ // Compute phi_0(x) - phi(x)
// After this block phi becomes the RHS
ierr = VecAXPY(phi,-1,phi_0);
CHKERRQ(ierr);
vec2tree(phi,fmm_data);
//fmm_data.tree->RunFMM();
//fmm_data.tree->Copy_FMMOutput();
fmm_data.tree->Write2File("results/rhs",0);
// tree2vec(fmm_data,b);
}
pvfmm::Profile::Toc();
// Create solution vector
pvfmm::Profile::Tic("Initial_Vector_eta_comp",&comm,true);
ierr = VecDuplicate(pt_sources,&eta_comp);CHKERRQ(ierr);
pvfmm::Profile::Toc();
// Create linear solver context
pvfmm::Profile::Tic("KSPCreate",&comm,true);
KSP ksp ; ierr = KSPCreate(PETSC_COMM_WORLD,&ksp );CHKERRQ(ierr);
pvfmm::Profile::Toc();
// Set operators. Here the matrix that defines the linear system
// also serves as the preconditioning matrix.
pvfmm::Profile::Tic("KSPSetOperators",&comm,true);
ierr = KSPSetOperators(ksp ,A ,A ,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
pvfmm::Profile::Toc();
// Set runtime options
KSPSetType(ksp ,KSPGMRES);
KSPSetNormType(ksp , KSP_NORM_UNPRECONDITIONED);
KSPSetTolerances(ksp ,GMRES_TOL ,PETSC_DEFAULT,PETSC_DEFAULT,MAX_ITER );
//KSPGMRESSetRestart(ksp , MAX_ITER );
KSPGMRESSetRestart(ksp , 100 );
ierr = KSPSetFromOptions(ksp );CHKERRQ(ierr);
// -------------------------------------------------------------------
// Solve the linear system
// -------------------------------------------------------------------
pvfmm::Profile::Tic("KSPSolve",&comm,true);
time_ksp=-omp_get_wtime();
ierr = KSPSolve(ksp,phi,eta_comp);CHKERRQ(ierr);
MPI_Barrier(comm);
time_ksp+=omp_get_wtime();
pvfmm::Profile::Toc();
// View info about the solver
KSPView(ksp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
// -------------------------------------------------------------------
// Check solution and clean up
// -------------------------------------------------------------------
// Iterations
PetscInt its;
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Iterations %D\n",its);CHKERRQ(ierr);
iter_ksp=its;
{ // Write output
vec2tree(eta_comp, fmm_data);
fmm_data.tree->Write2File("results/eta_comp",VTK_ORDER);
}
// Free work space. All PETSc objects should be destroyed when they
// are no longer needed.
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&eta_comp);CHKERRQ(ierr);
ierr = VecDestroy(&phi_0);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
// Delete fmm data
FMMDestroy(&fmm_data);
pvfmm::Profile::print(&comm);
}
ierr = PetscFinalize();
return 0;
}
/* This function returns the suitable solver for pressure. linear system */
KSP Solver_get_pressure_solver(Mat lhs, Parameters *params) {
KSP solver;
PC pc;
double rtol;
int ierr;
PetscLogDouble T1, T2;
rtol = params->resmax;
ierr = KSPCreate(PETSC_COMM_WORLD, &solver); PETScErrAct(ierr);
short int hasnullspace = NO;
if (hasnullspace) {
MatNullSpace nullsp;
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &nullsp);
KSPSetNullSpace(solver, nullsp);
//MatNullSpaceDestroy(nullsp);
}
ierr = KSPSetOperators(solver, lhs, lhs, SAME_PRECONDITIONER); PETScErrAct(ierr);
ierr = KSPSetType(solver, KSPGMRES); PETScErrAct(ierr);
ierr = KSPGMRESSetRestart(solver, 20); PETScErrAct(ierr);
ierr = KSPSetTolerances(solver, rtol, PETSC_DEFAULT, PETSC_DEFAULT, 300); PETScErrAct(ierr);
ierr = KSPGetPC(solver, &pc); PETScErrAct(ierr);
//PCSetType(pc, PCNONE);
//PCSetType(pc, PCASM);
PCSetType(pc, PCHYPRE);
PCHYPRESetType(pc,"boomeramg");
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_max_levels", "25"); PETScErrAct(ierr);
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_strong_threshold", "0.0"); PETScErrAct(ierr);
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_relax_type_all", "SOR/Jacobi"); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_cycle_type", ""); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_cycle_type", "V"); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_coarsen_type", "PMIS"); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_truncfactor", "0.9"); PETScErrAct(ierr);
/*******************************************************************************************************/
/*******************************************************************************************************/
/*******************************************************************************************************/
/* Hypre-Petsc Interface.
The most important parameters to be set are
1- Strong Threshold
2- Truncation Factor
3- Coarsennig Type
*/
/* Between 0 to 1 */
/* "0 "gives better convergence rate (in 3D). */
/* Suggested values (By Hypre manual): 0.25 for 2D, 0.5 for 3D
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_strong_threshold", "0.0"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Available Options:
"CLJP","Ruge-Stueben","modifiedRuge-Stueben","Falgout", "PMIS", "HMIS"
Falgout is usually the best.
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_coarsen_type", "Falgout"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Availble options:
"local", "global"
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_measure_type", "local"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Availble options:
Jacobi,sequential-Gauss-Seidel,
SOR/Jacobi,backward-SOR/Jacobi,symmetric-SOR/Jacobi,Gaussian-elimination
Important: If you are using a symmetric KSP solver (like CG), you should use a symmetric smoother
here.
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_relax_type_all", "symmetric-SOR/Jacobi"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Available options:
"V", "W"
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_cycle_type", "V"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Availble options:
"classical", "", "", "direct", "multipass", "multipass-wts", "ext+i", "ext+i-cc", "standard", "standard-wts", "", "", "FF", "FF1"
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_interp_type", ""); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Available options:
Greater than zero.
Use zero for the best convergence. However, if you have memory problems, use greate than zero to save some
memory.
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_truncfactor", "0.0"); PETScErrAct(ierr);
*/
/*
Preconditioner Generation Options
PCSetType(pc,PCHYPRE) or -pc_type hypre
-pc_hypre_boomeramg_max_levels nmax
-pc_hypre_boomeramg_truncfactor
-pc_hypre_boomeramg_strong_threshold
-pc_hypre_boomeramg_max_row_sum
-pc_hypre_boomeramg_no_CF
-pc_hypre_boomeramg_coarsen_type CLJP,Ruge-Stueben,modifiedRuge-Stueben,
-pc_hypre_boomeramg_measure_type local,global
Preconditioner Iteration Options
-pc_hypre_boomeramg_relax_type_all Jacobi,sequential-Gauss-Seidel,
SOR/Jacobi,backward-SOR/Jacobi,symmetric-SOR/Jacobi,Gaussian-eliminat
-pc_hypre_boomeramg_relax_type_fine
-pc_hypre_boomeramg_relax_type_down
-pc_hypre_boomeramg_relax_type_up
-pc_hypre_boomeramg_relax_weight_all r
-pc_hypre_boomeramg_outer_relax_weight_all r
-pc_hypre_boomeramg_grid_sweeps_down n
-pc_hypre_boomeramg_grid_sweeps_up n
-pc_hypre_boomeramg_grid_sweeps_coarse n
-pc_hypre_boomeramg_tol tol
-pc_hypre_boomeramg_max_iter it
*/
/*
//ierr = PCSetType(pc, PCASM); PETScErrAct(ierr);
ierr = PCSetType(pc, PCNONE); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCILU); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCBJACOBI); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCLU); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCEISENSTAT); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCSOR); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCJACOBI); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCNONE); PETScErrAct(ierr);
*/
/*
ierr = KSPGetPC(solver, &pc); PETScErrAct(ierr);
ierr = PCSetType(pc, PCILU); PETScErrAct(ierr);
ierr = PCFactorSetLevels(pc, 3); PETScErrAct(ierr);
//ierr = PCFactorSetUseDropTolerance(pc, 1e-3, .1, 50); PETScErrAct(ierr);
//ierr = PCFactorSetFill(pc, 30.7648); PETScErrAct(ierr);
ierr = PCFactorSetReuseOrdering(pc, PETSC_TRUE); PETScErrAct(ierr);
ierr = PCFactorSetReuseFill(pc, PETSC_TRUE); PETScErrAct(ierr);
ierr = PCFactorSetAllowDiagonalFill(pc); PETScErrAct(ierr);
//ierr = PCFactorSetUseInPlace(pc); PETScErrAct(ierr);
*/
ierr = KSPSetInitialGuessNonzero(solver, PETSC_TRUE); PETScErrAct(ierr);
ierr = PetscGetTime(&T1);PETScErrAct(ierr);
ierr = KSPSetFromOptions(solver); PETScErrAct(ierr);
ierr = PetscGetTime(&T2);PETScErrAct(ierr);
PetscPrintf(PCW, "Setup time for the Pressure solver was:%f\n", (T2 - T1));
ierr = KSPView(solver, PETSC_VIEWER_STDOUT_WORLD); PETScErrAct(ierr);
return solver;
}
