void createOuterPC(OuterContext* ctx) {
PCCreate(((ctx->data)->commAll), &(ctx->outerPC));
PCSetType(ctx->outerPC, PCSHELL);
PCShellSetName(ctx->outerPC, "RSD");
PCShellSetContext(ctx->outerPC, ctx);
PCShellSetApply(ctx->outerPC, &outerPCapply);
}
void PetscNonlinearSolver<T>::init ()
{
// Initialize the data structures if not done so already.
if (!this->initialized())
{
this->_is_initialized = true;
PetscErrorCode ierr=0;
#if PETSC_VERSION_LESS_THAN(2,1,2)
// At least until Petsc 2.1.1, the SNESCreate had a different calling syntax.
// The second argument was of type SNESProblemType, and could have a value of
// either SNES_NONLINEAR_EQUATIONS or SNES_UNCONSTRAINED_MINIMIZATION.
ierr = SNESCreate(this->comm().get(), SNES_NONLINEAR_EQUATIONS, &_snes);
LIBMESH_CHKERRABORT(ierr);
#else
ierr = SNESCreate(this->comm().get(),&_snes);
LIBMESH_CHKERRABORT(ierr);
#endif
#if PETSC_VERSION_LESS_THAN(2,3,3)
ierr = SNESSetMonitor (_snes, __libmesh_petsc_snes_monitor,
this, PETSC_NULL);
#else
// API name change in PETSc 2.3.3
ierr = SNESMonitorSet (_snes, __libmesh_petsc_snes_monitor,
this, PETSC_NULL);
#endif
LIBMESH_CHKERRABORT(ierr);
#if PETSC_VERSION_LESS_THAN(3,1,0)
// Cannot call SNESSetOptions before SNESSetFunction when using
// any matrix free options with PETSc 3.1.0+
ierr = SNESSetFromOptions(_snes);
LIBMESH_CHKERRABORT(ierr);
#endif
if(this->_preconditioner)
{
KSP ksp;
ierr = SNESGetKSP (_snes, &ksp);
LIBMESH_CHKERRABORT(ierr);
PC pc;
ierr = KSPGetPC(ksp,&pc);
LIBMESH_CHKERRABORT(ierr);
this->_preconditioner->init();
PCSetType(pc, PCSHELL);
PCShellSetContext(pc,(void*)this->_preconditioner);
//Re-Use the shell functions from petsc_linear_solver
PCShellSetSetUp(pc,__libmesh_petsc_preconditioner_setup);
PCShellSetApply(pc,__libmesh_petsc_preconditioner_apply);
}
}
}
static int TaoSolve_BNLS(TAO_SOLVER tao, void*solver){
TAO_BNLS *bnls = (TAO_BNLS *)solver;
int info;
TaoInt lsflag,iter=0;
TaoTerminateReason reason=TAO_CONTINUE_ITERATING;
double f,f_full,gnorm,gdx,stepsize=1.0;
TaoTruth success;
TaoVec *XU, *XL;
TaoVec *X, *G=bnls->G, *PG=bnls->PG;
TaoVec *R=bnls->R, *DXFree=bnls->DXFree;
TaoVec *DX=bnls->DX, *Work=bnls->Work;
TaoMat *H, *Hsub=bnls->Hsub;
TaoIndexSet *FreeVariables = bnls->FreeVariables;
TaoFunctionBegin;
/* Check if upper bound greater than lower bound. */
info = TaoGetSolution(tao,&X);CHKERRQ(info); bnls->X=X;
info = TaoGetVariableBounds(tao,&XL,&XU);CHKERRQ(info);
info = TaoEvaluateVariableBounds(tao,XL,XU); CHKERRQ(info);
info = TaoGetHessian(tao,&H);CHKERRQ(info); bnls->H=H;
/* Project the current point onto the feasible set */
info = X->Median(XL,X,XU); CHKERRQ(info);
TaoLinearSolver *tls;
// Modify the linear solver to a conjugate gradient method
info = TaoGetLinearSolver(tao, &tls); CHKERRQ(info);
TaoLinearSolverPetsc *pls;
pls = dynamic_cast <TaoLinearSolverPetsc *> (tls);
// set trust radius to zero
// PETSc ignores this case and should return the negative curvature direction
// at its current default length
pls->SetTrustRadius(0.0);
if(!bnls->M) bnls->M = new TaoLMVMMat(X);
TaoLMVMMat *M = bnls->M;
KSP pksp = pls->GetKSP();
// we will want to provide an initial guess in case neg curvature on the first iteration
info = KSPSetInitialGuessNonzero(pksp,PETSC_TRUE); CHKERRQ(info);
PC ppc;
// Modify the preconditioner to use the bfgs approximation
info = KSPGetPC(pksp, &ppc); CHKERRQ(info);
PetscTruth BFGSPreconditioner=PETSC_FALSE;// debug flag
info = PetscOptionsGetTruth(PETSC_NULL,"-bnls_pc_bfgs",
&BFGSPreconditioner,PETSC_NULL); CHKERRQ(info);
if( BFGSPreconditioner)
{
info=PetscInfo(tao,"TaoSolve_BNLS: using bfgs preconditioner\n");
info = KSPSetNormType(pksp, KSP_NORM_PRECONDITIONED); CHKERRQ(info);
info = PCSetType(ppc, PCSHELL); CHKERRQ(info);
info = PCShellSetName(ppc, "bfgs"); CHKERRQ(info);
info = PCShellSetContext(ppc, M); CHKERRQ(info);
info = PCShellSetApply(ppc, bfgs_apply); CHKERRQ(info);
}
else
{// default to none
info=PetscInfo(tao,"TaoSolve_BNLS: using no preconditioner\n");
info = PCSetType(ppc, PCNONE); CHKERRQ(info);
}
info = TaoComputeMeritFunctionGradient(tao,X,&f,G);CHKERRQ(info);
info = PG->BoundGradientProjection(G,XL,X,XU);CHKERRQ(info);
info = PG->Norm2(&gnorm); CHKERRQ(info);
// Set initial scaling for the function
if (f != 0.0) {
info = M->SetDelta(2.0 * TaoAbsDouble(f) / (gnorm*gnorm)); CHKERRQ(info);
}
else {
info = M->SetDelta(2.0 / (gnorm*gnorm)); CHKERRQ(info);
}
while (reason==TAO_CONTINUE_ITERATING){
/* Project the gradient and calculate the norm */
info = PG->BoundGradientProjection(G,XL,X,XU);CHKERRQ(info);
info = PG->Norm2(&gnorm); CHKERRQ(info);
info = M->Update(X, PG); CHKERRQ(info);
PetscScalar ewAtol = PetscMin(0.5,gnorm)*gnorm;
info = KSPSetTolerances(pksp,PETSC_DEFAULT,ewAtol,
PETSC_DEFAULT, PETSC_DEFAULT); CHKERRQ(info);
info=PetscInfo1(tao,"TaoSolve_BNLS: gnorm =%g\n",gnorm);
pksp->printreason = PETSC_TRUE;
info = KSPView(pksp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(info);
M->View();
info = TaoMonitor(tao,iter++,f,gnorm,0.0,stepsize,&reason);
CHKERRQ(info);
if (reason!=TAO_CONTINUE_ITERATING) break;
info = FreeVariables->WhichEqual(PG,G); CHKERRQ(info);
info = TaoComputeHessian(tao,X,H);CHKERRQ(info);
/* Create a reduced linear system */
info = R->SetReducedVec(G,FreeVariables);CHKERRQ(info);
info = R->Negate();CHKERRQ(info);
/* Use gradient as initial guess */
PetscTruth UseGradientIG=PETSC_FALSE;// debug flag
info = PetscOptionsGetTruth(PETSC_NULL,"-bnls_use_gradient_ig",
&UseGradientIG,PETSC_NULL); CHKERRQ(info);
if(UseGradientIG)
info = DX->CopyFrom(G);
else
{
info=PetscInfo(tao,"TaoSolve_BNLS: use bfgs init guess \n");
info = M->Solve(G, DX, &success);
}
CHKERRQ(info);
info = DXFree->SetReducedVec(DX,FreeVariables);CHKERRQ(info);
info = DXFree->Negate(); CHKERRQ(info);
info = Hsub->SetReducedMatrix(H,FreeVariables,FreeVariables);CHKERRQ(info);
bnls->gamma_factor /= 2;
success = TAO_FALSE;
while (success==TAO_FALSE) {
/* Approximately solve the reduced linear system */
info = TaoPreLinearSolve(tao,Hsub);CHKERRQ(info);
info = TaoLinearSolve(tao,Hsub,R,DXFree,&success);CHKERRQ(info);
info = DX->SetToZero(); CHKERRQ(info);
info = DX->ReducedXPY(DXFree,FreeVariables);CHKERRQ(info);
info = DX->Dot(G,&gdx); CHKERRQ(info);
if (gdx>=0 || success==TAO_FALSE) { /* use bfgs direction */
info = M->Solve(G, DX, &success); CHKERRQ(info);
info = DX->BoundGradientProjection(DX,XL,X,XU); CHKERRQ(info);
info = DX->Negate(); CHKERRQ(info);
// Check for success (descent direction)
info = DX->Dot(G,&gdx); CHKERRQ(info);
if (gdx >= 0) {
// Step is not descent or solve was not successful
// Use steepest descent direction (scaled)
if (f != 0.0) {
info = M->SetDelta(2.0 * TaoAbsDouble(f) / (gnorm*gnorm)); CHKERRQ(info);
}
else {
info = M->SetDelta(2.0 / (gnorm*gnorm)); CHKERRQ(info);
}
info = M->Reset(); CHKERRQ(info);
info = M->Update(X, G); CHKERRQ(info);
info = DX->CopyFrom(G);
info = DX->Negate(); CHKERRQ(info);
info = DX->Dot(G,&gdx); CHKERRQ(info);
info=PetscInfo1(tao,"LMVM Solve Fail use steepest descent, gdx %22.12e \n",gdx);
}
else {
info=PetscInfo1(tao,"Newton Solve Fail use BFGS direction, gdx %22.12e \n",gdx);
}
success = TAO_TRUE;
// bnls->gamma_factor *= 2;
// bnls->gamma = bnls->gamma_factor*(gnorm);
//#if !defined(PETSC_USE_COMPLEX)
// info=PetscInfo2(tao,"TaoSolve_NLS: modify diagonal (assume same nonzero structure), gamma_factor=%g, gamma=%g\n",bnls->gamma_factor,bnls->gamma);
// CHKERRQ(info);
//#else
// info=PetscInfo3(tao,"TaoSolve_NLS: modify diagonal (asuume same nonzero structure), gamma_factor=%g, gamma=%g, gdx %22.12e \n",
// bnls->gamma_factor,PetscReal(bnls->gamma),gdx);CHKERRQ(info);
//#endif
// info = Hsub->ShiftDiagonal(bnls->gamma);CHKERRQ(info);
// if (f != 0.0) {
// info = M->SetDelta(2.0 * TaoAbsDouble(f) / (gnorm*gnorm)); CHKERRQ(info);
// }
// else {
// info = M->SetDelta(2.0 / (gnorm*gnorm)); CHKERRQ(info);
// }
// info = M->Reset(); CHKERRQ(info);
// info = M->Update(X, G); CHKERRQ(info);
// success = TAO_FALSE;
} else {
info=PetscInfo1(tao,"Newton Solve is descent direction, gdx %22.12e \n",gdx);
success = TAO_TRUE;
}
}
stepsize=1.0;
info = TaoLineSearchApply(tao,X,G,DX,Work,
&f,&f_full,&stepsize,&lsflag);
CHKERRQ(info);
} /* END MAIN LOOP */
TaoFunctionReturn(0);
}
int main(int argc, char **argv)
{
PetscErrorCode ierr;
PetscInt n=10000,its,dfid=1;
Vec x,b,u;
Mat A;
KSP ksp;
PC pc,pcnoise;
PCNoise_Ctx ctx={0,NULL};
PetscReal eta=0.1,norm;
PetscScalar(*diagfunc)(PetscInt,PetscInt);
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Process command line options */
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,"-eta",&eta,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-diagfunc",&dfid,NULL);CHKERRQ(ierr);
switch(dfid){
case 1:
diagfunc = diagFunc1;
break;
case 2:
diagfunc = diagFunc2;
break;
case 3:
diagfunc = diagFunc3;
break;
default:
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Unrecognized diagfunc option");
}
/* Create a diagonal matrix with a given distribution of diagonal elements */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = AssembleDiagonalMatrix(A,diagfunc);CHKERRQ(ierr);
/* Allocate vectors and manufacture an exact solution and rhs */
ierr = MatCreateVecs(A,&x,NULL);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)x,"Computed Solution");CHKERRQ(ierr);
ierr = MatCreateVecs(A,&b,NULL);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)b,"RHS");CHKERRQ(ierr);
ierr = MatCreateVecs(A,&u,NULL);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)u,"Reference Solution");CHKERRQ(ierr);
ierr = VecSet(u,1.0);CHKERRQ(ierr);
ierr = MatMult(A,u,b);CHKERRQ(ierr);
/* Create a KSP object */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
/* Set up a composite preconditioner */
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCCOMPOSITE);CHKERRQ(ierr); /* default composite with single Identity PC */
ierr = PCCompositeSetType(pc,PC_COMPOSITE_ADDITIVE);CHKERRQ(ierr);
ierr = PCCompositeAddPC(pc,PCNONE);CHKERRQ(ierr);
if(eta > 0){
ierr = PCCompositeAddPC(pc,PCSHELL);CHKERRQ(ierr);
ierr = PCCompositeGetPC(pc,1,&pcnoise);CHKERRQ(ierr);
ctx.eta = eta;
ierr = PCShellSetContext(pcnoise,&ctx);CHKERRQ(ierr);
ierr = PCShellSetApply(pcnoise,PCApply_Noise);CHKERRQ(ierr);
ierr = PCShellSetSetUp(pcnoise,PCSetup_Noise);CHKERRQ(ierr);
ierr = PCShellSetDestroy(pcnoise,PCDestroy_Noise);CHKERRQ(ierr);
ierr = PCShellSetName(pcnoise,"Noise PC");CHKERRQ(ierr);
}
/* Set KSP from options (this can override the PC just defined) */
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* Solve */
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* Compute error */
ierr = VecAXPY(x,-1.0,u);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)x,"Error");CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g, Iterations %D\n",(double)norm,its);CHKERRQ(ierr);
/* Destroy objects and finalize */
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
PetscErrorCode PCBDDCNullSpaceAssembleCorrection(PC pc, PetscBool isdir, IS local_dofs)
{
PC_BDDC *pcbddc = (PC_BDDC*)pc->data;
PC_IS *pcis = (PC_IS*)pc->data;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
KSP local_ksp;
PC newpc;
NullSpaceCorrection_ctx shell_ctx;
Mat local_mat,local_pmat,small_mat,inv_small_mat;
Vec work1,work2;
const Vec *nullvecs;
VecScatter scatter_ctx;
IS is_aux;
MatFactorInfo matinfo;
PetscScalar *basis_mat,*Kbasis_mat,*array,*array_mat;
PetscScalar one = 1.0,zero = 0.0, m_one = -1.0;
PetscInt basis_dofs,basis_size,nnsp_size,i,k;
PetscBool nnsp_has_cnst;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Infer the local solver */
ierr = ISGetSize(local_dofs,&basis_dofs);CHKERRQ(ierr);
if (isdir) {
/* Dirichlet solver */
local_ksp = pcbddc->ksp_D;
} else {
/* Neumann solver */
local_ksp = pcbddc->ksp_R;
}
ierr = KSPGetOperators(local_ksp,&local_mat,&local_pmat);CHKERRQ(ierr);
/* Get null space vecs */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nnsp_has_cnst,&nnsp_size,&nullvecs);CHKERRQ(ierr);
basis_size = nnsp_size;
if (nnsp_has_cnst) {
basis_size++;
}
if (basis_dofs) {
/* Create shell ctx */
ierr = PetscNew(&shell_ctx);CHKERRQ(ierr);
/* Create work vectors in shell context */
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_small_1,basis_size,basis_size);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_small_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_small_1,&shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_full_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_full_1,basis_dofs,basis_dofs);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_full_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&shell_ctx->work_full_2);CHKERRQ(ierr);
/* Allocate workspace */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->basis_mat );CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->Kbasis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Restrict local null space on selected dofs (Dirichlet or Neumann)
and compute matrices N and K*N */
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,local_dofs,work1,(IS)0,&scatter_ctx);CHKERRQ(ierr);
}
for (k=0;k<nnsp_size;k++) {
ierr = VecScatterBegin(matis->rctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->rctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (basis_dofs) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
}
if (basis_dofs) {
if (nnsp_has_cnst) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecSet(work1,one);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter_ctx);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Assemble another Mat object in shell context */
ierr = MatTransposeMatMult(shell_ctx->basis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&small_mat);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&matinfo);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,basis_size,0,1,&is_aux);CHKERRQ(ierr);
ierr = MatLUFactor(small_mat,is_aux,is_aux,&matinfo);CHKERRQ(ierr);
ierr = ISDestroy(&is_aux);CHKERRQ(ierr);
ierr = PetscMalloc1(basis_size*basis_size,&array_mat);CHKERRQ(ierr);
for (k=0;k<basis_size;k++) {
ierr = VecSet(shell_ctx->work_small_1,zero);CHKERRQ(ierr);
ierr = VecSetValue(shell_ctx->work_small_1,k,one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecAssemblyEnd(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = MatSolve(small_mat,shell_ctx->work_small_1,shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecGetArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
for (i=0;i<basis_size;i++) {
array_mat[i*basis_size+k]=array[i];
}
ierr = VecRestoreArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
}
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_size,basis_size,array_mat,&inv_small_mat);CHKERRQ(ierr);
ierr = MatMatMult(shell_ctx->basis_mat,inv_small_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&shell_ctx->Lbasis_mat);CHKERRQ(ierr);
ierr = PetscFree(array_mat);CHKERRQ(ierr);
ierr = MatDestroy(&inv_small_mat);CHKERRQ(ierr);
ierr = MatDestroy(&small_mat);CHKERRQ(ierr);
ierr = MatScale(shell_ctx->Kbasis_mat,m_one);CHKERRQ(ierr);
/* Rebuild local PC */
ierr = KSPGetPC(local_ksp,&shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PCCreate(PETSC_COMM_SELF,&newpc);CHKERRQ(ierr);
ierr = PCSetOperators(newpc,local_mat,local_mat);CHKERRQ(ierr);
ierr = PCSetType(newpc,PCSHELL);CHKERRQ(ierr);
ierr = PCShellSetContext(newpc,shell_ctx);CHKERRQ(ierr);
ierr = PCShellSetApply(newpc,PCBDDCApplyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCShellSetDestroy(newpc,PCBDDCDestroyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCSetUp(newpc);CHKERRQ(ierr);
ierr = KSPSetPC(local_ksp,newpc);CHKERRQ(ierr);
ierr = PCDestroy(&newpc);CHKERRQ(ierr);
ierr = KSPSetUp(local_ksp);CHKERRQ(ierr);
}
/* test */
if (pcbddc->dbg_flag && basis_dofs) {
KSP check_ksp;
PC check_pc;
Mat test_mat;
Vec work3;
PetscReal test_err,lambda_min,lambda_max;
PetscBool setsym,issym=PETSC_FALSE;
PetscInt tabs;
ierr = PetscViewerASCIIGetTab(pcbddc->dbg_viewer,&tabs);CHKERRQ(ierr);
ierr = KSPGetPC(local_ksp,&check_pc);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work3);CHKERRQ(ierr);
ierr = VecSetRandom(shell_ctx->work_small_1,NULL);CHKERRQ(ierr);
ierr = MatMult(shell_ctx->basis_mat,shell_ctx->work_small_1,work1);CHKERRQ(ierr);
ierr = VecCopy(work1,work2);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work3);CHKERRQ(ierr);
ierr = PCApply(check_pc,work3,work1);CHKERRQ(ierr);
ierr = VecAXPY(work1,m_one,work2);CHKERRQ(ierr);
ierr = VecNorm(work1,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for nullspace correction for ",PetscGlobalRank);CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(pcbddc->dbg_viewer,PETSC_FALSE);CHKERRQ(ierr);
if (isdir) {
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Dirichlet ");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Neumann ");CHKERRQ(ierr);
}
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"solver is :%1.14e\n",test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISetTab(pcbddc->dbg_viewer,tabs);CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(pcbddc->dbg_viewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatTransposeMatMult(shell_ctx->Lbasis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&test_mat);CHKERRQ(ierr);
ierr = MatShift(test_mat,one);CHKERRQ(ierr);
ierr = MatNorm(test_mat,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = MatDestroy(&test_mat);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for nullspace matrices is :%1.14e\n",PetscGlobalRank,test_err);CHKERRQ(ierr);
/* Create ksp object suitable for extreme eigenvalues' estimation */
ierr = KSPCreate(PETSC_COMM_SELF,&check_ksp);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(check_ksp,pc->erroriffailure);CHKERRQ(ierr);
ierr = KSPSetOperators(check_ksp,local_mat,local_mat);CHKERRQ(ierr);
ierr = KSPSetTolerances(check_ksp,1.e-8,1.e-8,PETSC_DEFAULT,basis_dofs);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(check_ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatIsSymmetricKnown(pc->pmat,&setsym,&issym);CHKERRQ(ierr);
if (issym) {
ierr = KSPSetType(check_ksp,KSPCG);CHKERRQ(ierr);
}
ierr = KSPSetPC(check_ksp,check_pc);CHKERRQ(ierr);
ierr = KSPSetUp(check_ksp);CHKERRQ(ierr);
ierr = VecSetRandom(work1,NULL);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = KSPSolve(check_ksp,work2,work2);CHKERRQ(ierr);
ierr = VecAXPY(work2,m_one,work1);CHKERRQ(ierr);
ierr = VecNorm(work2,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = KSPComputeExtremeSingularValues(check_ksp,&lambda_max,&lambda_min);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(check_ksp,&k);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for adapted KSP %1.14e (it %d, eigs %1.6e %1.6e)\n",PetscGlobalRank,test_err,k,lambda_min,lambda_max);CHKERRQ(ierr);
ierr = KSPDestroy(&check_ksp);CHKERRQ(ierr);
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecDestroy(&work3);CHKERRQ(ierr);
}
/* all processes shoud call this, even the void ones */
if (pcbddc->dbg_flag) {
ierr = PetscViewerFlush(pcbddc->dbg_viewer);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
DMMG *dmmg; /* multilevel grid structure */
PetscErrorCode ierr;
DA da;
AppCtx app;
PC pc;
KSP ksp;
PetscTruth isshell;
PetscViewer v1;
PetscInitialize(&argc,&argv,(char *)0,help);
PreLoadBegin(PETSC_TRUE,"SetUp");
app.comm = PETSC_COMM_WORLD;
app.nxv = 6;
app.nyvf = 3;
app.nyv = app.nyvf + 2;
ierr = PetscOptionsBegin(app.comm,PETSC_NULL,"Options for Grid Sizes",PETSC_NULL);
ierr = PetscOptionsInt("-nxv","Grid spacing in X direction",PETSC_NULL,app.nxv,&app.nxv,PETSC_NULL);
CHKERRQ(ierr);
ierr = PetscOptionsInt("-nyvf","Grid spacing in Y direction of Fuel",PETSC_NULL,app.nyvf,&app.nyvf,PETSC_NULL);
CHKERRQ(ierr);
ierr = PetscOptionsInt("-nyv","Total Grid spacing in Y direction of",PETSC_NULL,app.nyv,&app.nyv,PETSC_NULL);
CHKERRQ(ierr);
ierr = PetscOptionsEnd();
ierr = PetscViewerDrawOpen(app.comm,PETSC_NULL,"",-1,-1,-1,-1,&v1);
CHKERRQ(ierr);
/*
Create the DMComposite object to manage the three grids/physics.
We use a 1d decomposition along the y direction (since one of the grids is 1d).
*/
ierr = DMCompositeCreate(app.comm,&app.pack);
CHKERRQ(ierr);
/* 6 fluid unknowns, 3 ghost points on each end for either periodicity or simply boundary conditions */
ierr = DACreate1d(app.comm,DA_XPERIODIC,app.nxv,6,3,0,&da);
CHKERRQ(ierr);
ierr = DASetFieldName(da,0,"prss");
CHKERRQ(ierr);
ierr = DASetFieldName(da,1,"ergg");
CHKERRQ(ierr);
ierr = DASetFieldName(da,2,"ergf");
CHKERRQ(ierr);
ierr = DASetFieldName(da,3,"alfg");
CHKERRQ(ierr);
ierr = DASetFieldName(da,4,"velg");
CHKERRQ(ierr);
ierr = DASetFieldName(da,5,"velf");
CHKERRQ(ierr);
ierr = DMCompositeAddDM(app.pack,(DM)da);
CHKERRQ(ierr);
ierr = DADestroy(da);
CHKERRQ(ierr);
ierr = DACreate2d(app.comm,DA_YPERIODIC,DA_STENCIL_STAR,app.nxv,app.nyv,PETSC_DETERMINE,1,1,1,0,0,&da);
CHKERRQ(ierr);
ierr = DASetFieldName(da,0,"Tempature");
CHKERRQ(ierr);
ierr = DMCompositeAddDM(app.pack,(DM)da);
CHKERRQ(ierr);
ierr = DADestroy(da);
CHKERRQ(ierr);
ierr = DACreate2d(app.comm,DA_XYPERIODIC,DA_STENCIL_STAR,app.nxv,app.nyvf,PETSC_DETERMINE,1,2,1,0,0,&da);
CHKERRQ(ierr);
ierr = DASetFieldName(da,0,"Phi");
CHKERRQ(ierr);
ierr = DASetFieldName(da,1,"Pre");
CHKERRQ(ierr);
ierr = DMCompositeAddDM(app.pack,(DM)da);
CHKERRQ(ierr);
ierr = DADestroy(da);
CHKERRQ(ierr);
app.pri = 1.0135e+5;
app.ugi = 2.5065e+6;
app.ufi = 4.1894e+5;
app.agi = 1.00e-1;
app.vgi = 1.0e-1 ;
app.vfi = 1.0e-1;
app.prin = 1.0135e+5;
app.ugin = 2.5065e+6;
app.ufin = 4.1894e+5;
app.agin = 1.00e-1;
app.vgin = 1.0e-1 ;
app.vfin = 1.0e-1;
app.prout = 1.0135e+5;
app.ugout = 2.5065e+6;
app.ufout = 4.1894e+5;
app.agout = 3.0e-1;
app.twi = 373.15e+0;
app.phii = 1.0e+0;
app.prei = 1.0e-5;
/*
Create the solver object and attach the grid/physics info
*/
ierr = DMMGCreate(app.comm,1,0,&dmmg);
CHKERRQ(ierr);
ierr = DMMGSetDM(dmmg,(DM)app.pack);
CHKERRQ(ierr);
ierr = DMMGSetUser(dmmg,0,&app);
CHKERRQ(ierr);
ierr = DMMGSetISColoringType(dmmg,IS_COLORING_GLOBAL);
CHKERRQ(ierr);
CHKMEMQ;
ierr = DMMGSetInitialGuess(dmmg,FormInitialGuess);
CHKERRQ(ierr);
ierr = DMMGSetSNES(dmmg,FormFunction,0);
CHKERRQ(ierr);
ierr = DMMGSetFromOptions(dmmg);
CHKERRQ(ierr);
/* Supply custom shell preconditioner if requested */
ierr = SNESGetKSP(DMMGGetSNES(dmmg),&ksp);
CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);
CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject)pc,PCSHELL,&isshell);
CHKERRQ(ierr);
if (isshell) {
ierr = PCShellSetContext(pc,&app);
CHKERRQ(ierr);
ierr = PCShellSetSetUp(pc,MyPCSetUp);
CHKERRQ(ierr);
ierr = PCShellSetApply(pc,MyPCApply);
CHKERRQ(ierr);
ierr = PCShellSetDestroy(pc,MyPCDestroy);
CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the nonlinear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
PreLoadStage("Solve");
ierr = DMMGSolve(dmmg);
CHKERRQ(ierr);
ierr = VecView(DMMGGetx(dmmg),v1);
CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscViewerDestroy(v1);
CHKERRQ(ierr);
ierr = DMCompositeDestroy(app.pack);
CHKERRQ(ierr);
ierr = DMMGDestroy(dmmg);
CHKERRQ(ierr);
PreLoadEnd();
ierr = PetscFinalize();
CHKERRQ(ierr);
return 0;
}
PETSC_EXTERN void PETSC_STDCALL pcshellsetcontext_(PC pc,void*ctx, int *__ierr ){
*__ierr = PCShellSetContext(
(PC)PetscToPointer((pc) ),ctx);
}
static PetscErrorCode TaoSolve_NTR(Tao tao)
{
TAO_NTR *tr = (TAO_NTR *)tao->data;
PC pc;
KSPConvergedReason ksp_reason;
TaoConvergedReason reason;
PetscReal fmin, ftrial, prered, actred, kappa, sigma, beta;
PetscReal tau, tau_1, tau_2, tau_max, tau_min, max_radius;
PetscReal f, gnorm;
PetscReal delta;
PetscReal norm_d;
PetscErrorCode ierr;
PetscInt iter = 0;
PetscInt bfgsUpdates = 0;
PetscInt needH;
PetscInt i_max = 5;
PetscInt j_max = 1;
PetscInt i, j, N, n, its;
PetscFunctionBegin;
if (tao->XL || tao->XU || tao->ops->computebounds) {
ierr = PetscPrintf(((PetscObject)tao)->comm,"WARNING: Variable bounds have been set but will be ignored by ntr algorithm\n");CHKERRQ(ierr);
}
tao->trust = tao->trust0;
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
if (NTR_PC_BFGS == tr->pc_type && !tr->M) {
ierr = VecGetLocalSize(tao->solution,&n);CHKERRQ(ierr);
ierr = VecGetSize(tao->solution,&N);CHKERRQ(ierr);
ierr = MatCreateLMVM(((PetscObject)tao)->comm,n,N,&tr->M);CHKERRQ(ierr);
ierr = MatLMVMAllocateVectors(tr->M,tao->solution);CHKERRQ(ierr);
}
/* Check convergence criteria */
ierr = TaoComputeObjectiveAndGradient(tao, tao->solution, &f, tao->gradient);CHKERRQ(ierr);
ierr = VecNorm(tao->gradient,NORM_2,&gnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
needH = 1;
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, 1.0, &reason);CHKERRQ(ierr);
if (reason != TAO_CONTINUE_ITERATING) PetscFunctionReturn(0);
/* Create vectors for the limited memory preconditioner */
if ((NTR_PC_BFGS == tr->pc_type) &&
(BFGS_SCALE_BFGS != tr->bfgs_scale_type)) {
if (!tr->Diag) {
ierr = VecDuplicate(tao->solution, &tr->Diag);CHKERRQ(ierr);
}
}
switch(tr->ksp_type) {
case NTR_KSP_NASH:
ierr = KSPSetType(tao->ksp, KSPNASH);CHKERRQ(ierr);
if (tao->ksp->ops->setfromoptions) {
(*tao->ksp->ops->setfromoptions)(tao->ksp);
}
break;
case NTR_KSP_STCG:
ierr = KSPSetType(tao->ksp, KSPSTCG);CHKERRQ(ierr);
if (tao->ksp->ops->setfromoptions) {
(*tao->ksp->ops->setfromoptions)(tao->ksp);
}
break;
default:
ierr = KSPSetType(tao->ksp, KSPGLTR);CHKERRQ(ierr);
if (tao->ksp->ops->setfromoptions) {
(*tao->ksp->ops->setfromoptions)(tao->ksp);
}
break;
}
/* Modify the preconditioner to use the bfgs approximation */
ierr = KSPGetPC(tao->ksp, &pc);CHKERRQ(ierr);
switch(tr->pc_type) {
case NTR_PC_NONE:
ierr = PCSetType(pc, PCNONE);CHKERRQ(ierr);
if (pc->ops->setfromoptions) {
(*pc->ops->setfromoptions)(pc);
}
break;
case NTR_PC_AHESS:
ierr = PCSetType(pc, PCJACOBI);CHKERRQ(ierr);
if (pc->ops->setfromoptions) {
(*pc->ops->setfromoptions)(pc);
}
ierr = PCJacobiSetUseAbs(pc);CHKERRQ(ierr);
break;
case NTR_PC_BFGS:
ierr = PCSetType(pc, PCSHELL);CHKERRQ(ierr);
if (pc->ops->setfromoptions) {
(*pc->ops->setfromoptions)(pc);
}
ierr = PCShellSetName(pc, "bfgs");CHKERRQ(ierr);
ierr = PCShellSetContext(pc, tr->M);CHKERRQ(ierr);
ierr = PCShellSetApply(pc, MatLMVMSolveShell);CHKERRQ(ierr);
break;
default:
/* Use the pc method set by pc_type */
break;
}
/* Initialize trust-region radius */
switch(tr->init_type) {
case NTR_INIT_CONSTANT:
/* Use the initial radius specified */
break;
case NTR_INIT_INTERPOLATION:
/* Use the initial radius specified */
max_radius = 0.0;
for (j = 0; j < j_max; ++j) {
fmin = f;
sigma = 0.0;
if (needH) {
ierr = TaoComputeHessian(tao,tao->solution,tao->hessian,tao->hessian_pre);CHKERRQ(ierr);
needH = 0;
}
for (i = 0; i < i_max; ++i) {
ierr = VecCopy(tao->solution, tr->W);CHKERRQ(ierr);
ierr = VecAXPY(tr->W, -tao->trust/gnorm, tao->gradient);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao, tr->W, &ftrial);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(ftrial)) {
tau = tr->gamma1_i;
}
else {
if (ftrial < fmin) {
fmin = ftrial;
sigma = -tao->trust / gnorm;
}
ierr = MatMult(tao->hessian, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = VecDot(tao->gradient, tao->stepdirection, &prered);CHKERRQ(ierr);
prered = tao->trust * (gnorm - 0.5 * tao->trust * prered / (gnorm * gnorm));
actred = f - ftrial;
if ((PetscAbsScalar(actred) <= tr->epsilon) &&
(PetscAbsScalar(prered) <= tr->epsilon)) {
kappa = 1.0;
}
else {
kappa = actred / prered;
}
tau_1 = tr->theta_i * gnorm * tao->trust / (tr->theta_i * gnorm * tao->trust + (1.0 - tr->theta_i) * prered - actred);
tau_2 = tr->theta_i * gnorm * tao->trust / (tr->theta_i * gnorm * tao->trust - (1.0 + tr->theta_i) * prered + actred);
tau_min = PetscMin(tau_1, tau_2);
tau_max = PetscMax(tau_1, tau_2);
if (PetscAbsScalar(kappa - 1.0) <= tr->mu1_i) {
/* Great agreement */
max_radius = PetscMax(max_radius, tao->trust);
if (tau_max < 1.0) {
tau = tr->gamma3_i;
}
else if (tau_max > tr->gamma4_i) {
tau = tr->gamma4_i;
}
else {
tau = tau_max;
}
}
else if (PetscAbsScalar(kappa - 1.0) <= tr->mu2_i) {
/* Good agreement */
max_radius = PetscMax(max_radius, tao->trust);
if (tau_max < tr->gamma2_i) {
tau = tr->gamma2_i;
}
else if (tau_max > tr->gamma3_i) {
tau = tr->gamma3_i;
}
else {
tau = tau_max;
}
}
else {
/* Not good agreement */
if (tau_min > 1.0) {
tau = tr->gamma2_i;
}
else if (tau_max < tr->gamma1_i) {
tau = tr->gamma1_i;
}
else if ((tau_min < tr->gamma1_i) && (tau_max >= 1.0)) {
tau = tr->gamma1_i;
}
else if ((tau_1 >= tr->gamma1_i) && (tau_1 < 1.0) &&
((tau_2 < tr->gamma1_i) || (tau_2 >= 1.0))) {
tau = tau_1;
}
else if ((tau_2 >= tr->gamma1_i) && (tau_2 < 1.0) &&
((tau_1 < tr->gamma1_i) || (tau_2 >= 1.0))) {
tau = tau_2;
}
else {
tau = tau_max;
}
}
}
tao->trust = tau * tao->trust;
}
if (fmin < f) {
f = fmin;
ierr = VecAXPY(tao->solution, sigma, tao->gradient);CHKERRQ(ierr);
ierr = TaoComputeGradient(tao,tao->solution, tao->gradient);CHKERRQ(ierr);
ierr = VecNorm(tao->gradient, NORM_2, &gnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
needH = 1;
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, 1.0, &reason);CHKERRQ(ierr);
if (reason != TAO_CONTINUE_ITERATING) {
PetscFunctionReturn(0);
}
}
}
tao->trust = PetscMax(tao->trust, max_radius);
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
break;
default:
/* Norm of the first direction will initialize radius */
tao->trust = 0.0;
break;
}
/* Set initial scaling for the BFGS preconditioner
This step is done after computing the initial trust-region radius
since the function value may have decreased */
if (NTR_PC_BFGS == tr->pc_type) {
if (f != 0.0) {
delta = 2.0 * PetscAbsScalar(f) / (gnorm*gnorm);
}
else {
delta = 2.0 / (gnorm*gnorm);
}
ierr = MatLMVMSetDelta(tr->M,delta);CHKERRQ(ierr);
}
/* Have not converged; continue with Newton method */
while (reason == TAO_CONTINUE_ITERATING) {
++iter;
tao->ksp_its=0;
/* Compute the Hessian */
if (needH) {
ierr = TaoComputeHessian(tao,tao->solution,tao->hessian,tao->hessian_pre);CHKERRQ(ierr);
needH = 0;
}
if (NTR_PC_BFGS == tr->pc_type) {
if (BFGS_SCALE_AHESS == tr->bfgs_scale_type) {
/* Obtain diagonal for the bfgs preconditioner */
ierr = MatGetDiagonal(tao->hessian, tr->Diag);CHKERRQ(ierr);
ierr = VecAbs(tr->Diag);CHKERRQ(ierr);
ierr = VecReciprocal(tr->Diag);CHKERRQ(ierr);
ierr = MatLMVMSetScale(tr->M,tr->Diag);CHKERRQ(ierr);
}
/* Update the limited memory preconditioner */
ierr = MatLMVMUpdate(tr->M, tao->solution, tao->gradient);CHKERRQ(ierr);
++bfgsUpdates;
}
while (reason == TAO_CONTINUE_ITERATING) {
ierr = KSPSetOperators(tao->ksp, tao->hessian, tao->hessian_pre);CHKERRQ(ierr);
/* Solve the trust region subproblem */
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPNASHGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPSTCGGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else { /* NTR_KSP_GLTR */
ierr = KSPGLTRSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPGLTRGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
}
if (0.0 == tao->trust) {
/* Radius was uninitialized; use the norm of the direction */
if (norm_d > 0.0) {
tao->trust = norm_d;
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
}
else {
/* The direction was bad; set radius to default value and re-solve
the trust-region subproblem to get a direction */
tao->trust = tao->trust0;
/* Modify the radius if it is too large or small */
tao->trust = PetscMax(tao->trust, tr->min_radius);
tao->trust = PetscMin(tao->trust, tr->max_radius);
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPNASHGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPSTCGGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
} else { /* NTR_KSP_GLTR */
ierr = KSPGLTRSetRadius(tao->ksp,tao->trust);CHKERRQ(ierr);
ierr = KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(tao->ksp,&its);CHKERRQ(ierr);
tao->ksp_its+=its;
tao->ksp_tot_its+=its;
ierr = KSPGLTRGetNormD(tao->ksp, &norm_d);CHKERRQ(ierr);
}
if (norm_d == 0.0) SETERRQ(PETSC_COMM_SELF,1, "Initial direction zero");
}
}
ierr = VecScale(tao->stepdirection, -1.0);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(tao->ksp, &ksp_reason);CHKERRQ(ierr);
if ((KSP_DIVERGED_INDEFINITE_PC == ksp_reason) &&
(NTR_PC_BFGS == tr->pc_type) && (bfgsUpdates > 1)) {
/* Preconditioner is numerically indefinite; reset the
approximate if using BFGS preconditioning. */
if (f != 0.0) {
delta = 2.0 * PetscAbsScalar(f) / (gnorm*gnorm);
}
else {
delta = 2.0 / (gnorm*gnorm);
}
ierr = MatLMVMSetDelta(tr->M, delta);CHKERRQ(ierr);
ierr = MatLMVMReset(tr->M);CHKERRQ(ierr);
ierr = MatLMVMUpdate(tr->M, tao->solution, tao->gradient);CHKERRQ(ierr);
bfgsUpdates = 1;
}
if (NTR_UPDATE_REDUCTION == tr->update_type) {
/* Get predicted reduction */
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else { /* gltr */
ierr = KSPGLTRGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
}
if (prered >= 0.0) {
/* The predicted reduction has the wrong sign. This cannot
happen in infinite precision arithmetic. Step should
be rejected! */
tao->trust = tr->alpha1 * PetscMin(tao->trust, norm_d);
}
else {
/* Compute trial step and function value */
ierr = VecCopy(tao->solution,tr->W);CHKERRQ(ierr);
ierr = VecAXPY(tr->W, 1.0, tao->stepdirection);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao, tr->W, &ftrial);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(ftrial)) {
tao->trust = tr->alpha1 * PetscMin(tao->trust, norm_d);
} else {
/* Compute and actual reduction */
actred = f - ftrial;
prered = -prered;
if ((PetscAbsScalar(actred) <= tr->epsilon) &&
(PetscAbsScalar(prered) <= tr->epsilon)) {
kappa = 1.0;
}
else {
kappa = actred / prered;
}
/* Accept or reject the step and update radius */
if (kappa < tr->eta1) {
/* Reject the step */
tao->trust = tr->alpha1 * PetscMin(tao->trust, norm_d);
}
else {
/* Accept the step */
if (kappa < tr->eta2) {
/* Marginal bad step */
tao->trust = tr->alpha2 * PetscMin(tao->trust, norm_d);
}
else if (kappa < tr->eta3) {
/* Reasonable step */
tao->trust = tr->alpha3 * tao->trust;
}
else if (kappa < tr->eta4) {
/* Good step */
tao->trust = PetscMax(tr->alpha4 * norm_d, tao->trust);
}
else {
/* Very good step */
tao->trust = PetscMax(tr->alpha5 * norm_d, tao->trust);
}
break;
}
}
}
}
else {
/* Get predicted reduction */
if (NTR_KSP_NASH == tr->ksp_type) {
ierr = KSPNASHGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else if (NTR_KSP_STCG == tr->ksp_type) {
ierr = KSPSTCGGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
} else { /* gltr */
ierr = KSPGLTRGetObjFcn(tao->ksp,&prered);CHKERRQ(ierr);
}
if (prered >= 0.0) {
/* The predicted reduction has the wrong sign. This cannot
happen in infinite precision arithmetic. Step should
be rejected! */
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else {
ierr = VecCopy(tao->solution, tr->W);CHKERRQ(ierr);
ierr = VecAXPY(tr->W, 1.0, tao->stepdirection);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao, tr->W, &ftrial);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(ftrial)) {
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else {
ierr = VecDot(tao->gradient, tao->stepdirection, &beta);CHKERRQ(ierr);
actred = f - ftrial;
prered = -prered;
if ((PetscAbsScalar(actred) <= tr->epsilon) &&
(PetscAbsScalar(prered) <= tr->epsilon)) {
kappa = 1.0;
}
else {
kappa = actred / prered;
}
tau_1 = tr->theta * beta / (tr->theta * beta - (1.0 - tr->theta) * prered + actred);
tau_2 = tr->theta * beta / (tr->theta * beta + (1.0 + tr->theta) * prered - actred);
tau_min = PetscMin(tau_1, tau_2);
tau_max = PetscMax(tau_1, tau_2);
if (kappa >= 1.0 - tr->mu1) {
/* Great agreement; accept step and update radius */
if (tau_max < 1.0) {
tao->trust = PetscMax(tao->trust, tr->gamma3 * norm_d);
}
else if (tau_max > tr->gamma4) {
tao->trust = PetscMax(tao->trust, tr->gamma4 * norm_d);
}
else {
tao->trust = PetscMax(tao->trust, tau_max * norm_d);
}
break;
}
else if (kappa >= 1.0 - tr->mu2) {
/* Good agreement */
if (tau_max < tr->gamma2) {
tao->trust = tr->gamma2 * PetscMin(tao->trust, norm_d);
}
else if (tau_max > tr->gamma3) {
tao->trust = PetscMax(tao->trust, tr->gamma3 * norm_d);
}
else if (tau_max < 1.0) {
tao->trust = tau_max * PetscMin(tao->trust, norm_d);
}
else {
tao->trust = PetscMax(tao->trust, tau_max * norm_d);
}
break;
}
else {
/* Not good agreement */
if (tau_min > 1.0) {
tao->trust = tr->gamma2 * PetscMin(tao->trust, norm_d);
}
else if (tau_max < tr->gamma1) {
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else if ((tau_min < tr->gamma1) && (tau_max >= 1.0)) {
tao->trust = tr->gamma1 * PetscMin(tao->trust, norm_d);
}
else if ((tau_1 >= tr->gamma1) && (tau_1 < 1.0) &&
((tau_2 < tr->gamma1) || (tau_2 >= 1.0))) {
tao->trust = tau_1 * PetscMin(tao->trust, norm_d);
}
else if ((tau_2 >= tr->gamma1) && (tau_2 < 1.0) &&
((tau_1 < tr->gamma1) || (tau_2 >= 1.0))) {
tao->trust = tau_2 * PetscMin(tao->trust, norm_d);
}
else {
tao->trust = tau_max * PetscMin(tao->trust, norm_d);
}
}
}
}
}
/* The step computed was not good and the radius was decreased.
Monitor the radius to terminate. */
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, tao->trust, &reason);CHKERRQ(ierr);
}
/* The radius may have been increased; modify if it is too large */
tao->trust = PetscMin(tao->trust, tr->max_radius);
if (reason == TAO_CONTINUE_ITERATING) {
ierr = VecCopy(tr->W, tao->solution);CHKERRQ(ierr);
f = ftrial;
ierr = TaoComputeGradient(tao, tao->solution, tao->gradient);
ierr = VecNorm(tao->gradient, NORM_2, &gnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
needH = 1;
ierr = TaoMonitor(tao, iter, f, gnorm, 0.0, tao->trust, &reason);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
krylov_petsc_info_t
krylov_petsc_solve
(
p4est_t* p4est,
problem_data_t* vecs,
weakeqn_ptrs_t* fcns,
p4est_ghost_t** ghost,
element_data_t** ghost_data,
dgmath_jit_dbase_t* dgmath_jit_dbase,
krylov_petsc_params_t* krylov_params
)
{
krylov_petsc_info_t info;
KSP ksp;
Vec x,b;
PC pc;
/* double* u_temp; */
/* double* rhs_temp; */
KSPCreate(PETSC_COMM_WORLD,&ksp);
VecCreate(PETSC_COMM_WORLD,&x);//CHKERRQ(ierr);
VecSetSizes(x, vecs->local_nodes, PETSC_DECIDE);//CHKERRQ(ierr);
VecSetFromOptions(x);//CHKERRQ(ierr);
VecDuplicate(x,&b);//CHKERRQ(ierr);
/* VecGetArray(x,&u_temp); */
/* VecGetArray(b,&rhs_temp); */
krylov_pc_ctx_t kct;
kct.p4est = p4est;
kct.vecs = vecs;
kct.fcns = fcns;
kct.ghost = ghost;
kct.ghost_data = ghost_data;
kct.dgmath_jit_dbase = dgmath_jit_dbase;
kct.pc_data = krylov_params->pc_data;
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_monitor","");
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_view","");
/* KSPMonitorSet(ksp, KSPMonitorDefault, NULL, NULL); */
/* PetscOptionsSetValue(NULL,"-ksp_converged_reason",""); */
PetscOptionsSetValue(NULL,"-ksp_atol","1e-20");
/* PetscOptionsSetValue(NULL,"-with-debugging","1"); */
PetscOptionsSetValue(NULL,"-ksp_rtol","1e-20");
PetscOptionsSetValue(NULL,"-ksp_max_it","1000000");
KSPGetPC(ksp,&pc);
krylov_pc_t* kp = NULL;
if (krylov_params != NULL && krylov_params->user_defined_pc) {
PCSetType(pc,PCSHELL);//CHKERRQ(ierr);
kp = krylov_params->pc_create(&kct);
PCShellSetApply(pc, krylov_petsc_pc_apply);//CHKERRQ(ierr);
PCShellSetSetUp(pc, krylov_petsc_pc_setup);
PCShellSetContext(pc, kp);//CHKERRQ(ierr);
}
else {
PCSetType(pc,PCNONE);//CHKERRQ(ierr);
}
KSPSetType(ksp, krylov_params->krylov_type);
KSPSetFromOptions(ksp);
/* Create matrix-free shell for Aij */
Mat A;
MatCreateShell
(
PETSC_COMM_WORLD,
vecs->local_nodes,
vecs->local_nodes,
PETSC_DETERMINE,
PETSC_DETERMINE,
(void*)&kct,
&A
);
MatShellSetOperation(A,MATOP_MULT,(void(*)())krylov_petsc_apply_aij);
/* Set Amat and Pmat, where Pmat is the matrix the Preconditioner needs */
KSPSetOperators(ksp,A,A);
/* linalg_copy_1st_to_2nd(vecs->u, u_temp, vecs->local_nodes); */
/* linalg_copy_1st_to_2nd(vecs->rhs, rhs_temp, vecs->local_nodes); */
VecPlaceArray(b, vecs->rhs);
VecPlaceArray(x, vecs->u);
KSPSolve(ksp,b,x);
if (krylov_params != NULL && krylov_params->user_defined_pc) {
krylov_params->pc_destroy(kp);
}
KSPGetIterationNumber(ksp, &(info.iterations));
KSPGetResidualNorm(ksp, &(info.residual_norm));
/* linalg_copy_1st_to_2nd(u_temp, vecs->u, vecs->local_nodes); */
/* VecRestoreArray(x,&u_temp); */
/* VecRestoreArray(b,&rhs_temp); */
VecResetArray(b);
VecResetArray(x);
VecDestroy(&x);
VecDestroy(&b);
KSPDestroy(&ksp);
return info;
}
void SolverLinearPetsc<T>::init ()
{
// Initialize the data structures if not done so already.
if ( !this->initialized() )
{
this->setInitialized( true );
int ierr=0;
// 2.1.x & earlier style
#if (PETSC_VERSION_MAJOR == 2) && (PETSC_VERSION_MINOR <= 1)
// Create the linear solver context
ierr = SLESCreate ( this->worldComm().globalComm(), &M_sles );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Create the Krylov subspace & preconditioner contexts
ierr = SLESGetKSP ( M_sles, &M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = SLESGetPC ( M_sles, &M_pc );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Have the Krylov subspace method use our good initial guess rather than 0
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_TRUE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set user-specified solver and preconditioner types
this->setPetscSolverType();
this->setPetscPreconditionerType();
this->setPetscConstantNullSpace();
// Set the options from user-input
// Set runtime options, e.g.,
// -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
// These options will override those specified above as long as
// SLESSetFromOptions() is called _after_ any other customization
// routines.
ierr = SLESSetFromOptions ( M_sles );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// 2.2.0 & newer style
#else
// Create the linear solver context
ierr = KSPCreate ( this->worldComm().globalComm(), &M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Create the preconditioner context
ierr = KSPGetPC ( M_ksp, &M_pc );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Have the Krylov subspace method use our good initial guess rather than 0
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_TRUE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set user-specified solver and preconditioner types
this->setPetscSolverType();
this->setPetscConstantNullSpace();
// Set the options from user-input
// Set runtime options, e.g.,
// -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
// These options will override those specified above as long as
// KSPSetFromOptions() is called _after_ any other customization
// routines.
//ierr = PCSetFromOptions ( M_pc );
//CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = KSPSetFromOptions ( M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
#endif
// Have the Krylov subspace method use our good initial guess
// rather than 0, unless the user requested a KSPType of
// preonly, which complains if asked to use initial guesses.
#if PETSC_VERSION_LESS_THAN(3,0,0)
KSPType ksp_type;
#else
#if PETSC_VERSION_LESS_THAN(3,4,0)
const KSPType ksp_type;
#else
KSPType ksp_type;
#endif
#endif
ierr = KSPGetType ( M_ksp, &ksp_type );
CHKERRABORT( this->worldComm().globalComm(),ierr );
if ( std::string((char*)ksp_type) == std::string( ( char* )KSPPREONLY ) )
{
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_FALSE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
}
if ( std::string((char*)ksp_type) == std::string( ( char* )KSPGMRES ) )
{
int nRestartGMRES = option(_name="gmres-restart", _prefix=this->prefix() ).template as<int>();
ierr = KSPGMRESSetRestart( M_ksp, nRestartGMRES );
CHKERRABORT( this->worldComm().globalComm(),ierr );
}
// Notify PETSc of location to store residual history.
// This needs to be called before any solves, since
// it sets the residual history length to zero. The default
// behavior is for PETSc to allocate (internally) an array
// of size 1000 to hold the residual norm history.
ierr = KSPSetResidualHistory( M_ksp,
PETSC_NULL, // pointer to the array which holds the history
PETSC_DECIDE, // size of the array holding the history
PETSC_TRUE ); // Whether or not to reset the history for each solve.
CHKERRABORT( this->worldComm().globalComm(),ierr );
//If there is a preconditioner object we need to set the internal setup and apply routines
if ( this->M_preconditioner )
{
VLOG(2) << "preconditioner: " << this->M_preconditioner << "\n";
PCSetType(M_pc, PCSHELL);
PCShellSetName( M_pc, this->M_preconditioner->name().c_str() );
PCShellSetContext( M_pc,( void* )this->M_preconditioner.get() );
PCShellSetSetUp( M_pc,__feel_petsc_preconditioner_setup );
PCShellSetApply( M_pc,__feel_petsc_preconditioner_apply );
PCShellSetView( M_pc,__feel_petsc_preconditioner_view );
#if PETSC_VERSION_LESS_THAN(3,4,0)
const PCType pc_type;
#else
PCType pc_type;
#endif
ierr = PCGetType ( M_pc, &pc_type );
CHKERRABORT( this->worldComm().globalComm(),ierr );
VLOG(2) << "preconditioner set as " << pc_type << "\n";
}
else
{
this->setPetscPreconditionerType();
// sets the software that is used to perform the factorization
PetscPCFactorSetMatSolverPackage( M_pc,this->matSolverPackageType() );
}
if ( Environment::vm(_name="ksp-monitor",_prefix=this->prefix()).template as<bool>() )
{
KSPMonitorSet( M_ksp,KSPMonitorDefault,PETSC_NULL,PETSC_NULL );
}
}
}
void IBImplicitStaggeredHierarchyIntegrator::integrateHierarchy(const double current_time,
const double new_time,
const int cycle_num)
{
IBHierarchyIntegrator::integrateHierarchy(current_time, new_time, cycle_num);
Pointer<INSStaggeredHierarchyIntegrator> ins_hier_integrator = d_ins_hier_integrator;
TBOX_ASSERT(ins_hier_integrator);
PetscErrorCode ierr;
int n_local;
const int coarsest_ln = 0;
const int finest_ln = d_hierarchy->getFinestLevelNumber();
// const double half_time = current_time+0.5*(new_time-current_time);
VariableDatabase<NDIM>* var_db = VariableDatabase<NDIM>::getDatabase();
Pointer<VariableContext> current_ctx = ins_hier_integrator->getCurrentContext();
Pointer<VariableContext> scratch_ctx = ins_hier_integrator->getScratchContext();
Pointer<VariableContext> new_ctx = ins_hier_integrator->getNewContext();
const int wgt_cc_idx = d_hier_math_ops->getCellWeightPatchDescriptorIndex();
const int wgt_sc_idx = d_hier_math_ops->getSideWeightPatchDescriptorIndex();
Pointer<Variable<NDIM> > u_var = ins_hier_integrator->getVelocityVariable();
// const int u_current_idx = var_db->mapVariableAndContextToIndex(u_var, current_ctx);
const int u_scratch_idx = var_db->mapVariableAndContextToIndex(u_var, scratch_ctx);
// const int u_new_idx = var_db->mapVariableAndContextToIndex(u_var, new_ctx);
Pointer<Variable<NDIM> > p_var = ins_hier_integrator->getPressureVariable();
// const int p_current_idx = var_db->mapVariableAndContextToIndex(p_var, current_ctx);
const int p_scratch_idx = var_db->mapVariableAndContextToIndex(p_var, scratch_ctx);
// const int p_new_idx = var_db->mapVariableAndContextToIndex(p_var, new_ctx);
// Skip all cycles in the INS solver --- we advance the state data here.
ins_hier_integrator->skipCycle(current_time, new_time, cycle_num);
// Setup Eulerian vectors used in solving the implicit IB equations.
Pointer<SAMRAIVectorReal<NDIM, double> > eul_sol_vec = new SAMRAIVectorReal<NDIM, double>(
d_object_name + "::eulerian_sol_vec", d_hierarchy, coarsest_ln, finest_ln);
eul_sol_vec->addComponent(u_var, u_scratch_idx, wgt_sc_idx, d_hier_velocity_data_ops);
eul_sol_vec->addComponent(p_var, p_scratch_idx, wgt_cc_idx, d_hier_pressure_data_ops);
Pointer<SAMRAIVectorReal<NDIM, double> > eul_rhs_vec =
eul_sol_vec->cloneVector(d_object_name + "::eulerian_rhs_vec");
eul_rhs_vec->allocateVectorData(current_time);
d_u_scratch_vec = eul_sol_vec->cloneVector(d_object_name + "::u_scratch_vec");
d_f_scratch_vec = eul_rhs_vec->cloneVector(d_object_name + "::f_scratch_vec");
d_u_scratch_vec->allocateVectorData(current_time);
d_f_scratch_vec->allocateVectorData(current_time);
ins_hier_integrator->setupSolverVectors(
eul_sol_vec, eul_rhs_vec, current_time, new_time, cycle_num);
d_stokes_solver = ins_hier_integrator->getStokesSolver();
Pointer<KrylovLinearSolver> p_stokes_solver = d_stokes_solver;
TBOX_ASSERT(p_stokes_solver);
d_stokes_op = p_stokes_solver->getOperator();
TBOX_ASSERT(d_stokes_op);
// Setup Lagrangian vectors used in solving the implicit IB equations.
Vec lag_sol_petsc_vec, lag_rhs_petsc_vec;
d_ib_implicit_ops->createSolverVecs(lag_sol_petsc_vec, lag_rhs_petsc_vec);
d_ib_implicit_ops->setupSolverVecs(lag_sol_petsc_vec, lag_rhs_petsc_vec);
// Indicate that the current approximation to position of the structure
// should be used for Lagrangian-Eulerian coupling.
d_ib_implicit_ops->updateFixedLEOperators();
// Setup multi-vec objects to store the composite solution and
// right-hand-side vectors.
Vec eul_sol_petsc_vec =
PETScSAMRAIVectorReal::createPETScVector(eul_sol_vec, PETSC_COMM_WORLD);
Vec eul_rhs_petsc_vec =
PETScSAMRAIVectorReal::createPETScVector(eul_rhs_vec, PETSC_COMM_WORLD);
Vec sol_petsc_vecs[] = { eul_sol_petsc_vec, lag_sol_petsc_vec };
Vec rhs_petsc_vecs[] = { eul_rhs_petsc_vec, lag_rhs_petsc_vec };
Vec composite_sol_petsc_vec, composite_rhs_petsc_vec, composite_res_petsc_vec;
ierr = VecCreateMultiVec(PETSC_COMM_WORLD, 2, sol_petsc_vecs, &composite_sol_petsc_vec);
IBTK_CHKERRQ(ierr);
ierr = VecCreateMultiVec(PETSC_COMM_WORLD, 2, rhs_petsc_vecs, &composite_rhs_petsc_vec);
IBTK_CHKERRQ(ierr);
ierr = VecDuplicate(composite_rhs_petsc_vec, &composite_res_petsc_vec);
IBTK_CHKERRQ(ierr);
// Solve the implicit IB equations.
d_ib_implicit_ops->preprocessSolveFluidEquations(current_time, new_time, cycle_num);
SNES snes;
ierr = SNESCreate(PETSC_COMM_WORLD, &snes);
IBTK_CHKERRQ(ierr);
ierr = SNESSetFunction(snes, composite_res_petsc_vec, compositeIBFunction_SAMRAI, this);
IBTK_CHKERRQ(ierr);
ierr = SNESSetOptionsPrefix(snes, "ib_");
IBTK_CHKERRQ(ierr);
Mat jac;
ierr = VecGetLocalSize(composite_sol_petsc_vec, &n_local);
IBTK_CHKERRQ(ierr);
ierr = MatCreateShell(
PETSC_COMM_WORLD, n_local, n_local, PETSC_DETERMINE, PETSC_DETERMINE, this, &jac);
IBTK_CHKERRQ(ierr);
ierr = MatShellSetOperation(
jac, MATOP_MULT, reinterpret_cast<void (*)(void)>(compositeIBJacobianApply_SAMRAI));
IBTK_CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, jac, jac, compositeIBJacobianSetup_SAMRAI, this);
IBTK_CHKERRQ(ierr);
Mat schur;
ierr = VecGetLocalSize(lag_sol_petsc_vec, &n_local);
IBTK_CHKERRQ(ierr);
ierr = MatCreateShell(
PETSC_COMM_WORLD, n_local, n_local, PETSC_DETERMINE, PETSC_DETERMINE, this, &schur);
IBTK_CHKERRQ(ierr);
ierr = MatShellSetOperation(
schur, MATOP_MULT, reinterpret_cast<void (*)(void)>(lagrangianSchurApply_SAMRAI));
IBTK_CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD, &d_schur_solver);
IBTK_CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(d_schur_solver, "ib_schur_");
IBTK_CHKERRQ(ierr);
ierr = KSPSetOperators(d_schur_solver, schur, schur, SAME_PRECONDITIONER);
IBTK_CHKERRQ(ierr);
PC schur_pc;
ierr = KSPGetPC(d_schur_solver, &schur_pc);
IBTK_CHKERRQ(ierr);
ierr = PCSetType(schur_pc, PCNONE);
IBTK_CHKERRQ(ierr);
ierr = KSPSetFromOptions(d_schur_solver);
IBTK_CHKERRQ(ierr);
KSP snes_ksp;
ierr = SNESGetKSP(snes, &snes_ksp);
IBTK_CHKERRQ(ierr);
ierr = KSPSetType(snes_ksp, KSPFGMRES);
IBTK_CHKERRQ(ierr);
PC snes_pc;
ierr = KSPGetPC(snes_ksp, &snes_pc);
IBTK_CHKERRQ(ierr);
ierr = PCSetType(snes_pc, PCSHELL);
IBTK_CHKERRQ(ierr);
ierr = PCShellSetContext(snes_pc, this);
IBTK_CHKERRQ(ierr);
ierr = PCShellSetApply(snes_pc, compositeIBPCApply_SAMRAI);
IBTK_CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);
IBTK_CHKERRQ(ierr);
ierr = SNESSolve(snes, composite_rhs_petsc_vec, composite_sol_petsc_vec);
IBTK_CHKERRQ(ierr);
ierr = SNESDestroy(&snes);
IBTK_CHKERRQ(ierr);
ierr = MatDestroy(&jac);
IBTK_CHKERRQ(ierr);
ierr = MatDestroy(&schur);
IBTK_CHKERRQ(ierr);
ierr = KSPDestroy(&d_schur_solver);
IBTK_CHKERRQ(ierr);
d_ib_implicit_ops->postprocessSolveFluidEquations(current_time, new_time, cycle_num);
// Reset Eulerian solver vectors and Eulerian state data.
ins_hier_integrator->resetSolverVectors(
eul_sol_vec, eul_rhs_vec, current_time, new_time, cycle_num);
// Interpolate the Eulerian velocity to the curvilinear mesh.
d_ib_implicit_ops->setUpdatedPosition(lag_sol_petsc_vec);
#if 0
d_hier_velocity_data_ops->linearSum(d_u_idx, 0.5, u_current_idx, 0.5, u_new_idx);
if (d_enable_logging) plog << d_object_name << "::integrateHierarchy(): interpolating Eulerian velocity to the Lagrangian mesh\n";
d_ib_implicit_ops->interpolateVelocity(d_u_idx, getCoarsenSchedules(d_object_name+"::u::CONSERVATIVE_COARSEN"), getGhostfillRefineSchedules(d_object_name+"::u"), half_time);
// Compute the final value of the updated positions of the Lagrangian
// structure.
d_ib_implicit_ops->midpointStep(current_time, new_time);
#endif
// Deallocate temporary data.
ierr = VecDestroy(&composite_sol_petsc_vec);
IBTK_CHKERRQ(ierr);
ierr = VecDestroy(&composite_rhs_petsc_vec);
IBTK_CHKERRQ(ierr);
ierr = VecDestroy(&composite_res_petsc_vec);
IBTK_CHKERRQ(ierr);
PETScSAMRAIVectorReal::destroyPETScVector(eul_sol_petsc_vec);
PETScSAMRAIVectorReal::destroyPETScVector(eul_rhs_petsc_vec);
eul_rhs_vec->freeVectorComponents();
d_u_scratch_vec->freeVectorComponents();
d_f_scratch_vec->freeVectorComponents();
ierr = VecDestroy(&lag_sol_petsc_vec);
IBTK_CHKERRQ(ierr);
ierr = VecDestroy(&lag_rhs_petsc_vec);
IBTK_CHKERRQ(ierr);
// Execute any registered callbacks.
executeIntegrateHierarchyCallbackFcns(current_time, new_time, cycle_num);
return;
} // integrateHierarchy
