static PetscErrorCode KSPPGMRESCycle(PetscInt *itcount,KSP ksp)
{
KSP_PGMRES *pgmres = (KSP_PGMRES*)(ksp->data);
PetscReal res_norm,res,newnorm;
PetscErrorCode ierr;
PetscInt it = 0,j,k;
PetscBool hapend = PETSC_FALSE;
PetscFunctionBegin;
if (itcount) *itcount = 0;
ierr = VecNormalize(VEC_VV(0),&res_norm);CHKERRQ(ierr);
res = res_norm;
*RS(0) = res_norm;
/* check for the convergence */
ierr = PetscObjectAMSTakeAccess((PetscObject)ksp);CHKERRQ(ierr);
ksp->rnorm = res;
ierr = PetscObjectAMSGrantAccess((PetscObject)ksp);CHKERRQ(ierr);
pgmres->it = it-2;
ierr = KSPLogResidualHistory(ksp,res);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,ksp->its,res);CHKERRQ(ierr);
if (!res) {
ksp->reason = KSP_CONVERGED_ATOL;
ierr = PetscInfo(ksp,"Converged due to zero residual norm on entry\n");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
ierr = (*ksp->converged)(ksp,ksp->its,res,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
for (; !ksp->reason; it++) {
Vec Zcur,Znext;
if (pgmres->vv_allocated <= it + VEC_OFFSET + 1) {
ierr = KSPGMRESGetNewVectors(ksp,it+1);CHKERRQ(ierr);
}
/* VEC_VV(it-1) is orthogonal, it will be normalized once the VecNorm arrives. */
Zcur = VEC_VV(it); /* Zcur is not yet orthogonal, but the VecMDot to orthogonalize it has been started. */
Znext = VEC_VV(it+1); /* This iteration will compute Znext, update with a deferred correction once we know how
* Zcur relates to the previous vectors, and start the reduction to orthogonalize it. */
if (it < pgmres->max_k+1 && ksp->its+1 < PetscMax(2,ksp->max_it)) { /* We don't know whether what we have computed is enough, so apply the matrix. */
ierr = KSP_PCApplyBAorAB(ksp,Zcur,Znext,VEC_TEMP_MATOP);CHKERRQ(ierr);
}
if (it > 1) { /* Complete the pending reduction */
ierr = VecNormEnd(VEC_VV(it-1),NORM_2,&newnorm);CHKERRQ(ierr);
*HH(it-1,it-2) = newnorm;
}
if (it > 0) { /* Finish the reduction computing the latest column of H */
ierr = VecMDotEnd(Zcur,it,&(VEC_VV(0)),HH(0,it-1));CHKERRQ(ierr);
}
if (it > 1) {
/* normalize the base vector from two iterations ago, basis is complete up to here */
ierr = VecScale(VEC_VV(it-1),1./ *HH(it-1,it-2));CHKERRQ(ierr);
ierr = KSPPGMRESUpdateHessenberg(ksp,it-2,&hapend,&res);CHKERRQ(ierr);
pgmres->it = it-2;
ksp->its++;
ksp->rnorm = res;
ierr = (*ksp->converged)(ksp,ksp->its,res,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
if (it < pgmres->max_k+1 || ksp->reason || ksp->its == ksp->max_it) { /* Monitor if we are done or still iterating, but not before a restart. */
ierr = KSPLogResidualHistory(ksp,res);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,ksp->its,res);CHKERRQ(ierr);
}
if (ksp->reason) break;
/* Catch error in happy breakdown and signal convergence and break from loop */
if (hapend) {
if (ksp->errorifnotconverged) SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_NOT_CONVERGED,"You reached the happy break down, but convergence was not indicated. Residual norm = %G",res);
else {
ksp->reason = KSP_DIVERGED_BREAKDOWN;
break;
}
}
if (!(it < pgmres->max_k+1 && ksp->its < ksp->max_it)) break;
/* The it-2 column of H was not scaled when we computed Zcur, apply correction */
ierr = VecScale(Zcur,1./ *HH(it-1,it-2));CHKERRQ(ierr);
/* And Znext computed in this iteration was computed using the under-scaled Zcur */
ierr = VecScale(Znext,1./ *HH(it-1,it-2));CHKERRQ(ierr);
/* In the previous iteration, we projected an unnormalized Zcur against the Krylov basis, so we need to fix the column of H resulting from that projection. */
for (k=0; k<it; k++) *HH(k,it-1) /= *HH(it-1,it-2);
/* When Zcur was projected against the Krylov basis, VV(it-1) was still not normalized, so fix that too. This
* column is complete except for HH(it,it-1) which we won't know until the next iteration. */
*HH(it-1,it-1) /= *HH(it-1,it-2);
}
if (it > 0) {
PetscScalar *work;
if (!pgmres->orthogwork) {ierr = PetscMalloc((pgmres->max_k + 2)*sizeof(PetscScalar),&pgmres->orthogwork);CHKERRQ(ierr);}
work = pgmres->orthogwork;
/* Apply correction computed by the VecMDot in the last iteration to Znext. The original form is
*
* Znext -= sum_{j=0}^{i-1} Z[j+1] * H[j,i-1]
*
* where
*
* Z[j] = sum_{k=0}^j V[k] * H[k,j-1]
*
* substituting
*
* Znext -= sum_{j=0}^{i-1} sum_{k=0}^{j+1} V[k] * H[k,j] * H[j,i-1]
*
* rearranging the iteration space from row-column to column-row
*
* Znext -= sum_{k=0}^i sum_{j=k-1}^{i-1} V[k] * H[k,j] * H[j,i-1]
*
* Note that column it-1 of HH is correct. For all previous columns, we must look at HES because HH has already
* been transformed to upper triangular form.
*/
for (k=0; k<it+1; k++) {
work[k] = 0;
for (j=PetscMax(0,k-1); j<it-1; j++) work[k] -= *HES(k,j) * *HH(j,it-1);
}
ierr = VecMAXPY(Znext,it+1,work,&VEC_VV(0));CHKERRQ(ierr);
ierr = VecAXPY(Znext,-*HH(it-1,it-1),Zcur);CHKERRQ(ierr);
/* Orthogonalize Zcur against existing basis vectors. */
for (k=0; k<it; k++) work[k] = -*HH(k,it-1);
ierr = VecMAXPY(Zcur,it,work,&VEC_VV(0));CHKERRQ(ierr);
/* Zcur is now orthogonal, and will be referred to as VEC_VV(it) again, though it is still not normalized. */
/* Begin computing the norm of the new vector, will be normalized after the MatMult in the next iteration. */
ierr = VecNormBegin(VEC_VV(it),NORM_2,&newnorm);CHKERRQ(ierr);
}
/* Compute column of H (to the diagonal, but not the subdiagonal) to be able to orthogonalize the newest vector. */
ierr = VecMDotBegin(Znext,it+1,&VEC_VV(0),HH(0,it));CHKERRQ(ierr);
/* Start an asynchronous split-mode reduction, the result of the MDot and Norm will be collected on the next iteration. */
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)Znext));CHKERRQ(ierr);
}
if (itcount) *itcount = it-1; /* Number of iterations actually completed. */
/*
Down here we have to solve for the "best" coefficients of the Krylov
columns, add the solution values together, and possibly unwind the
preconditioning from the solution
*/
/* Form the solution (or the solution so far) */
ierr = KSPPGMRESBuildSoln(RS(0),ksp->vec_sol,ksp->vec_sol,ksp,it-2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode KSPSolve_PIPEFCG_cycle(KSP ksp)
{
PetscErrorCode ierr;
PetscInt i,j,k,idx,kdx,mi;
KSP_PIPEFCG *pipefcg;
PetscScalar alpha=0.0,gamma,*betas,*dots;
PetscReal dp=0.0, delta,*eta,*etas;
Vec B,R,Z,X,Qcurr,W,ZETAcurr,M,N,Pcurr,Scurr,*redux;
Mat Amat,Pmat;
PetscFunctionBegin;
/* We have not checked these routines for use with complex numbers. The inner products
are likely not defined correctly for that case */
#if (defined(PETSC_USE_COMPLEX) && !defined(PETSC_SKIP_COMPLEX))
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"PIPEFGMRES has not been implemented for use with complex scalars");
#endif
#define VecXDot(x,y,a) (((pipefcg->type) == (KSP_CG_HERMITIAN)) ? VecDot (x,y,a) : VecTDot (x,y,a))
#define VecXDotBegin(x,y,a) (((pipefcg->type) == (KSP_CG_HERMITIAN)) ? VecDotBegin (x,y,a) : VecTDotBegin (x,y,a))
#define VecXDotEnd(x,y,a) (((pipefcg->type) == (KSP_CG_HERMITIAN)) ? VecDotEnd (x,y,a) : VecTDotEnd (x,y,a))
#define VecMXDot(x,n,y,a) (((pipefcg->type) == (KSP_CG_HERMITIAN)) ? VecMDot (x,n,y,a) : VecMTDot (x,n,y,a))
#define VecMXDotBegin(x,n,y,a) (((pipefcg->type) == (KSP_CG_HERMITIAN)) ? VecMDotBegin (x,n,y,a) : VecMTDotBegin (x,n,y,a))
#define VecMXDotEnd(x,n,y,a) (((pipefcg->type) == (KSP_CG_HERMITIAN)) ? VecMDotEnd (x,n,y,a) : VecMTDotEnd (x,n,y,a))
pipefcg = (KSP_PIPEFCG*)ksp->data;
X = ksp->vec_sol;
B = ksp->vec_rhs;
R = ksp->work[0];
Z = ksp->work[1];
W = ksp->work[2];
M = ksp->work[3];
N = ksp->work[4];
redux = pipefcg->redux;
dots = pipefcg->dots;
etas = pipefcg->etas;
betas = dots; /* dots takes the result of all dot products of which the betas are a subset */
ierr = PCGetOperators(ksp->pc,&Amat,&Pmat);CHKERRQ(ierr);
/* Compute cycle initial residual */
ierr = KSP_MatMult(ksp,Amat,X,R);CHKERRQ(ierr);
ierr = VecAYPX(R,-1.0,B);CHKERRQ(ierr); /* r <- b - Ax */
ierr = KSP_PCApply(ksp,R,Z);CHKERRQ(ierr); /* z <- Br */
Pcurr = pipefcg->Pvecs[0];
Scurr = pipefcg->Svecs[0];
Qcurr = pipefcg->Qvecs[0];
ZETAcurr = pipefcg->ZETAvecs[0];
ierr = VecCopy(Z,Pcurr);CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,Pcurr,Scurr);CHKERRQ(ierr); /* S = Ap */
ierr = VecCopy(Scurr,W);CHKERRQ(ierr); /* w = s = Az */
/* Initial state of pipelining intermediates */
redux[0] = R;
redux[1] = W;
ierr = VecMXDotBegin(Z,2,redux,dots);CHKERRQ(ierr);
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)Z));CHKERRQ(ierr); /* perform asynchronous reduction */
ierr = KSP_PCApply(ksp,W,M);CHKERRQ(ierr); /* m = B(w) */
ierr = KSP_MatMult(ksp,Amat,M,N);CHKERRQ(ierr); /* n = Am */
ierr = VecCopy(M,Qcurr);CHKERRQ(ierr); /* q = m */
ierr = VecCopy(N,ZETAcurr);CHKERRQ(ierr); /* zeta = n */
ierr = VecMXDotEnd(Z,2,redux,dots);CHKERRQ(ierr);
gamma = dots[0];
delta = PetscRealPart(dots[1]);
etas[0] = delta;
alpha = gamma/delta;
i = 0;
do {
ksp->its++;
/* Update X, R, Z, W */
ierr = VecAXPY(X,+alpha,Pcurr);CHKERRQ(ierr); /* x <- x + alpha * pi */
ierr = VecAXPY(R,-alpha,Scurr);CHKERRQ(ierr); /* r <- r - alpha * si */
ierr = VecAXPY(Z,-alpha,Qcurr);CHKERRQ(ierr); /* z <- z - alpha * qi */
ierr = VecAXPY(W,-alpha,ZETAcurr);CHKERRQ(ierr); /* w <- w - alpha * zetai */
/* Compute norm for convergence check */
switch (ksp->normtype) {
case KSP_NORM_PRECONDITIONED:
ierr = VecNorm(Z,NORM_2,&dp);CHKERRQ(ierr); /* dp <- sqrt(z'*z) = sqrt(e'*A'*B'*B*A*e) */
break;
case KSP_NORM_UNPRECONDITIONED:
ierr = VecNorm(R,NORM_2,&dp);CHKERRQ(ierr); /* dp <- sqrt(r'*r) = sqrt(e'*A'*A*e) */
break;
case KSP_NORM_NATURAL:
dp = PetscSqrtReal(PetscAbsScalar(gamma)); /* dp <- sqrt(r'*z) = sqrt(e'*A'*B*A*e) */
break;
case KSP_NORM_NONE:
dp = 0.0;
break;
default: SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"%s",KSPNormTypes[ksp->normtype]);
}
/* Check for convergence */
ksp->rnorm = dp;
KSPLogResidualHistory(ksp,dp);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,ksp->its,dp);CHKERRQ(ierr);
ierr = (*ksp->converged)(ksp,ksp->its+1,dp,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
if (ksp->reason) break;
/* Computations of current iteration done */
++i;
/* If needbe, allocate a new chunk of vectors in P and C */
ierr = KSPAllocateVectors_PIPEFCG(ksp,i+1,pipefcg->vecb);CHKERRQ(ierr);
/* Note that we wrap around and start clobbering old vectors */
idx = i % (pipefcg->mmax+1);
Pcurr = pipefcg->Pvecs[idx];
Scurr = pipefcg->Svecs[idx];
Qcurr = pipefcg->Qvecs[idx];
ZETAcurr = pipefcg->ZETAvecs[idx];
eta = pipefcg->etas+idx;
/* number of old directions to orthogonalize against */
switch(pipefcg->truncstrat){
case KSP_FCD_TRUNC_TYPE_STANDARD:
mi = pipefcg->mmax;
break;
case KSP_FCD_TRUNC_TYPE_NOTAY:
mi = ((i-1) % pipefcg->mmax)+1;
break;
default:
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Unrecognized Truncation Strategy");
}
/* Pick old p,s,q,zeta in a way suitable for VecMDot */
ierr = VecCopy(Z,Pcurr);CHKERRQ(ierr);
for(k=PetscMax(0,i-mi),j=0;k<i;++j,++k){
kdx = k % (pipefcg->mmax+1);
pipefcg->Pold[j] = pipefcg->Pvecs[kdx];
pipefcg->Sold[j] = pipefcg->Svecs[kdx];
pipefcg->Qold[j] = pipefcg->Qvecs[kdx];
pipefcg->ZETAold[j] = pipefcg->ZETAvecs[kdx];
redux[j] = pipefcg->Svecs[kdx];
}
redux[j] = R; /* If the above loop is not executed redux contains only R => all beta_k = 0, only gamma, delta != 0 */
redux[j+1] = W;
ierr = VecMXDotBegin(Z,j+2,redux,betas);CHKERRQ(ierr); /* Start split reductions for beta_k = (z,s_k), gamma = (z,r), delta = (z,w) */
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)Z));CHKERRQ(ierr); /* perform asynchronous reduction */
ierr = VecWAXPY(N,-1.0,R,W);CHKERRQ(ierr); /* m = u + B(w-r): (a) ntmp = w-r */
ierr = KSP_PCApply(ksp,N,M);CHKERRQ(ierr); /* m = u + B(w-r): (b) mtmp = B(ntmp) = B(w-r) */
ierr = VecAXPY(M,1.0,Z);CHKERRQ(ierr); /* m = u + B(w-r): (c) m = z + mtmp */
ierr = KSP_MatMult(ksp,Amat,M,N);CHKERRQ(ierr); /* n = Am */
ierr = VecMXDotEnd(Z,j+2,redux,betas);CHKERRQ(ierr); /* Finish split reductions */
gamma = betas[j];
delta = PetscRealPart(betas[j+1]);
*eta = 0.;
for(k=PetscMax(0,i-mi),j=0;k<i;++j,++k){
kdx = k % (pipefcg->mmax+1);
betas[j] /= -etas[kdx]; /* betak /= etak */
*eta -= ((PetscReal)(PetscAbsScalar(betas[j])*PetscAbsScalar(betas[j]))) * etas[kdx];
/* etaitmp = -betaik^2 * etak */
}
*eta += delta; /* etai = delta -betaik^2 * etak */
if(*eta < 0.) {
pipefcg->norm_breakdown = PETSC_TRUE;
ierr = PetscInfo1(ksp,"Restart due to square root breakdown at it = \n",ksp->its);CHKERRQ(ierr);
break;
} else {
alpha= gamma/(*eta); /* alpha = gamma/etai */
}
/* project out stored search directions using classical G-S */
ierr = VecCopy(Z,Pcurr);CHKERRQ(ierr);
ierr = VecCopy(W,Scurr);CHKERRQ(ierr);
ierr = VecCopy(M,Qcurr);CHKERRQ(ierr);
ierr = VecCopy(N,ZETAcurr);CHKERRQ(ierr);
ierr = VecMAXPY(Pcurr ,j,betas,pipefcg->Pold);CHKERRQ(ierr); /* pi <- ui - sum_k beta_k p_k */
ierr = VecMAXPY(Scurr ,j,betas,pipefcg->Sold);CHKERRQ(ierr); /* si <- wi - sum_k beta_k s_k */
ierr = VecMAXPY(Qcurr ,j,betas,pipefcg->Qold);CHKERRQ(ierr); /* qi <- m - sum_k beta_k q_k */
ierr = VecMAXPY(ZETAcurr,j,betas,pipefcg->ZETAold);CHKERRQ(ierr); /* zetai <- n - sum_k beta_k zeta_k */
} while (ksp->its < ksp->max_it);
PetscFunctionReturn(0);
}
PetscErrorCode KSPSolve_GROPPCG(KSP ksp)
{
PetscErrorCode ierr;
PetscInt i;
PetscScalar alpha,beta = 0.0,gamma,gammaNew,t;
PetscReal dp = 0.0;
Vec x,b,r,p,s,S,z,Z;
Mat Amat,Pmat;
PetscBool diagonalscale;
PetscFunctionBegin;
ierr = PCGetDiagonalScale(ksp->pc,&diagonalscale);CHKERRQ(ierr);
if (diagonalscale) SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"Krylov method %s does not support diagonal scaling",((PetscObject)ksp)->type_name);
x = ksp->vec_sol;
b = ksp->vec_rhs;
r = ksp->work[0];
p = ksp->work[1];
s = ksp->work[2];
S = ksp->work[3];
z = ksp->work[4];
Z = ksp->work[5];
ierr = PCGetOperators(ksp->pc,&Amat,&Pmat);CHKERRQ(ierr);
ksp->its = 0;
if (!ksp->guess_zero) {
ierr = KSP_MatMult(ksp,Amat,x,r);CHKERRQ(ierr); /* r <- b - Ax */
ierr = VecAYPX(r,-1.0,b);CHKERRQ(ierr);
} else {
ierr = VecCopy(b,r);CHKERRQ(ierr); /* r <- b (x is 0) */
}
ierr = KSP_PCApply(ksp,r,z);CHKERRQ(ierr); /* z <- Br */
ierr = VecCopy(z,p);CHKERRQ(ierr); /* p <- z */
ierr = VecDotBegin(r,z,&gamma);CHKERRQ(ierr); /* gamma <- z'*r */
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)r));CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,p,s);CHKERRQ(ierr); /* s <- Ap */
ierr = VecDotEnd(r,z,&gamma);CHKERRQ(ierr); /* gamma <- z'*r */
switch (ksp->normtype) {
case KSP_NORM_PRECONDITIONED:
/* This could be merged with the computation of gamma above */
ierr = VecNorm(z,NORM_2,&dp);CHKERRQ(ierr); /* dp <- z'*z = e'*A'*B'*B*A'*e' */
break;
case KSP_NORM_UNPRECONDITIONED:
/* This could be merged with the computation of gamma above */
ierr = VecNorm(r,NORM_2,&dp);CHKERRQ(ierr); /* dp <- r'*r = e'*A'*A*e */
break;
case KSP_NORM_NATURAL:
if (PetscIsInfOrNanScalar(gamma)) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_FP,"Infinite or not-a-number generated in dot product");
dp = PetscSqrtReal(PetscAbsScalar(gamma)); /* dp <- r'*z = r'*B*r = e'*A'*B*A*e */
break;
case KSP_NORM_NONE:
dp = 0.0;
break;
default: SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"%s",KSPNormTypes[ksp->normtype]);
}
ierr = KSPLogResidualHistory(ksp,dp);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,0,dp);CHKERRQ(ierr);
ksp->rnorm = dp;
ierr = (*ksp->converged)(ksp,0,dp,&ksp->reason,ksp->cnvP);CHKERRQ(ierr); /* test for convergence */
if (ksp->reason) PetscFunctionReturn(0);
i = 0;
do {
ksp->its = i+1;
i++;
ierr = VecDotBegin(p,s,&t);CHKERRQ(ierr);
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)p));CHKERRQ(ierr);
ierr = KSP_PCApply(ksp,s,S);CHKERRQ(ierr); /* S <- Bs */
ierr = VecDotEnd(p,s,&t);CHKERRQ(ierr);
alpha = gamma / t;
ierr = VecAXPY(x, alpha,p);CHKERRQ(ierr); /* x <- x + alpha * p */
ierr = VecAXPY(r,-alpha,s);CHKERRQ(ierr); /* r <- r - alpha * s */
ierr = VecAXPY(z,-alpha,S);CHKERRQ(ierr); /* z <- z - alpha * S */
if (ksp->normtype == KSP_NORM_UNPRECONDITIONED) {
ierr = VecNormBegin(r,NORM_2,&dp);CHKERRQ(ierr);
} else if (ksp->normtype == KSP_NORM_PRECONDITIONED) {
ierr = VecNormBegin(z,NORM_2,&dp);CHKERRQ(ierr);
}
ierr = VecDotBegin(r,z,&gammaNew);CHKERRQ(ierr);
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)r));CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,z,Z);CHKERRQ(ierr); /* Z <- Az */
if (ksp->normtype == KSP_NORM_UNPRECONDITIONED) {
ierr = VecNormEnd(r,NORM_2,&dp);CHKERRQ(ierr);
} else if (ksp->normtype == KSP_NORM_PRECONDITIONED) {
ierr = VecNormEnd(z,NORM_2,&dp);CHKERRQ(ierr);
}
ierr = VecDotEnd(r,z,&gammaNew);CHKERRQ(ierr);
if (ksp->normtype == KSP_NORM_NATURAL) {
if (PetscIsInfOrNanScalar(gammaNew)) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_FP,"Infinite or not-a-number generated in dot product");
dp = PetscSqrtReal(PetscAbsScalar(gammaNew)); /* dp <- r'*z = r'*B*r = e'*A'*B*A*e */
} else if (ksp->normtype == KSP_NORM_NONE) {
dp = 0.0;
}
ksp->rnorm = dp;
ierr = KSPLogResidualHistory(ksp,dp);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,i,dp);CHKERRQ(ierr);
ierr = (*ksp->converged)(ksp,i,dp,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
if (ksp->reason) break;
beta = gammaNew / gamma;
gamma = gammaNew;
ierr = VecAYPX(p,beta,z);CHKERRQ(ierr); /* p <- z + beta * p */
ierr = VecAYPX(s,beta,Z);CHKERRQ(ierr); /* s <- Z + beta * s */
} while (i<ksp->max_it);
if (i >= ksp->max_it) ksp->reason = KSP_DIVERGED_ITS;
PetscFunctionReturn(0);
}
PetscErrorCode KSPSolve_PIPECG(KSP ksp)
{
PetscErrorCode ierr;
PetscInt i;
PetscScalar alpha = 0.0,beta = 0.0,gamma = 0.0,gammaold = 0.0,delta = 0.0;
PetscReal dp = 0.0;
Vec X,B,Z,P,W,Q,U,M,N,R,S;
Mat Amat,Pmat;
PetscBool diagonalscale;
PetscFunctionBegin;
ierr = PCGetDiagonalScale(ksp->pc,&diagonalscale);CHKERRQ(ierr);
if (diagonalscale) SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"Krylov method %s does not support diagonal scaling",((PetscObject)ksp)->type_name);
X = ksp->vec_sol;
B = ksp->vec_rhs;
M = ksp->work[0];
Z = ksp->work[1];
P = ksp->work[2];
N = ksp->work[3];
W = ksp->work[4];
Q = ksp->work[5];
U = ksp->work[6];
R = ksp->work[7];
S = ksp->work[8];
ierr = PCGetOperators(ksp->pc,&Amat,&Pmat);CHKERRQ(ierr);
ksp->its = 0;
if (!ksp->guess_zero) {
ierr = KSP_MatMult(ksp,Amat,X,R);CHKERRQ(ierr); /* r <- b - Ax */
ierr = VecAYPX(R,-1.0,B);CHKERRQ(ierr);
} else {
ierr = VecCopy(B,R);CHKERRQ(ierr); /* r <- b (x is 0) */
}
ierr = KSP_PCApply(ksp,R,U);CHKERRQ(ierr); /* u <- Br */
switch (ksp->normtype) {
case KSP_NORM_PRECONDITIONED:
ierr = VecNormBegin(U,NORM_2,&dp);CHKERRQ(ierr); /* dp <- u'*u = e'*A'*B'*B*A'*e' */
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)U));CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,U,W);CHKERRQ(ierr); /* w <- Au */
ierr = VecNormEnd(U,NORM_2,&dp);CHKERRQ(ierr);
break;
case KSP_NORM_UNPRECONDITIONED:
ierr = VecNormBegin(R,NORM_2,&dp);CHKERRQ(ierr); /* dp <- r'*r = e'*A'*A*e */
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)R));CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,U,W);CHKERRQ(ierr); /* w <- Au */
ierr = VecNormEnd(R,NORM_2,&dp);CHKERRQ(ierr);
break;
case KSP_NORM_NATURAL:
ierr = VecDotBegin(R,U,&gamma);CHKERRQ(ierr); /* gamma <- u'*r */
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)R));CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,U,W);CHKERRQ(ierr); /* w <- Au */
ierr = VecDotEnd(R,U,&gamma);CHKERRQ(ierr);
KSPCheckDot(ksp,gamma);
dp = PetscSqrtReal(PetscAbsScalar(gamma)); /* dp <- r'*u = r'*B*r = e'*A'*B*A*e */
break;
case KSP_NORM_NONE:
ierr = KSP_MatMult(ksp,Amat,U,W);CHKERRQ(ierr);
dp = 0.0;
break;
default: SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"%s",KSPNormTypes[ksp->normtype]);
}
ierr = KSPLogResidualHistory(ksp,dp);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,0,dp);CHKERRQ(ierr);
ksp->rnorm = dp;
ierr = (*ksp->converged)(ksp,0,dp,&ksp->reason,ksp->cnvP);CHKERRQ(ierr); /* test for convergence */
if (ksp->reason) PetscFunctionReturn(0);
i = 0;
do {
if (i > 0 && ksp->normtype == KSP_NORM_UNPRECONDITIONED) {
ierr = VecNormBegin(R,NORM_2,&dp);CHKERRQ(ierr);
} else if (i > 0 && ksp->normtype == KSP_NORM_PRECONDITIONED) {
ierr = VecNormBegin(U,NORM_2,&dp);CHKERRQ(ierr);
}
if (!(i == 0 && ksp->normtype == KSP_NORM_NATURAL)) {
ierr = VecDotBegin(R,U,&gamma);CHKERRQ(ierr);
}
ierr = VecDotBegin(W,U,&delta);CHKERRQ(ierr);
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)R));CHKERRQ(ierr);
ierr = KSP_PCApply(ksp,W,M);CHKERRQ(ierr); /* m <- Bw */
ierr = KSP_MatMult(ksp,Amat,M,N);CHKERRQ(ierr); /* n <- Am */
if (i > 0 && ksp->normtype == KSP_NORM_UNPRECONDITIONED) {
ierr = VecNormEnd(R,NORM_2,&dp);CHKERRQ(ierr);
} else if (i > 0 && ksp->normtype == KSP_NORM_PRECONDITIONED) {
ierr = VecNormEnd(U,NORM_2,&dp);CHKERRQ(ierr);
}
if (!(i == 0 && ksp->normtype == KSP_NORM_NATURAL)) {
ierr = VecDotEnd(R,U,&gamma);CHKERRQ(ierr);
}
ierr = VecDotEnd(W,U,&delta);CHKERRQ(ierr);
if (i > 0) {
if (ksp->normtype == KSP_NORM_NATURAL) dp = PetscSqrtReal(PetscAbsScalar(gamma));
else if (ksp->normtype == KSP_NORM_NONE) dp = 0.0;
ksp->rnorm = dp;
ierr = KSPLogResidualHistory(ksp,dp);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,i,dp);CHKERRQ(ierr);
ierr = (*ksp->converged)(ksp,i,dp,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
if (ksp->reason) break;
}
if (i == 0) {
alpha = gamma / delta;
ierr = VecCopy(N,Z);CHKERRQ(ierr); /* z <- n */
ierr = VecCopy(M,Q);CHKERRQ(ierr); /* q <- m */
ierr = VecCopy(U,P);CHKERRQ(ierr); /* p <- u */
ierr = VecCopy(W,S);CHKERRQ(ierr); /* s <- w */
} else {
beta = gamma / gammaold;
alpha = gamma / (delta - beta / alpha * gamma);
ierr = VecAYPX(Z,beta,N);CHKERRQ(ierr); /* z <- n + beta * z */
ierr = VecAYPX(Q,beta,M);CHKERRQ(ierr); /* q <- m + beta * q */
ierr = VecAYPX(P,beta,U);CHKERRQ(ierr); /* p <- u + beta * p */
ierr = VecAYPX(S,beta,W);CHKERRQ(ierr); /* s <- w + beta * s */
}
ierr = VecAXPY(X, alpha,P);CHKERRQ(ierr); /* x <- x + alpha * p */
ierr = VecAXPY(U,-alpha,Q);CHKERRQ(ierr); /* u <- u - alpha * q */
ierr = VecAXPY(W,-alpha,Z);CHKERRQ(ierr); /* w <- w - alpha * z */
ierr = VecAXPY(R,-alpha,S);CHKERRQ(ierr); /* r <- r - alpha * s */
gammaold = gamma;
i++;
ksp->its = i;
/* if (i%50 == 0) { */
/* ierr = KSP_MatMult(ksp,Amat,X,R);CHKERRQ(ierr); /\* w <- b - Ax *\/ */
/* ierr = VecAYPX(R,-1.0,B);CHKERRQ(ierr); */
/* ierr = KSP_PCApply(ksp,R,U);CHKERRQ(ierr); */
/* ierr = KSP_MatMult(ksp,Amat,U,W);CHKERRQ(ierr); */
/* } */
} while (i<ksp->max_it);
if (i >= ksp->max_it) ksp->reason = KSP_DIVERGED_ITS;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt n = 25,i,row0 = 0;
PetscScalar one = 1.0,two = 2.0,result1,result2,results[40],value,ten = 10.0;
PetscScalar result1a,result2a;
PetscReal result3,result4,result[2],result3a,result4a,resulta[2];
Vec x,y,vecs[40];
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* create vector */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
ierr = VecSet(x,one);CHKERRQ(ierr);
ierr = VecSet(y,two);CHKERRQ(ierr);
/*
Test mixing dot products and norms that require sums
*/
result1 = result2 = 0.0;
result3 = result4 = 0.0;
ierr = VecDotBegin(x,y,&result1);CHKERRQ(ierr);
ierr = VecDotBegin(y,x,&result2);CHKERRQ(ierr);
ierr = VecNormBegin(y,NORM_2,&result3);CHKERRQ(ierr);
ierr = VecNormBegin(x,NORM_1,&result4);CHKERRQ(ierr);
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)x));CHKERRQ(ierr);
ierr = VecDotEnd(x,y,&result1);CHKERRQ(ierr);
ierr = VecDotEnd(y,x,&result2);CHKERRQ(ierr);
ierr = VecNormEnd(y,NORM_2,&result3);CHKERRQ(ierr);
ierr = VecNormEnd(x,NORM_1,&result4);CHKERRQ(ierr);
ierr = VecDot(x,y,&result1a);CHKERRQ(ierr);
ierr = VecDot(y,x,&result2a);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&result3a);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_1,&result4a);CHKERRQ(ierr);
if (result1 != result1a || result2 != result2a) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error dot: result1 %g result2 %g\n",(double)PetscRealPart(result1),(double)PetscRealPart(result2));CHKERRQ(ierr);
}
if (result3 != result3a || result4 != result4a) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error 1,2 norms: result3 %g result4 %g\n",(double)result3,(double)result4);CHKERRQ(ierr);
}
/*
Test norms that only require abs
*/
result1 = result2 = 0.0;
result3 = result4 = 0.0;
ierr = VecNormBegin(y,NORM_MAX,&result3);CHKERRQ(ierr);
ierr = VecNormBegin(x,NORM_MAX,&result4);CHKERRQ(ierr);
ierr = VecNormEnd(y,NORM_MAX,&result3);CHKERRQ(ierr);
ierr = VecNormEnd(x,NORM_MAX,&result4);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_MAX,&result4a);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_MAX,&result3a);CHKERRQ(ierr);
if (result3 != result3a || result4 != result4a) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error max norm: result3 %g result4 %g\n",(double)result3,(double)result4);CHKERRQ(ierr);
}
/*
Tests dot, max, 1, norm
*/
result1 = result2 = 0.0;
result3 = result4 = 0.0;
ierr = VecSetValues(x,1,&row0,&ten,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecDotBegin(x,y,&result1);CHKERRQ(ierr);
ierr = VecDotBegin(y,x,&result2);CHKERRQ(ierr);
ierr = VecNormBegin(x,NORM_MAX,&result3);CHKERRQ(ierr);
ierr = VecNormBegin(x,NORM_1,&result4);CHKERRQ(ierr);
ierr = VecDotEnd(x,y,&result1);CHKERRQ(ierr);
ierr = VecDotEnd(y,x,&result2);CHKERRQ(ierr);
ierr = VecNormEnd(x,NORM_MAX,&result3);CHKERRQ(ierr);
ierr = VecNormEnd(x,NORM_1,&result4);CHKERRQ(ierr);
ierr = VecDot(x,y,&result1a);CHKERRQ(ierr);
ierr = VecDot(y,x,&result2a);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_MAX,&result3a);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_1,&result4a);CHKERRQ(ierr);
if (result1 != result1a || result2 != result2a) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error dot: result1 %g result2 %g\n",(double)PetscRealPart(result1),(double)PetscRealPart(result2));CHKERRQ(ierr);
}
if (result3 != result3a || result4 != result4a) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error max 1 norms: result3 %g result4 %g\n",(double)result3,(double)result4);CHKERRQ(ierr);
}
/*
tests 1_and_2 norm
*/
ierr = VecNormBegin(x,NORM_MAX,&result3);CHKERRQ(ierr);
ierr = VecNormBegin(x,NORM_1_AND_2,result);CHKERRQ(ierr);
ierr = VecNormBegin(y,NORM_MAX,&result4);CHKERRQ(ierr);
ierr = VecNormEnd(x,NORM_MAX,&result3);CHKERRQ(ierr);
ierr = VecNormEnd(x,NORM_1_AND_2,result);CHKERRQ(ierr);
ierr = VecNormEnd(y,NORM_MAX,&result4);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_MAX,&result3a);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_1_AND_2,resulta);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_MAX,&result4a);CHKERRQ(ierr);
if (result3 != result3a || result4 != result4a) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error max: result1 %g result2 %g\n",(double)result3,(double)result4);CHKERRQ(ierr);
}
if (PetscAbsReal(result[0]-resulta[0]) > .01 || PetscAbsReal(result[1]-resulta[1]) > .01) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error 1 and 2 norms: result[0] %g result[1] %g\n",(double)result[0],(double)result[1]);CHKERRQ(ierr);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
/*
Tests computing a large number of operations that require
allocating a larger data structure internally
*/
for (i=0; i<40; i++) {
ierr = VecCreate(PETSC_COMM_WORLD,vecs+i);CHKERRQ(ierr);
ierr = VecSetSizes(vecs[i],PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(vecs[i]);CHKERRQ(ierr);
value = (PetscReal)i;
ierr = VecSet(vecs[i],value);CHKERRQ(ierr);
}
for (i=0; i<39; i++) {
ierr = VecDotBegin(vecs[i],vecs[i+1],results+i);CHKERRQ(ierr);
}
for (i=0; i<39; i++) {
ierr = VecDotEnd(vecs[i],vecs[i+1],results+i);CHKERRQ(ierr);
if (results[i] != 25.0*i*(i+1)) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"i %D expected %g got %g\n",i,25.0*i*(i+1),(double)PetscRealPart(results[i]));CHKERRQ(ierr);
}
}
for (i=0; i<40; i++) {
ierr = VecDestroy(&vecs[i]);CHKERRQ(ierr);
}
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode KSPPIPEFGMRESCycle(PetscInt *itcount,KSP ksp)
{
KSP_PIPEFGMRES *pipefgmres = (KSP_PIPEFGMRES*)(ksp->data);
PetscReal res_norm;
PetscReal hapbnd,tt;
PetscScalar *hh,*hes,*lhh,shift = pipefgmres->shift;
PetscBool hapend = PETSC_FALSE; /* indicates happy breakdown ending */
PetscErrorCode ierr;
PetscInt loc_it; /* local count of # of dir. in Krylov space */
PetscInt max_k = pipefgmres->max_k; /* max # of directions Krylov space */
PetscInt i,j,k;
Mat Amat,Pmat;
Vec Q,W; /* Pipelining vectors */
Vec *redux = pipefgmres->redux; /* workspace for single reduction */
PetscFunctionBegin;
if (itcount) *itcount = 0;
/* Assign simpler names to these vectors, allocated as pipelining workspace */
Q = VEC_Q;
W = VEC_W;
/* Allocate memory for orthogonalization work (freed in the GMRES Destroy routine)*/
/* Note that we add an extra value here to allow for a single reduction */
if (!pipefgmres->orthogwork) { ierr = PetscMalloc1(pipefgmres->max_k + 2 ,&pipefgmres->orthogwork);CHKERRQ(ierr);
}
lhh = pipefgmres->orthogwork;
/* Number of pseudo iterations since last restart is the number
of prestart directions */
loc_it = 0;
/* note: (pipefgmres->it) is always set one less than (loc_it) It is used in
KSPBUILDSolution_PIPEFGMRES, where it is passed to KSPPIPEFGMRESBuildSoln.
Note that when KSPPIPEFGMRESBuildSoln is called from this function,
(loc_it -1) is passed, so the two are equivalent */
pipefgmres->it = (loc_it - 1);
/* initial residual is in VEC_VV(0) - compute its norm*/
ierr = VecNorm(VEC_VV(0),NORM_2,&res_norm);CHKERRQ(ierr);
/* first entry in right-hand-side of hessenberg system is just
the initial residual norm */
*RS(0) = res_norm;
ksp->rnorm = res_norm;
ierr = KSPLogResidualHistory(ksp,res_norm);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,ksp->its,res_norm);CHKERRQ(ierr);
/* check for the convergence - maybe the current guess is good enough */
ierr = (*ksp->converged)(ksp,ksp->its,res_norm,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
if (ksp->reason) {
if (itcount) *itcount = 0;
PetscFunctionReturn(0);
}
/* scale VEC_VV (the initial residual) */
ierr = VecScale(VEC_VV(0),1.0/res_norm);CHKERRQ(ierr);
/* Fill the pipeline */
ierr = KSP_PCApply(ksp,VEC_VV(loc_it),PREVEC(loc_it));CHKERRQ(ierr);
ierr = PCGetOperators(ksp->pc,&Amat,&Pmat);CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,PREVEC(loc_it),ZVEC(loc_it));CHKERRQ(ierr);
ierr = VecAXPY(ZVEC(loc_it),-shift,VEC_VV(loc_it));CHKERRQ(ierr); /* Note shift */
/* MAIN ITERATION LOOP BEGINNING*/
/* keep iterating until we have converged OR generated the max number
of directions OR reached the max number of iterations for the method */
while (!ksp->reason && loc_it < max_k && ksp->its < ksp->max_it) {
if (loc_it) {
ierr = KSPLogResidualHistory(ksp,res_norm);CHKERRQ(ierr);
ierr = KSPMonitor(ksp,ksp->its,res_norm);CHKERRQ(ierr);
}
pipefgmres->it = (loc_it - 1);
/* see if more space is needed for work vectors */
if (pipefgmres->vv_allocated <= loc_it + VEC_OFFSET + 1) {
ierr = KSPPIPEFGMRESGetNewVectors(ksp,loc_it+1);CHKERRQ(ierr);
/* (loc_it+1) is passed in as number of the first vector that should
be allocated */
}
/* Note that these inner products are with "Z" now, so
in particular, lhh[loc_it] is the 'barred' or 'shifted' value,
not the value from the equivalent FGMRES run (even in exact arithmetic)
That is, the H we need for the Arnoldi relation is different from the
coefficients we use in the orthogonalization process,because of the shift */
/* Do some local twiddling to allow for a single reduction */
for(i=0;i<loc_it+1;i++){
redux[i] = VEC_VV(i);
}
redux[loc_it+1] = ZVEC(loc_it);
/* note the extra dot product which ends up in lh[loc_it+1], which computes ||z||^2 */
ierr = VecMDotBegin(ZVEC(loc_it),loc_it+2,redux,lhh);CHKERRQ(ierr);
/* Start the split reduction (This actually calls the MPI_Iallreduce, otherwise, the reduction is simply delayed until the "end" call)*/
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)ZVEC(loc_it)));CHKERRQ(ierr);
/* The work to be overlapped with the inner products follows.
This is application of the preconditioner and the operator
to compute intermediate quantites which will be combined (locally)
with the results of the inner products.
*/
ierr = KSP_PCApply(ksp,ZVEC(loc_it),Q);CHKERRQ(ierr);
ierr = PCGetOperators(ksp->pc,&Amat,&Pmat);CHKERRQ(ierr);
ierr = KSP_MatMult(ksp,Amat,Q,W);CHKERRQ(ierr);
/* Compute inner products of the new direction with previous directions,
and the norm of the to-be-orthogonalized direction "Z".
This information is enough to build the required entries
of H. The inner product with VEC_VV(it_loc) is
*different* than in the standard FGMRES and need to be dealt with specially.
That is, for standard FGMRES the orthogonalization coefficients are the same
as the coefficients used in the Arnoldi relation to reconstruct, but here this
is not true (albeit only for the one entry of H which we "unshift" below. */
/* Finish the dot product, retrieving the extra entry */
ierr = VecMDotEnd(ZVEC(loc_it),loc_it+2,redux,lhh);CHKERRQ(ierr);
tt = PetscRealPart(lhh[loc_it+1]);
/* Hessenberg entries, and entries for (naive) classical Graham-Schmidt
Note that the Hessenberg entries require a shift, as these are for the
relation AU = VH, which is wrt unshifted basis vectors */
hh = HH(0,loc_it); hes=HES(0,loc_it);
for (j=0; j<loc_it; j++) {
hh[j] = lhh[j];
hes[j] = lhh[j];
}
hh[loc_it] = lhh[loc_it] + shift;
hes[loc_it] = lhh[loc_it] + shift;
/* we delay applying the shift here */
for (j=0; j<=loc_it; j++) {
lhh[j] = -lhh[j]; /* flip sign */
}
/* Compute the norm of the un-normalized new direction using the rearranged formula
Note that these are shifted ("barred") quantities */
for(k=0;k<=loc_it;k++) tt -= ((PetscReal)(PetscAbsScalar(lhh[k]) * PetscAbsScalar(lhh[k])));
/* On AVX512 this is accumulating roundoff errors for eg: tt=-2.22045e-16 */
if ((tt < 0.0) && tt > -PETSC_SMALL) tt = 0.0 ;
if (tt < 0.0) {
/* If we detect square root breakdown in the norm, we must restart the algorithm.
Here this means we simply break the current loop and reconstruct the solution
using the basis we have computed thus far. Note that by breaking immediately,
we do not update the iteration count, so computation done in this iteration
should be disregarded.
*/
ierr = PetscInfo2(ksp,"Restart due to square root breakdown at it = %D, tt=%g\n",ksp->its,(double)tt);CHKERRQ(ierr);
break;
} else {
tt = PetscSqrtReal(tt);
}
/* new entry in hessenburg is the 2-norm of our new direction */
hh[loc_it+1] = tt;
hes[loc_it+1] = tt;
/* The recurred computation for the new direction
The division by tt is delayed to the happy breakdown check later
Note placement BEFORE the unshift
*/
ierr = VecCopy(ZVEC(loc_it),VEC_VV(loc_it+1));CHKERRQ(ierr);
ierr = VecMAXPY(VEC_VV(loc_it+1),loc_it+1,lhh,&VEC_VV(0));CHKERRQ(ierr);
/* (VEC_VV(loc_it+1) is not normalized yet) */
/* The recurred computation for the preconditioned vector (u) */
ierr = VecCopy(Q,PREVEC(loc_it+1));CHKERRQ(ierr);
ierr = VecMAXPY(PREVEC(loc_it+1),loc_it+1,lhh,&PREVEC(0));CHKERRQ(ierr);
ierr = VecScale(PREVEC(loc_it+1),1.0/tt);CHKERRQ(ierr);
/* Unshift an entry in the GS coefficients ("removing the bar") */
lhh[loc_it] -= shift;
/* The recurred computation for z (Au)
Note placement AFTER the "unshift" */
ierr = VecCopy(W,ZVEC(loc_it+1));CHKERRQ(ierr);
ierr = VecMAXPY(ZVEC(loc_it+1),loc_it+1,lhh,&ZVEC(0));CHKERRQ(ierr);
ierr = VecScale(ZVEC(loc_it+1),1.0/tt);CHKERRQ(ierr);
/* Happy Breakdown Check */
hapbnd = PetscAbsScalar((tt) / *RS(loc_it));
/* RS(loc_it) contains the res_norm from the last iteration */
hapbnd = PetscMin(pipefgmres->haptol,hapbnd);
if (tt > hapbnd) {
/* scale new direction by its norm */
ierr = VecScale(VEC_VV(loc_it+1),1.0/tt);CHKERRQ(ierr);
} else {
/* This happens when the solution is exactly reached. */
/* So there is no new direction... */
ierr = VecSet(VEC_TEMP,0.0);CHKERRQ(ierr); /* set VEC_TEMP to 0 */
hapend = PETSC_TRUE;
}
/* note that for pipefgmres we could get HES(loc_it+1, loc_it) = 0 and the
current solution would not be exact if HES was singular. Note that
HH non-singular implies that HES is not singular, and HES is guaranteed
to be nonsingular when PREVECS are linearly independent and A is
nonsingular (in GMRES, the nonsingularity of A implies the nonsingularity
of HES). So we should really add a check to verify that HES is nonsingular.*/
/* Note that to be thorough, in debug mode, one could call a LAPACK routine
here to check that the hessenberg matrix is indeed non-singular (since
FGMRES does not guarantee this) */
/* Now apply rotations to new col of hessenberg (and right side of system),
calculate new rotation, and get new residual norm at the same time*/
ierr = KSPPIPEFGMRESUpdateHessenberg(ksp,loc_it,&hapend,&res_norm);CHKERRQ(ierr);
if (ksp->reason) break;
loc_it++;
pipefgmres->it = (loc_it-1); /* Add this here in case it has converged */
ierr = PetscObjectSAWsTakeAccess((PetscObject)ksp);CHKERRQ(ierr);
ksp->its++;
ksp->rnorm = res_norm;
ierr = PetscObjectSAWsGrantAccess((PetscObject)ksp);CHKERRQ(ierr);
ierr = (*ksp->converged)(ksp,ksp->its,res_norm,&ksp->reason,ksp->cnvP);CHKERRQ(ierr);
/* Catch error in happy breakdown and signal convergence and break from loop */
if (hapend) {
if (!ksp->reason) {
if (ksp->errorifnotconverged) SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_NOT_CONVERGED,"You reached the happy break down, but convergence was not indicated. Residual norm = %g",(double)res_norm);
else {
ksp->reason = KSP_DIVERGED_BREAKDOWN;
break;
}
}
}
}
/* END OF ITERATION LOOP */
ierr = KSPLogResidualHistory(ksp,res_norm);CHKERRQ(ierr);
/*
Monitor if we know that we will not return for a restart */
if (loc_it && (ksp->reason || ksp->its >= ksp->max_it)) {
ierr = KSPMonitor(ksp,ksp->its,res_norm);CHKERRQ(ierr);
}
if (itcount) *itcount = loc_it;
/*
Down here we have to solve for the "best" coefficients of the Krylov
columns, add the solution values together, and possibly unwind the
preconditioning from the solution
*/
/* Form the solution (or the solution so far) */
/* Note: must pass in (loc_it-1) for iteration count so that KSPPIPEGMRESIIBuildSoln
properly navigates */
ierr = KSPPIPEFGMRESBuildSoln(RS(0),ksp->vec_sol,ksp->vec_sol,ksp,loc_it-1);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt n = 5,N,i;
PetscMPIInt size,rank;
PetscScalar value,zero = 0.0;
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* create two vectors */
N = size*n;
ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr);
ierr = VecSetSizes(y,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(y);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
/* create two index sets */
ierr = ISCreateStride(PETSC_COMM_WORLD,n,n*rank,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_WORLD,n,(n*(rank+1))%N,1,&is2);CHKERRQ(ierr);
/* fill local part of parallel vector x */
value = (PetscScalar)(rank+1);
for (i=n*rank; i<n*(rank+1); i++) {
ierr = VecSetValues(x,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecSet(y,zero);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
for (i=0; i<100; i++) {
PetscReal ynorm;
PetscInt j;
ierr = VecNormBegin(y,NORM_2,&ynorm);CHKERRQ(ierr);
ierr = PetscCommSplitReductionBegin(PetscObjectComm((PetscObject)y));CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecScatterBegin(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
ierr = VecNormEnd(y,NORM_2,&ynorm);CHKERRQ(ierr);
/* ierr = PetscPrintf(PETSC_COMM_WORLD,"ynorm = %8.2G\n",ynorm);CHKERRQ(ierr); */
}
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
