void CG::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b) { profile.Start(QUDA_PROFILE_INIT); // Check to see that we're not trying to invert on a zero-field source const double b2 = norm2(b); if(b2 == 0){ profile.Stop(QUDA_PROFILE_INIT); printfQuda("Warning: inverting on zero-field source\n"); x=b; param.true_res = 0.0; param.true_res_hq = 0.0; return; } cudaColorSpinorField r(b); ColorSpinorParam csParam(x); csParam.create = QUDA_ZERO_FIELD_CREATE; cudaColorSpinorField y(b, csParam); mat(r, x, y); // zeroCuda(y); double r2 = xmyNormCuda(b, r); csParam.setPrecision(param.precision_sloppy); cudaColorSpinorField Ap(x, csParam); cudaColorSpinorField tmp(x, csParam); cudaColorSpinorField *tmp2_p = &tmp; // tmp only needed for multi-gpu Wilson-like kernels if (mat.Type() != typeid(DiracStaggeredPC).name() && mat.Type() != typeid(DiracStaggered).name()) { tmp2_p = new cudaColorSpinorField(x, csParam); } cudaColorSpinorField &tmp2 = *tmp2_p; cudaColorSpinorField *x_sloppy, *r_sloppy; if (param.precision_sloppy == x.Precision()) { csParam.create = QUDA_REFERENCE_FIELD_CREATE; x_sloppy = &x; r_sloppy = &r; } else { csParam.create = QUDA_COPY_FIELD_CREATE; x_sloppy = new cudaColorSpinorField(x, csParam); r_sloppy = new cudaColorSpinorField(r, csParam); } cudaColorSpinorField &xSloppy = *x_sloppy; cudaColorSpinorField &rSloppy = *r_sloppy; cudaColorSpinorField p(rSloppy); if(&x != &xSloppy){ copyCuda(y,x); zeroCuda(xSloppy); }else{ zeroCuda(y); } const bool use_heavy_quark_res = (param.residual_type & QUDA_HEAVY_QUARK_RESIDUAL) ? true : false; profile.Stop(QUDA_PROFILE_INIT); profile.Start(QUDA_PROFILE_PREAMBLE); double r2_old; double stop = b2*param.tol*param.tol; // stopping condition of solver double heavy_quark_res = 0.0; // heavy quark residual if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(x,r).z); int heavy_quark_check = 10; // how often to check the heavy quark residual double alpha=0.0, beta=0.0; double pAp; int rUpdate = 0; double rNorm = sqrt(r2); double r0Norm = rNorm; double maxrx = rNorm; double maxrr = rNorm; double delta = param.delta; // this parameter determines how many consective reliable update // reisudal increases we tolerate before terminating the solver, // i.e., how long do we want to keep trying to converge int maxResIncrease = 0; // 0 means we have no tolerance profile.Stop(QUDA_PROFILE_PREAMBLE); profile.Start(QUDA_PROFILE_COMPUTE); blas_flops = 0; int k=0; PrintStats("CG", k, r2, b2, heavy_quark_res); int steps_since_reliable = 1; while ( !convergence(r2, heavy_quark_res, stop, param.tol_hq) && k < param.maxiter) { matSloppy(Ap, p, tmp, tmp2); // tmp as tmp double sigma; bool breakdown = false; if (param.pipeline) { double3 triplet = tripleCGReductionCuda(rSloppy, Ap, p); r2 = triplet.x; double Ap2 = triplet.y; pAp = triplet.z; r2_old = r2; alpha = r2 / pAp; sigma = alpha*(alpha * Ap2 - pAp); if (sigma < 0.0 || steps_since_reliable==0) { // sigma condition has broken down r2 = axpyNormCuda(-alpha, Ap, rSloppy); sigma = r2; breakdown = true; } r2 = sigma; } else { r2_old = r2; pAp = reDotProductCuda(p, Ap); alpha = r2 / pAp; // here we are deploying the alternative beta computation Complex cg_norm = axpyCGNormCuda(-alpha, Ap, rSloppy); r2 = real(cg_norm); // (r_new, r_new) sigma = imag(cg_norm) >= 0.0 ? imag(cg_norm) : r2; // use r2 if (r_k+1, r_k+1-r_k) breaks } // reliable update conditions rNorm = sqrt(r2); if (rNorm > maxrx) maxrx = rNorm; if (rNorm > maxrr) maxrr = rNorm; int updateX = (rNorm < delta*r0Norm && r0Norm <= maxrx) ? 1 : 0; int updateR = ((rNorm < delta*maxrr && r0Norm <= maxrr) || updateX) ? 1 : 0; // force a reliable update if we are within target tolerance (only if doing reliable updates) if ( convergence(r2, heavy_quark_res, stop, param.tol_hq) && delta >= param.tol) updateX = 1; if ( !(updateR || updateX)) { //beta = r2 / r2_old; beta = sigma / r2_old; // use the alternative beta computation if (param.pipeline && !breakdown) tripleCGUpdateCuda(alpha, beta, Ap, rSloppy, xSloppy, p); else axpyZpbxCuda(alpha, p, xSloppy, rSloppy, beta); if (use_heavy_quark_res && k%heavy_quark_check==0) { copyCuda(tmp,y); heavy_quark_res = sqrt(xpyHeavyQuarkResidualNormCuda(xSloppy, tmp, rSloppy).z); } steps_since_reliable++; } else { axpyCuda(alpha, p, xSloppy); if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy); xpyCuda(x, y); // swap these around? mat(r, y, x); // here we can use x as tmp r2 = xmyNormCuda(b, r); if (x.Precision() != rSloppy.Precision()) copyCuda(rSloppy, r); zeroCuda(xSloppy); // break-out check if we have reached the limit of the precision static int resIncrease = 0; if (sqrt(r2) > r0Norm && updateX) { // reuse r0Norm for this warningQuda("CG: new reliable residual norm %e is greater than previous reliable residual norm %e", sqrt(r2), r0Norm); k++; rUpdate++; if (++resIncrease > maxResIncrease) break; } else { resIncrease = 0; } rNorm = sqrt(r2); maxrr = rNorm; maxrx = rNorm; r0Norm = rNorm; rUpdate++; // explicitly restore the orthogonality of the gradient vector double rp = reDotProductCuda(rSloppy, p) / (r2); axpyCuda(-rp, rSloppy, p); beta = r2 / r2_old; xpayCuda(rSloppy, beta, p); if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(y,r).z); steps_since_reliable = 0; } breakdown = false; k++; PrintStats("CG", k, r2, b2, heavy_quark_res); } if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy); xpyCuda(y, x); profile.Stop(QUDA_PROFILE_COMPUTE); profile.Start(QUDA_PROFILE_EPILOGUE); param.secs = profile.Last(QUDA_PROFILE_COMPUTE); double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9; reduceDouble(gflops); param.gflops = gflops; param.iter += k; if (k==param.maxiter) warningQuda("Exceeded maximum iterations %d", param.maxiter); if (getVerbosity() >= QUDA_VERBOSE) printfQuda("CG: Reliable updates = %d\n", rUpdate); // compute the true residuals mat(r, x, y); param.true_res = sqrt(xmyNormCuda(b, r) / b2); #if (__COMPUTE_CAPABILITY__ >= 200) param.true_res_hq = sqrt(HeavyQuarkResidualNormCuda(x,r).z); #else param.true_res_hq = 0.0; #endif PrintSummary("CG", k, r2, b2); // reset the flops counters quda::blas_flops = 0; mat.flops(); matSloppy.flops(); profile.Stop(QUDA_PROFILE_EPILOGUE); profile.Start(QUDA_PROFILE_FREE); if (&tmp2 != &tmp) delete tmp2_p; if (param.precision_sloppy != x.Precision()) { delete r_sloppy; delete x_sloppy; } profile.Stop(QUDA_PROFILE_FREE); return; }
void MultiShiftCG::operator()(cudaColorSpinorField **x, cudaColorSpinorField &b) { profile.Start(QUDA_PROFILE_INIT); int num_offset = param.num_offset; double *offset = param.offset; if (num_offset == 0) return; const double b2 = normCuda(b); // Check to see that we're not trying to invert on a zero-field source if(b2 == 0){ profile.Stop(QUDA_PROFILE_INIT); printfQuda("Warning: inverting on zero-field source\n"); for(int i=0; i<num_offset; ++i){ *(x[i]) = b; param.true_res_offset[i] = 0.0; param.true_res_hq_offset[i] = 0.0; } return; } double *zeta = new double[num_offset]; double *zeta_old = new double[num_offset]; double *alpha = new double[num_offset]; double *beta = new double[num_offset]; int j_low = 0; int num_offset_now = num_offset; for (int i=0; i<num_offset; i++) { zeta[i] = zeta_old[i] = 1.0; beta[i] = 0.0; alpha[i] = 1.0; } // flag whether we will be using reliable updates or not bool reliable = false; for (int j=0; j<num_offset; j++) if (param.tol_offset[j] < param.delta) reliable = true; cudaColorSpinorField *r = new cudaColorSpinorField(b); cudaColorSpinorField **y = reliable ? new cudaColorSpinorField*[num_offset] : NULL; ColorSpinorParam csParam(b); csParam.create = QUDA_ZERO_FIELD_CREATE; if (reliable) for (int i=0; i<num_offset; i++) y[i] = new cudaColorSpinorField(*r, csParam); csParam.setPrecision(param.precision_sloppy); cudaColorSpinorField *r_sloppy; if (param.precision_sloppy == x[0]->Precision()) { r_sloppy = r; } else { csParam.create = QUDA_COPY_FIELD_CREATE; r_sloppy = new cudaColorSpinorField(*r, csParam); } cudaColorSpinorField **x_sloppy = new cudaColorSpinorField*[num_offset]; if (param.precision_sloppy == x[0]->Precision() || !param.use_sloppy_partial_accumulator) { for (int i=0; i<num_offset; i++) x_sloppy[i] = x[i]; } else { csParam.create = QUDA_ZERO_FIELD_CREATE; for (int i=0; i<num_offset; i++) x_sloppy[i] = new cudaColorSpinorField(*x[i], csParam); } cudaColorSpinorField **p = new cudaColorSpinorField*[num_offset]; for (int i=0; i<num_offset; i++) p[i]= new cudaColorSpinorField(*r_sloppy); csParam.create = QUDA_ZERO_FIELD_CREATE; cudaColorSpinorField* Ap = new cudaColorSpinorField(*r_sloppy, csParam); cudaColorSpinorField tmp1(*Ap, csParam); // tmp2 only needed for multi-gpu Wilson-like kernels cudaColorSpinorField *tmp2_p = !mat.isStaggered() ? new cudaColorSpinorField(*Ap, csParam) : &tmp1; cudaColorSpinorField &tmp2 = *tmp2_p; // additional high-precision temporary if Wilson and mixed-precision csParam.setPrecision(param.precision); cudaColorSpinorField *tmp3_p = (param.precision != param.precision_sloppy && !mat.isStaggered()) ? new cudaColorSpinorField(*r, csParam) : &tmp1; cudaColorSpinorField &tmp3 = *tmp3_p; profile.Stop(QUDA_PROFILE_INIT); profile.Start(QUDA_PROFILE_PREAMBLE); // stopping condition of each shift double stop[QUDA_MAX_MULTI_SHIFT]; double r2[QUDA_MAX_MULTI_SHIFT]; for (int i=0; i<num_offset; i++) { r2[i] = b2; stop[i] = Solver::stopping(param.tol_offset[i], b2, param.residual_type); } double r2_old; double pAp; double rNorm[QUDA_MAX_MULTI_SHIFT]; double r0Norm[QUDA_MAX_MULTI_SHIFT]; double maxrx[QUDA_MAX_MULTI_SHIFT]; double maxrr[QUDA_MAX_MULTI_SHIFT]; for (int i=0; i<num_offset; i++) { rNorm[i] = sqrt(r2[i]); r0Norm[i] = rNorm[i]; maxrx[i] = rNorm[i]; maxrr[i] = rNorm[i]; } double delta = param.delta; // this parameter determines how many consective reliable update // reisudal increases we tolerate before terminating the solver, // i.e., how long do we want to keep trying to converge const int maxResIncrease = param.max_res_increase; // check if we reached the limit of our tolerance const int maxResIncreaseTotal = param.max_res_increase_total; int resIncrease = 0; int resIncreaseTotal[QUDA_MAX_MULTI_SHIFT]; for (int i=0; i<num_offset; i++) { resIncreaseTotal[i]=0; } int k = 0; int rUpdate = 0; quda::blas_flops = 0; profile.Stop(QUDA_PROFILE_PREAMBLE); profile.Start(QUDA_PROFILE_COMPUTE); if (getVerbosity() >= QUDA_VERBOSE) printfQuda("MultiShift CG: %d iterations, <r,r> = %e, |r|/|b| = %e\n", k, r2[0], sqrt(r2[0]/b2)); while (r2[0] > stop[0] && k < param.maxiter) { matSloppy(*Ap, *p[0], tmp1, tmp2); // FIXME - this should be curried into the Dirac operator if (r->Nspin()==4) axpyCuda(offset[0], *p[0], *Ap); pAp = reDotProductCuda(*p[0], *Ap); // compute zeta and alpha updateAlphaZeta(alpha, zeta, zeta_old, r2, beta, pAp, offset, num_offset_now, j_low); r2_old = r2[0]; Complex cg_norm = axpyCGNormCuda(-alpha[j_low], *Ap, *r_sloppy); r2[0] = real(cg_norm); double zn = imag(cg_norm); // reliable update conditions rNorm[0] = sqrt(r2[0]); for (int j=1; j<num_offset_now; j++) rNorm[j] = rNorm[0] * zeta[j]; int updateX=0, updateR=0; int reliable_shift = -1; // this is the shift that sets the reliable_shift for (int j=num_offset_now-1; j>=0; j--) { if (rNorm[j] > maxrx[j]) maxrx[j] = rNorm[j]; if (rNorm[j] > maxrr[j]) maxrr[j] = rNorm[j]; updateX = (rNorm[j] < delta*r0Norm[j] && r0Norm[j] <= maxrx[j]) ? 1 : updateX; updateR = ((rNorm[j] < delta*maxrr[j] && r0Norm[j] <= maxrr[j]) || updateX) ? 1 : updateR; if ((updateX || updateR) && reliable_shift == -1) reliable_shift = j; } if ( !(updateR || updateX) || !reliable) { //beta[0] = r2[0] / r2_old; beta[0] = zn / r2_old; // update p[0] and x[0] axpyZpbxCuda(alpha[0], *p[0], *x_sloppy[0], *r_sloppy, beta[0]); for (int j=1; j<num_offset_now; j++) { beta[j] = beta[j_low] * zeta[j] * alpha[j] / (zeta_old[j] * alpha[j_low]); // update p[i] and x[i] axpyBzpcxCuda(alpha[j], *p[j], *x_sloppy[j], zeta[j], *r_sloppy, beta[j]); } } else { for (int j=0; j<num_offset_now; j++) { axpyCuda(alpha[j], *p[j], *x_sloppy[j]); copyCuda(*x[j], *x_sloppy[j]); xpyCuda(*x[j], *y[j]); } mat(*r, *y[0], *x[0], tmp3); // here we can use x as tmp if (r->Nspin()==4) axpyCuda(offset[0], *y[0], *r); r2[0] = xmyNormCuda(b, *r); for (int j=1; j<num_offset_now; j++) r2[j] = zeta[j] * zeta[j] * r2[0]; for (int j=0; j<num_offset_now; j++) zeroCuda(*x_sloppy[j]); copyCuda(*r_sloppy, *r); // break-out check if we have reached the limit of the precision if (sqrt(r2[reliable_shift]) > r0Norm[reliable_shift]) { // reuse r0Norm for this resIncrease++; resIncreaseTotal[reliable_shift]++; warningQuda("MultiShiftCG: Shift %d, updated residual %e is greater than previous residual %e (total #inc %i)", reliable_shift, sqrt(r2[reliable_shift]), r0Norm[reliable_shift], resIncreaseTotal[reliable_shift]); if (resIncrease > maxResIncrease or resIncreaseTotal[reliable_shift] > maxResIncreaseTotal) break; // check if we reached the limit of our tolerancebreak; } else { resIncrease = 0; } // explicitly restore the orthogonality of the gradient vector for (int j=0; j<num_offset_now; j++) { double rp = reDotProductCuda(*r_sloppy, *p[j]) / (r2[0]); axpyCuda(-rp, *r_sloppy, *p[j]); } // update beta and p beta[0] = r2[0] / r2_old; xpayCuda(*r_sloppy, beta[0], *p[0]); for (int j=1; j<num_offset_now; j++) { beta[j] = beta[j_low] * zeta[j] * alpha[j] / (zeta_old[j] * alpha[j_low]); axpbyCuda(zeta[j], *r_sloppy, beta[j], *p[j]); } // update reliable update parameters for the system that triggered the update int m = reliable_shift; rNorm[m] = sqrt(r2[0]) * zeta[m]; maxrr[m] = rNorm[m]; maxrx[m] = rNorm[m]; r0Norm[m] = rNorm[m]; rUpdate++; } // now we can check if any of the shifts have converged and remove them for (int j=1; j<num_offset_now; j++) { if (zeta[j] == 0.0) { num_offset_now--; if (getVerbosity() >= QUDA_VERBOSE) printfQuda("MultiShift CG: Shift %d converged after %d iterations\n", j, k + 1); } else { r2[j] = zeta[j] * zeta[j] * r2[0]; if (r2[j] < stop[j]) { num_offset_now--; if (getVerbosity() >= QUDA_VERBOSE) printfQuda("MultiShift CG: Shift %d converged after %d iterations\n", j, k+1); } } } k++; if (getVerbosity() >= QUDA_VERBOSE) printfQuda("MultiShift CG: %d iterations, <r,r> = %e, |r|/|b| = %e\n", k, r2[0], sqrt(r2[0]/b2)); } for (int i=0; i<num_offset; i++) { copyCuda(*x[i], *x_sloppy[i]); if (reliable) xpyCuda(*y[i], *x[i]); } profile.Stop(QUDA_PROFILE_COMPUTE); profile.Start(QUDA_PROFILE_EPILOGUE); if (getVerbosity() >= QUDA_VERBOSE) printfQuda("MultiShift CG: Reliable updates = %d\n", rUpdate); if (k==param.maxiter) warningQuda("Exceeded maximum iterations %d\n", param.maxiter); param.secs = profile.Last(QUDA_PROFILE_COMPUTE); double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9; reduceDouble(gflops); param.gflops = gflops; param.iter += k; for(int i=0; i < num_offset; i++) { mat(*r, *x[i]); if (r->Nspin()==4) { axpyCuda(offset[i], *x[i], *r); // Offset it. } else if (i!=0) { axpyCuda(offset[i]-offset[0], *x[i], *r); // Offset it. } double true_res = xmyNormCuda(b, *r); param.true_res_offset[i] = sqrt(true_res/b2); #if (__COMPUTE_CAPABILITY__ >= 200) param.true_res_hq_offset[i] = sqrt(HeavyQuarkResidualNormCuda(*x[i], *r).z); #else param.true_res_hq_offset[i] = 0.0; #endif } if (getVerbosity() >= QUDA_SUMMARIZE){ printfQuda("MultiShift CG: Converged after %d iterations\n", k); for(int i=0; i < num_offset; i++) { printfQuda(" shift=%d, relative residual: iterated = %e, true = %e\n", i, sqrt(r2[i]/b2), param.true_res_offset[i]); } } // reset the flops counters quda::blas_flops = 0; mat.flops(); matSloppy.flops(); profile.Stop(QUDA_PROFILE_EPILOGUE); profile.Start(QUDA_PROFILE_FREE); if (&tmp3 != &tmp1) delete tmp3_p; if (&tmp2 != &tmp1) delete tmp2_p; if (r_sloppy->Precision() != r->Precision()) delete r_sloppy; for (int i=0; i<num_offset; i++) if (x_sloppy[i]->Precision() != x[i]->Precision()) delete x_sloppy[i]; delete []x_sloppy; delete r; for (int i=0; i<num_offset; i++) delete p[i]; delete []p; if (reliable) { for (int i=0; i<num_offset; i++) delete y[i]; delete []y; } delete Ap; delete []zeta_old; delete []zeta; delete []alpha; delete []beta; profile.Stop(QUDA_PROFILE_FREE); return; }
void PreconCG::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b) { profile.Start(QUDA_PROFILE_INIT); // Check to see that we're not trying to invert on a zero-field source const double b2 = norm2(b); if(b2 == 0){ profile.Stop(QUDA_PROFILE_INIT); printfQuda("Warning: inverting on zero-field source\n"); x=b; param.true_res = 0.0; param.true_res_hq = 0.0; } int k=0; int rUpdate=0; cudaColorSpinorField* minvrPre; cudaColorSpinorField* rPre; cudaColorSpinorField* minvr; cudaColorSpinorField* minvrSloppy; cudaColorSpinorField* p; ColorSpinorParam csParam(b); cudaColorSpinorField r(b); if(K) minvr = new cudaColorSpinorField(b); csParam.create = QUDA_ZERO_FIELD_CREATE; cudaColorSpinorField y(b,csParam); mat(r, x, y); // => r = A*x; double r2 = xmyNormCuda(b,r); csParam.setPrecision(param.precision_sloppy); cudaColorSpinorField tmpSloppy(x,csParam); cudaColorSpinorField Ap(x,csParam); cudaColorSpinorField *r_sloppy; if(param.precision_sloppy == x.Precision()) { r_sloppy = &r; minvrSloppy = minvr; }else{ csParam.create = QUDA_COPY_FIELD_CREATE; r_sloppy = new cudaColorSpinorField(r,csParam); if(K) minvrSloppy = new cudaColorSpinorField(*minvr,csParam); } cudaColorSpinorField *x_sloppy; if(param.precision_sloppy == x.Precision() || !param.use_sloppy_partial_accumulator) { csParam.create = QUDA_REFERENCE_FIELD_CREATE; x_sloppy = &x; }else{ csParam.create = QUDA_COPY_FIELD_CREATE; x_sloppy = new cudaColorSpinorField(x,csParam); } cudaColorSpinorField &xSloppy = *x_sloppy; cudaColorSpinorField &rSloppy = *r_sloppy; if(&x != &xSloppy){ copyCuda(y, x); // copy x to y zeroCuda(xSloppy); }else{ zeroCuda(y); // no reliable updates // NB: check this } const bool use_heavy_quark_res = (param.residual_type & QUDA_HEAVY_QUARK_RESIDUAL) ? true : false; if(K){ csParam.create = QUDA_COPY_FIELD_CREATE; csParam.setPrecision(param.precision_precondition); rPre = new cudaColorSpinorField(rSloppy,csParam); // Create minvrPre minvrPre = new cudaColorSpinorField(*rPre); globalReduce = false; (*K)(*minvrPre, *rPre); globalReduce = true; *minvrSloppy = *minvrPre; p = new cudaColorSpinorField(*minvrSloppy); }else{ p = new cudaColorSpinorField(rSloppy); } profile.Stop(QUDA_PROFILE_INIT); profile.Start(QUDA_PROFILE_PREAMBLE); double stop = stopping(param.tol, b2, param.residual_type); // stopping condition of solver double heavy_quark_res = 0.0; // heavy quark residual if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(x,r).z); int heavy_quark_check = 10; // how often to check the heavy quark residual double alpha = 0.0, beta=0.0; double pAp; double rMinvr = 0; double rMinvr_old = 0.0; double r_new_Minvr_old = 0.0; double r2_old = 0; r2 = norm2(r); double rNorm = sqrt(r2); double r0Norm = rNorm; double maxrx = rNorm; double maxrr = rNorm; double delta = param.delta; if(K) rMinvr = reDotProductCuda(rSloppy,*minvrSloppy); profile.Stop(QUDA_PROFILE_PREAMBLE); profile.Start(QUDA_PROFILE_COMPUTE); quda::blas_flops = 0; int steps_since_reliable = 1; const int maxResIncrease = 0; while(!convergence(r2, heavy_quark_res, stop, param.tol_hq) && k < param.maxiter){ matSloppy(Ap, *p, tmpSloppy); double sigma; bool breakdown = false; pAp = reDotProductCuda(*p,Ap); alpha = (K) ? rMinvr/pAp : r2/pAp; Complex cg_norm = axpyCGNormCuda(-alpha, Ap, rSloppy); // r --> r - alpha*A*p r2_old = r2; r2 = real(cg_norm); sigma = imag(cg_norm) >= 0.0 ? imag(cg_norm) : r2; // use r2 if (r_k+1, r_k-1 - r_k) breaks if(K) rMinvr_old = rMinvr; rNorm = sqrt(r2); if(rNorm > maxrx) maxrx = rNorm; if(rNorm > maxrr) maxrr = rNorm; int updateX = (rNorm < delta*r0Norm && r0Norm <= maxrx) ? 1 : 0; int updateR = ((rNorm < delta*maxrr && r0Norm <= maxrr) || updateX) ? 1 : 0; // force a reliable update if we are within target tolerance (only if doing reliable updates) if( convergence(r2, heavy_quark_res, stop, param.tol_hq) && delta >= param.tol) updateX = 1; if( !(updateR || updateX) ){ if(K){ r_new_Minvr_old = reDotProductCuda(rSloppy,*minvrSloppy); *rPre = rSloppy; globalReduce = false; (*K)(*minvrPre, *rPre); globalReduce = true; *minvrSloppy = *minvrPre; rMinvr = reDotProductCuda(rSloppy,*minvrSloppy); beta = (rMinvr - r_new_Minvr_old)/rMinvr_old; axpyZpbxCuda(alpha, *p, xSloppy, *minvrSloppy, beta); }else{ beta = sigma/r2_old; // use the alternative beta computation axpyZpbxCuda(alpha, *p, xSloppy, rSloppy, beta); } } else { // reliable update axpyCuda(alpha, *p, xSloppy); // xSloppy += alpha*p copyCuda(x, xSloppy); xpyCuda(x, y); // y += x // Now compute r mat(r, y, x); // x is just a temporary here r2 = xmyNormCuda(b, r); copyCuda(rSloppy, r); // copy r to rSloppy zeroCuda(xSloppy); // break-out check if we have reached the limit of the precision static int resIncrease = 0; if(sqrt(r2) > r0Norm && updateX) { // reuse r0Norm for this warningQuda("PCG: new reliable residual norm %e is greater than previous reliable residual norm %e", sqrt(r2), r0Norm); k++; rUpdate++; if(++resIncrease > maxResIncrease) break; }else{ resIncrease = 0; } rNorm = sqrt(r2); maxrr = rNorm; maxrx = rNorm; r0Norm = rNorm; ++rUpdate; if(K){ *rPre = rSloppy; globalReduce = false; (*K)(*minvrPre, *rPre); globalReduce = true; *minvrSloppy = *minvrPre; rMinvr = reDotProductCuda(rSloppy,*minvrSloppy); beta = rMinvr/rMinvr_old; xpayCuda(*minvrSloppy, beta, *p); // p = minvrSloppy + beta*p }else{ // standard CG - no preconditioning // explicitly restore the orthogonality of the gradient vector double rp = reDotProductCuda(rSloppy, *p)/(r2); axpyCuda(-rp, rSloppy, *p); beta = r2/r2_old; xpayCuda(rSloppy, beta, *p); steps_since_reliable = 0; } } breakdown = false; ++k; PrintStats("PCG", k, r2, b2, heavy_quark_res); } profile.Stop(QUDA_PROFILE_COMPUTE); profile.Start(QUDA_PROFILE_EPILOGUE); if(x.Precision() != param.precision_sloppy) copyCuda(x, xSloppy); xpyCuda(y, x); // x += y param.secs = profile.Last(QUDA_PROFILE_COMPUTE); double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops() + matPrecon.flops())*1e-9; reduceDouble(gflops); param.gflops = gflops; param.iter += k; if (k==param.maxiter) warningQuda("Exceeded maximum iterations %d", param.maxiter); if (getVerbosity() >= QUDA_VERBOSE) printfQuda("CG: Reliable updates = %d\n", rUpdate); // compute the true residual mat(r, x, y); double true_res = xmyNormCuda(b, r); param.true_res = sqrt(true_res / b2); // reset the flops counters quda::blas_flops = 0; mat.flops(); matSloppy.flops(); matPrecon.flops(); profile.Stop(QUDA_PROFILE_EPILOGUE); profile.Start(QUDA_PROFILE_FREE); if(K){ // These are only needed if preconditioning is used delete minvrPre; delete rPre; delete minvr; if(x.Precision() != param.precision_sloppy) delete minvrSloppy; } delete p; if(x.Precision() != param.precision_sloppy){ delete x_sloppy; delete r_sloppy; } profile.Stop(QUDA_PROFILE_FREE); return; }