void CG::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b)
{
int k=0;
int rUpdate = 0;
cudaColorSpinorField r(b);
ColorSpinorParam param(x);
param.create = QUDA_ZERO_FIELD_CREATE;
cudaColorSpinorField y(b, param);
mat(r, x, y);
zeroCuda(y);
double r2 = xmyNormCuda(b, r);
rUpdate ++;
param.precision = invParam.cuda_prec_sloppy;
cudaColorSpinorField Ap(x, param);
cudaColorSpinorField tmp(x, param);
cudaColorSpinorField tmp2(x, param); // only needed for clover and twisted mass
cudaColorSpinorField *x_sloppy, *r_sloppy;
if (invParam.cuda_prec_sloppy == x.Precision()) {
param.create = QUDA_REFERENCE_FIELD_CREATE;
x_sloppy = &x;
r_sloppy = &r;
} else {
param.create = QUDA_COPY_FIELD_CREATE;
x_sloppy = new cudaColorSpinorField(x, param);
r_sloppy = new cudaColorSpinorField(r, param);
}
cudaColorSpinorField &xSloppy = *x_sloppy;
cudaColorSpinorField &rSloppy = *r_sloppy;
cudaColorSpinorField p(rSloppy);
double r2_old;
double src_norm = norm2(b);
double stop = src_norm*invParam.tol*invParam.tol; // stopping condition of solver
double alpha, beta;
double pAp;
double rNorm = sqrt(r2);
double r0Norm = rNorm;
double maxrx = rNorm;
double maxrr = rNorm;
double delta = invParam.reliable_delta;
if (invParam.verbosity >= QUDA_VERBOSE) printfQuda("CG: %d iterations, r2 = %e\n", k, r2);
quda::blas_flops = 0;
stopwatchStart();
while (r2 > stop && k<invParam.maxiter) {
matSloppy(Ap, p, tmp, tmp2); // tmp as tmp
pAp = reDotProductCuda(p, Ap);
alpha = r2 / pAp;
r2_old = r2;
r2 = axpyNormCuda(-alpha, Ap, rSloppy);
// reliable update conditions
rNorm = sqrt(r2);
if (rNorm > maxrx) maxrx = rNorm;
if (rNorm > maxrr) maxrr = rNorm;
int updateX = (rNorm < delta*r0Norm && r0Norm <= maxrx) ? 1 : 0;
int updateR = ((rNorm < delta*maxrr && r0Norm <= maxrr) || updateX) ? 1 : 0;
if (!(updateR || updateX)) {
beta = r2 / r2_old;
axpyZpbxCuda(alpha, p, xSloppy, rSloppy, beta);
} else {
axpyCuda(alpha, p, xSloppy);
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(x, y); // swap these around?
mat(r, y, x); // here we can use x as tmp
r2 = xmyNormCuda(b, r);
if (x.Precision() != rSloppy.Precision()) copyCuda(rSloppy, r);
zeroCuda(xSloppy);
rNorm = sqrt(r2);
maxrr = rNorm;
maxrx = rNorm;
r0Norm = rNorm;
rUpdate++;
beta = r2 / r2_old;
xpayCuda(rSloppy, beta, p);
}
k++;
if (invParam.verbosity >= QUDA_VERBOSE)
printfQuda("CG: %d iterations, r2 = %e\n", k, r2);
}
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(y, x);
invParam.secs = stopwatchReadSeconds();
if (k==invParam.maxiter)
warningQuda("Exceeded maximum iterations %d", invParam.maxiter);
if (invParam.verbosity >= QUDA_SUMMARIZE)
printfQuda("CG: Reliable updates = %d\n", rUpdate);
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9;
reduceDouble(gflops);
// printfQuda("%f gflops\n", gflops / stopwatchReadSeconds());
invParam.gflops = gflops;
invParam.iter = k;
quda::blas_flops = 0;
if (invParam.verbosity >= QUDA_SUMMARIZE){
mat(r, x, y);
double true_res = xmyNormCuda(b, r);
printfQuda("CG: Converged after %d iterations, relative residua: iterated = %e, true = %e\n",
k, sqrt(r2/src_norm), sqrt(true_res / src_norm));
}
if (invParam.cuda_prec_sloppy != x.Precision()) {
delete r_sloppy;
delete x_sloppy;
}
return;
}
void MultiShiftCG::operator()(cudaColorSpinorField **x, cudaColorSpinorField &b)
{
int num_offset = invParam.num_offset;
double *offset = invParam.offset;
double *residue_sq = invParam.tol_offset;
if (num_offset == 0) return;
int *finished = new int [num_offset];
double *zeta_i = new double[num_offset];
double *zeta_im1 = new double[num_offset];
double *zeta_ip1 = new double[num_offset];
double *beta_i = new double[num_offset];
double *beta_im1 = new double[num_offset];
double *alpha = new double[num_offset];
int i, j;
int j_low = 0;
int num_offset_now = num_offset;
for (i=0; i<num_offset; i++) {
finished[i] = 0;
zeta_im1[i] = zeta_i[i] = 1.0;
beta_im1[i] = -1.0;
alpha[i] = 0.0;
}
//double msq_x4 = offset[0];
cudaColorSpinorField *r = new cudaColorSpinorField(b);
cudaColorSpinorField **x_sloppy = new cudaColorSpinorField*[num_offset], *r_sloppy;
ColorSpinorParam param;
param.create = QUDA_ZERO_FIELD_CREATE;
param.precision = invParam.cuda_prec_sloppy;
if (invParam.cuda_prec_sloppy == x[0]->Precision()) {
for (i=0; i<num_offset; i++){
x_sloppy[i] = x[i];
zeroCuda(*x_sloppy[i]);
}
r_sloppy = r;
} else {
for (i=0; i<num_offset; i++) {
x_sloppy[i] = new cudaColorSpinorField(*x[i], param);
}
param.create = QUDA_COPY_FIELD_CREATE;
r_sloppy = new cudaColorSpinorField(*r, param);
}
cudaColorSpinorField **p = new cudaColorSpinorField*[num_offset];
for(i=0;i < num_offset;i++){
p[i]= new cudaColorSpinorField(*r_sloppy);
}
param.create = QUDA_ZERO_FIELD_CREATE;
param.precision = invParam.cuda_prec_sloppy;
cudaColorSpinorField* Ap = new cudaColorSpinorField(*r_sloppy, param);
double b2 = 0.0;
b2 = normCuda(b);
double r2 = b2;
double r2_old;
double stop = r2*invParam.tol*invParam.tol; // stopping condition of solver
double pAp;
int k = 0;
stopwatchStart();
while (r2 > stop && k < invParam.maxiter) {
//dslashCuda_st(tmp_sloppy, fatlinkSloppy, longlinkSloppy, p[0], 1 - oddBit, 0);
//dslashAxpyCuda(Ap, fatlinkSloppy, longlinkSloppy, tmp_sloppy, oddBit, 0, p[0], msq_x4);
matSloppy(*Ap, *p[0]);
if (invParam.dslash_type != QUDA_ASQTAD_DSLASH){
axpyCuda(offset[0], *p[0], *Ap);
}
pAp = reDotProductCuda(*p[0], *Ap);
beta_i[0] = r2 / pAp;
zeta_ip1[0] = 1.0;
for (j=1; j<num_offset_now; j++) {
zeta_ip1[j] = zeta_i[j] * zeta_im1[j] * beta_im1[j_low];
double c1 = beta_i[j_low] * alpha[j_low] * (zeta_im1[j]-zeta_i[j]);
double c2 = zeta_im1[j] * beta_im1[j_low] * (1.0+(offset[j]-offset[0])*beta_i[j_low]);
/*THISBLOWSUP
zeta_ip1[j] /= c1 + c2;
beta_i[j] = beta_i[j_low] * zeta_ip1[j] / zeta_i[j];
*/
/*TRYTHIS*/
if( (c1+c2) != 0.0 )
zeta_ip1[j] /= (c1 + c2);
else {
zeta_ip1[j] = 0.0;
finished[j] = 1;
}
if( zeta_i[j] != 0.0) {
beta_i[j] = beta_i[j_low] * zeta_ip1[j] / zeta_i[j];
} else {
zeta_ip1[j] = 0.0;
beta_i[j] = 0.0;
finished[j] = 1;
if (invParam.verbosity >= QUDA_VERBOSE)
printfQuda("SETTING A ZERO, j=%d, num_offset_now=%d\n",j,num_offset_now);
//if(j==num_offset_now-1)node0_PRINTF("REDUCING OFFSET\n");
if(j==num_offset_now-1) num_offset_now--;
// don't work any more on finished solutions
// this only works if largest offsets are last, otherwise
// just wastes time multiplying by zero
}
}
r2_old = r2;
r2 = axpyNormCuda(-beta_i[j_low], *Ap, *r_sloppy);
alpha[0] = r2 / r2_old;
for (j=1; j<num_offset_now; j++) {
/*THISBLOWSUP
alpha[j] = alpha[j_low] * zeta_ip1[j] * beta_i[j] /
(zeta_i[j] * beta_i[j_low]);
*/
/*TRYTHIS*/
if( zeta_i[j] * beta_i[j_low] != 0.0)
alpha[j] = alpha[j_low] * zeta_ip1[j] * beta_i[j] /
(zeta_i[j] * beta_i[j_low]);
else {
alpha[j] = 0.0;
finished[j] = 1;
}
}
axpyZpbxCuda(beta_i[0], *p[0], *x_sloppy[0], *r_sloppy, alpha[0]);
for (j=1; j<num_offset_now; j++) {
axpyBzpcxCuda(beta_i[j], *p[j], *x_sloppy[j], zeta_ip1[j], *r_sloppy, alpha[j]);
}
for (j=0; j<num_offset_now; j++) {
beta_im1[j] = beta_i[j];
zeta_im1[j] = zeta_i[j];
zeta_i[j] = zeta_ip1[j];
}
k++;
if (invParam.verbosity >= QUDA_VERBOSE){
printfQuda("Multimass CG: %d iterations, r2 = %e\n", k, r2);
}
}
if (x[0]->Precision() != x_sloppy[0]->Precision()) {
for(i=0;i < num_offset; i++){
copyCuda(*x[i], *x_sloppy[i]);
}
}
*residue_sq = r2;
invParam.secs = stopwatchReadSeconds();
if (k==invParam.maxiter) {
warningQuda("Exceeded maximum iterations %d\n", invParam.maxiter);
}
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9;
reduceDouble(gflops);
invParam.gflops = gflops;
invParam.iter = k;
// Calculate the true residual of the system with the smallest shift
mat(*r, *x[0]);
axpyCuda(offset[0],*x[0], *r); // Offset it.
double true_res = xmyNormCuda(b, *r);
if (invParam.verbosity >= QUDA_SUMMARIZE){
printfQuda("MultiShift CG: Converged after %d iterations, r2 = %e, relative true_r2 = %e\n",
k,r2, (true_res / b2));
}
if (invParam.verbosity >= QUDA_VERBOSE){
printfQuda("MultiShift CG: Converged after %d iterations\n", k);
printfQuda(" shift=0 resid_rel=%e\n", sqrt(true_res/b2));
for(int i=1; i < num_offset; i++) {
mat(*r, *x[i]);
axpyCuda(offset[i],*x[i], *r); // Offset it.
true_res = xmyNormCuda(b, *r);
printfQuda(" shift=%d resid_rel=%e\n",i, sqrt(true_res/b2));
}
}
delete r;
for(i=0;i < num_offset; i++){
delete p[i];
}
delete p;
delete Ap;
if (invParam.cuda_prec_sloppy != x[0]->Precision()) {
for(i=0;i < num_offset;i++){
delete x_sloppy[i];
}
delete r_sloppy;
}
delete x_sloppy;
delete []finished;
delete []zeta_i;
delete []zeta_im1;
delete []zeta_ip1;
delete []beta_i;
delete []beta_im1;
delete []alpha;
}
void MultiShiftCG::operator()(cudaColorSpinorField **x, cudaColorSpinorField &b)
{
profile.Start(QUDA_PROFILE_INIT);
int num_offset = param.num_offset;
double *offset = param.offset;
if (num_offset == 0) return;
const double b2 = normCuda(b);
// Check to see that we're not trying to invert on a zero-field source
if(b2 == 0){
profile.Stop(QUDA_PROFILE_INIT);
printfQuda("Warning: inverting on zero-field source\n");
for(int i=0; i<num_offset; ++i){
*(x[i]) = b;
param.true_res_offset[i] = 0.0;
param.true_res_hq_offset[i] = 0.0;
}
return;
}
double *zeta = new double[num_offset];
double *zeta_old = new double[num_offset];
double *alpha = new double[num_offset];
double *beta = new double[num_offset];
int j_low = 0;
int num_offset_now = num_offset;
for (int i=0; i<num_offset; i++) {
zeta[i] = zeta_old[i] = 1.0;
beta[i] = 0.0;
alpha[i] = 1.0;
}
// flag whether we will be using reliable updates or not
bool reliable = false;
for (int j=0; j<num_offset; j++)
if (param.tol_offset[j] < param.delta) reliable = true;
cudaColorSpinorField *r = new cudaColorSpinorField(b);
cudaColorSpinorField **y = reliable ? new cudaColorSpinorField*[num_offset] : NULL;
ColorSpinorParam csParam(b);
csParam.create = QUDA_ZERO_FIELD_CREATE;
if (reliable)
for (int i=0; i<num_offset; i++) y[i] = new cudaColorSpinorField(*r, csParam);
csParam.setPrecision(param.precision_sloppy);
cudaColorSpinorField *r_sloppy;
if (param.precision_sloppy == x[0]->Precision()) {
r_sloppy = r;
} else {
csParam.create = QUDA_COPY_FIELD_CREATE;
r_sloppy = new cudaColorSpinorField(*r, csParam);
}
cudaColorSpinorField **x_sloppy = new cudaColorSpinorField*[num_offset];
if (param.precision_sloppy == x[0]->Precision() ||
!param.use_sloppy_partial_accumulator) {
for (int i=0; i<num_offset; i++) x_sloppy[i] = x[i];
} else {
csParam.create = QUDA_ZERO_FIELD_CREATE;
for (int i=0; i<num_offset; i++)
x_sloppy[i] = new cudaColorSpinorField(*x[i], csParam);
}
cudaColorSpinorField **p = new cudaColorSpinorField*[num_offset];
for (int i=0; i<num_offset; i++) p[i]= new cudaColorSpinorField(*r_sloppy);
csParam.create = QUDA_ZERO_FIELD_CREATE;
cudaColorSpinorField* Ap = new cudaColorSpinorField(*r_sloppy, csParam);
cudaColorSpinorField tmp1(*Ap, csParam);
// tmp2 only needed for multi-gpu Wilson-like kernels
cudaColorSpinorField *tmp2_p = !mat.isStaggered() ?
new cudaColorSpinorField(*Ap, csParam) : &tmp1;
cudaColorSpinorField &tmp2 = *tmp2_p;
// additional high-precision temporary if Wilson and mixed-precision
csParam.setPrecision(param.precision);
cudaColorSpinorField *tmp3_p =
(param.precision != param.precision_sloppy && !mat.isStaggered()) ?
new cudaColorSpinorField(*r, csParam) : &tmp1;
cudaColorSpinorField &tmp3 = *tmp3_p;
profile.Stop(QUDA_PROFILE_INIT);
profile.Start(QUDA_PROFILE_PREAMBLE);
// stopping condition of each shift
double stop[QUDA_MAX_MULTI_SHIFT];
double r2[QUDA_MAX_MULTI_SHIFT];
for (int i=0; i<num_offset; i++) {
r2[i] = b2;
stop[i] = Solver::stopping(param.tol_offset[i], b2, param.residual_type);
}
double r2_old;
double pAp;
double rNorm[QUDA_MAX_MULTI_SHIFT];
double r0Norm[QUDA_MAX_MULTI_SHIFT];
double maxrx[QUDA_MAX_MULTI_SHIFT];
double maxrr[QUDA_MAX_MULTI_SHIFT];
for (int i=0; i<num_offset; i++) {
rNorm[i] = sqrt(r2[i]);
r0Norm[i] = rNorm[i];
maxrx[i] = rNorm[i];
maxrr[i] = rNorm[i];
}
double delta = param.delta;
// this parameter determines how many consective reliable update
// reisudal increases we tolerate before terminating the solver,
// i.e., how long do we want to keep trying to converge
const int maxResIncrease = param.max_res_increase; // check if we reached the limit of our tolerance
const int maxResIncreaseTotal = param.max_res_increase_total;
int resIncrease = 0;
int resIncreaseTotal[QUDA_MAX_MULTI_SHIFT];
for (int i=0; i<num_offset; i++) {
resIncreaseTotal[i]=0;
}
int k = 0;
int rUpdate = 0;
quda::blas_flops = 0;
profile.Stop(QUDA_PROFILE_PREAMBLE);
profile.Start(QUDA_PROFILE_COMPUTE);
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("MultiShift CG: %d iterations, <r,r> = %e, |r|/|b| = %e\n", k, r2[0], sqrt(r2[0]/b2));
while (r2[0] > stop[0] && k < param.maxiter) {
matSloppy(*Ap, *p[0], tmp1, tmp2);
// FIXME - this should be curried into the Dirac operator
if (r->Nspin()==4) axpyCuda(offset[0], *p[0], *Ap);
pAp = reDotProductCuda(*p[0], *Ap);
// compute zeta and alpha
updateAlphaZeta(alpha, zeta, zeta_old, r2, beta, pAp, offset, num_offset_now, j_low);
r2_old = r2[0];
Complex cg_norm = axpyCGNormCuda(-alpha[j_low], *Ap, *r_sloppy);
r2[0] = real(cg_norm);
double zn = imag(cg_norm);
// reliable update conditions
rNorm[0] = sqrt(r2[0]);
for (int j=1; j<num_offset_now; j++) rNorm[j] = rNorm[0] * zeta[j];
int updateX=0, updateR=0;
int reliable_shift = -1; // this is the shift that sets the reliable_shift
for (int j=num_offset_now-1; j>=0; j--) {
if (rNorm[j] > maxrx[j]) maxrx[j] = rNorm[j];
if (rNorm[j] > maxrr[j]) maxrr[j] = rNorm[j];
updateX = (rNorm[j] < delta*r0Norm[j] && r0Norm[j] <= maxrx[j]) ? 1 : updateX;
updateR = ((rNorm[j] < delta*maxrr[j] && r0Norm[j] <= maxrr[j]) || updateX) ? 1 : updateR;
if ((updateX || updateR) && reliable_shift == -1) reliable_shift = j;
}
if ( !(updateR || updateX) || !reliable) {
//beta[0] = r2[0] / r2_old;
beta[0] = zn / r2_old;
// update p[0] and x[0]
axpyZpbxCuda(alpha[0], *p[0], *x_sloppy[0], *r_sloppy, beta[0]);
for (int j=1; j<num_offset_now; j++) {
beta[j] = beta[j_low] * zeta[j] * alpha[j] / (zeta_old[j] * alpha[j_low]);
// update p[i] and x[i]
axpyBzpcxCuda(alpha[j], *p[j], *x_sloppy[j], zeta[j], *r_sloppy, beta[j]);
}
} else {
for (int j=0; j<num_offset_now; j++) {
axpyCuda(alpha[j], *p[j], *x_sloppy[j]);
copyCuda(*x[j], *x_sloppy[j]);
xpyCuda(*x[j], *y[j]);
}
mat(*r, *y[0], *x[0], tmp3); // here we can use x as tmp
if (r->Nspin()==4) axpyCuda(offset[0], *y[0], *r);
r2[0] = xmyNormCuda(b, *r);
for (int j=1; j<num_offset_now; j++) r2[j] = zeta[j] * zeta[j] * r2[0];
for (int j=0; j<num_offset_now; j++) zeroCuda(*x_sloppy[j]);
copyCuda(*r_sloppy, *r);
// break-out check if we have reached the limit of the precision
if (sqrt(r2[reliable_shift]) > r0Norm[reliable_shift]) { // reuse r0Norm for this
resIncrease++;
resIncreaseTotal[reliable_shift]++;
warningQuda("MultiShiftCG: Shift %d, updated residual %e is greater than previous residual %e (total #inc %i)",
reliable_shift, sqrt(r2[reliable_shift]), r0Norm[reliable_shift], resIncreaseTotal[reliable_shift]);
if (resIncrease > maxResIncrease or resIncreaseTotal[reliable_shift] > maxResIncreaseTotal) break; // check if we reached the limit of our tolerancebreak;
} else {
resIncrease = 0;
}
// explicitly restore the orthogonality of the gradient vector
for (int j=0; j<num_offset_now; j++) {
double rp = reDotProductCuda(*r_sloppy, *p[j]) / (r2[0]);
axpyCuda(-rp, *r_sloppy, *p[j]);
}
// update beta and p
beta[0] = r2[0] / r2_old;
xpayCuda(*r_sloppy, beta[0], *p[0]);
for (int j=1; j<num_offset_now; j++) {
beta[j] = beta[j_low] * zeta[j] * alpha[j] / (zeta_old[j] * alpha[j_low]);
axpbyCuda(zeta[j], *r_sloppy, beta[j], *p[j]);
}
// update reliable update parameters for the system that triggered the update
int m = reliable_shift;
rNorm[m] = sqrt(r2[0]) * zeta[m];
maxrr[m] = rNorm[m];
maxrx[m] = rNorm[m];
r0Norm[m] = rNorm[m];
rUpdate++;
}
// now we can check if any of the shifts have converged and remove them
for (int j=1; j<num_offset_now; j++) {
if (zeta[j] == 0.0) {
num_offset_now--;
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("MultiShift CG: Shift %d converged after %d iterations\n", j, k + 1);
}
else {
r2[j] = zeta[j] * zeta[j] * r2[0];
if (r2[j] < stop[j]) {
num_offset_now--;
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("MultiShift CG: Shift %d converged after %d iterations\n", j, k+1);
}
}
}
k++;
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("MultiShift CG: %d iterations, <r,r> = %e, |r|/|b| = %e\n", k, r2[0], sqrt(r2[0]/b2));
}
for (int i=0; i<num_offset; i++) {
copyCuda(*x[i], *x_sloppy[i]);
if (reliable) xpyCuda(*y[i], *x[i]);
}
profile.Stop(QUDA_PROFILE_COMPUTE);
profile.Start(QUDA_PROFILE_EPILOGUE);
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("MultiShift CG: Reliable updates = %d\n", rUpdate);
if (k==param.maxiter) warningQuda("Exceeded maximum iterations %d\n", param.maxiter);
param.secs = profile.Last(QUDA_PROFILE_COMPUTE);
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9;
reduceDouble(gflops);
param.gflops = gflops;
param.iter += k;
for(int i=0; i < num_offset; i++) {
mat(*r, *x[i]);
if (r->Nspin()==4) {
axpyCuda(offset[i], *x[i], *r); // Offset it.
} else if (i!=0) {
axpyCuda(offset[i]-offset[0], *x[i], *r); // Offset it.
}
double true_res = xmyNormCuda(b, *r);
param.true_res_offset[i] = sqrt(true_res/b2);
#if (__COMPUTE_CAPABILITY__ >= 200)
param.true_res_hq_offset[i] = sqrt(HeavyQuarkResidualNormCuda(*x[i], *r).z);
#else
param.true_res_hq_offset[i] = 0.0;
#endif
}
if (getVerbosity() >= QUDA_SUMMARIZE){
printfQuda("MultiShift CG: Converged after %d iterations\n", k);
for(int i=0; i < num_offset; i++) {
printfQuda(" shift=%d, relative residual: iterated = %e, true = %e\n",
i, sqrt(r2[i]/b2), param.true_res_offset[i]);
}
}
// reset the flops counters
quda::blas_flops = 0;
mat.flops();
matSloppy.flops();
profile.Stop(QUDA_PROFILE_EPILOGUE);
profile.Start(QUDA_PROFILE_FREE);
if (&tmp3 != &tmp1) delete tmp3_p;
if (&tmp2 != &tmp1) delete tmp2_p;
if (r_sloppy->Precision() != r->Precision()) delete r_sloppy;
for (int i=0; i<num_offset; i++)
if (x_sloppy[i]->Precision() != x[i]->Precision()) delete x_sloppy[i];
delete []x_sloppy;
delete r;
for (int i=0; i<num_offset; i++) delete p[i];
delete []p;
if (reliable) {
for (int i=0; i<num_offset; i++) delete y[i];
delete []y;
}
delete Ap;
delete []zeta_old;
delete []zeta;
delete []alpha;
delete []beta;
profile.Stop(QUDA_PROFILE_FREE);
return;
}
void MR::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b)
{
globalReduce = false; // use local reductions for DD solver
if (!init) {
ColorSpinorParam csParam(x);
csParam.create = QUDA_ZERO_FIELD_CREATE;
if (param.preserve_source == QUDA_PRESERVE_SOURCE_YES) {
rp = new cudaColorSpinorField(x, csParam);
allocate_r = true;
}
Arp = new cudaColorSpinorField(x);
tmpp = new cudaColorSpinorField(x, csParam); //temporary for mat-vec
init = true;
}
cudaColorSpinorField &r =
(param.preserve_source == QUDA_PRESERVE_SOURCE_YES) ? *rp : b;
cudaColorSpinorField &Ar = *Arp;
cudaColorSpinorField &tmp = *tmpp;
// set initial guess to zero and thus the residual is just the source
zeroCuda(x); // can get rid of this for a special first update kernel
double b2 = normCuda(b);
if (&r != &b) copyCuda(r, b);
// domain-wise normalization of the initial residual to prevent underflow
double r2=0.0; // if zero source then we will exit immediately doing no work
if (b2 > 0.0) {
axCuda(1/sqrt(b2), r); // can merge this with the prior copy
r2 = 1.0; // by definition by this is now true
}
if (param.inv_type_precondition != QUDA_GCR_INVERTER) {
quda::blas_flops = 0;
profile.TPSTART(QUDA_PROFILE_COMPUTE);
}
double omega = 1.0;
int k = 0;
if (getVerbosity() >= QUDA_DEBUG_VERBOSE) {
double x2 = norm2(x);
double3 Ar3 = cDotProductNormBCuda(Ar, r);
printfQuda("MR: %d iterations, r2 = %e, <r|A|r> = (%e, %e), x2 = %e\n",
k, Ar3.z, Ar3.x, Ar3.y, x2);
}
while (k < param.maxiter && r2 > 0.0) {
mat(Ar, r, tmp);
double3 Ar3 = cDotProductNormACuda(Ar, r);
Complex alpha = Complex(Ar3.x, Ar3.y) / Ar3.z;
// x += omega*alpha*r, r -= omega*alpha*Ar, r2 = norm2(r)
//r2 = caxpyXmazNormXCuda(omega*alpha, r, x, Ar);
caxpyXmazCuda(omega*alpha, r, x, Ar);
if (getVerbosity() >= QUDA_DEBUG_VERBOSE) {
double x2 = norm2(x);
double r2 = norm2(r);
printfQuda("MR: %d iterations, r2 = %e, <r|A|r> = (%e,%e) x2 = %e\n",
k+1, r2, Ar3.x, Ar3.y, x2);
} else if (getVerbosity() >= QUDA_VERBOSE) {
printfQuda("MR: %d iterations, <r|A|r> = (%e, %e)\n", k, Ar3.x, Ar3.y);
}
k++;
}
if (getVerbosity() >= QUDA_VERBOSE) {
mat(Ar, r, tmp);
Complex Ar2 = cDotProductCuda(Ar, r);
printfQuda("MR: %d iterations, <r|A|r> = (%e, %e)\n", k, real(Ar2), imag(Ar2));
}
// Obtain global solution by rescaling
if (b2 > 0.0) axCuda(sqrt(b2), x);
if (param.inv_type_precondition != QUDA_GCR_INVERTER) {
profile.TPSTOP(QUDA_PROFILE_COMPUTE);
profile.TPSTART(QUDA_PROFILE_EPILOGUE);
param.secs += profile.Last(QUDA_PROFILE_COMPUTE);
double gflops = (quda::blas_flops + mat.flops())*1e-9;
reduceDouble(gflops);
param.gflops += gflops;
param.iter += k;
// this is the relative residual since it has been scaled by b2
r2 = norm2(r);
if (param.preserve_source == QUDA_PRESERVE_SOURCE_YES) {
// Calculate the true residual
mat(r, x);
double true_res = xmyNormCuda(b, r);
param.true_res = sqrt(true_res / b2);
if (getVerbosity() >= QUDA_SUMMARIZE) {
printfQuda("MR: Converged after %d iterations, relative residua: iterated = %e, true = %e\n",
k, sqrt(r2), param.true_res);
}
} else {
if (getVerbosity() >= QUDA_SUMMARIZE) {
printfQuda("MR: Converged after %d iterations, relative residua: iterated = %e\n", k, sqrt(r2));
}
}
// reset the flops counters
quda::blas_flops = 0;
mat.flops();
profile.TPSTOP(QUDA_PROFILE_EPILOGUE);
}
globalReduce = true; // renable global reductions for outer solver
return;
}
void CG::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b)
{
profile.Start(QUDA_PROFILE_INIT);
// Check to see that we're not trying to invert on a zero-field source
const double b2 = norm2(b);
if(b2 == 0){
profile.Stop(QUDA_PROFILE_INIT);
printfQuda("Warning: inverting on zero-field source\n");
x=b;
param.true_res = 0.0;
param.true_res_hq = 0.0;
return;
}
cudaColorSpinorField r(b);
ColorSpinorParam csParam(x);
csParam.create = QUDA_ZERO_FIELD_CREATE;
cudaColorSpinorField y(b, csParam);
mat(r, x, y);
// zeroCuda(y);
double r2 = xmyNormCuda(b, r);
csParam.setPrecision(param.precision_sloppy);
cudaColorSpinorField Ap(x, csParam);
cudaColorSpinorField tmp(x, csParam);
cudaColorSpinorField *tmp2_p = &tmp;
// tmp only needed for multi-gpu Wilson-like kernels
if (mat.Type() != typeid(DiracStaggeredPC).name() &&
mat.Type() != typeid(DiracStaggered).name()) {
tmp2_p = new cudaColorSpinorField(x, csParam);
}
cudaColorSpinorField &tmp2 = *tmp2_p;
cudaColorSpinorField *x_sloppy, *r_sloppy;
if (param.precision_sloppy == x.Precision()) {
csParam.create = QUDA_REFERENCE_FIELD_CREATE;
x_sloppy = &x;
r_sloppy = &r;
} else {
csParam.create = QUDA_COPY_FIELD_CREATE;
x_sloppy = new cudaColorSpinorField(x, csParam);
r_sloppy = new cudaColorSpinorField(r, csParam);
}
cudaColorSpinorField &xSloppy = *x_sloppy;
cudaColorSpinorField &rSloppy = *r_sloppy;
cudaColorSpinorField p(rSloppy);
if(&x != &xSloppy){
copyCuda(y,x);
zeroCuda(xSloppy);
}else{
zeroCuda(y);
}
const bool use_heavy_quark_res =
(param.residual_type & QUDA_HEAVY_QUARK_RESIDUAL) ? true : false;
profile.Stop(QUDA_PROFILE_INIT);
profile.Start(QUDA_PROFILE_PREAMBLE);
double r2_old;
double stop = b2*param.tol*param.tol; // stopping condition of solver
double heavy_quark_res = 0.0; // heavy quark residual
if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(x,r).z);
int heavy_quark_check = 10; // how often to check the heavy quark residual
double alpha=0.0, beta=0.0;
double pAp;
int rUpdate = 0;
double rNorm = sqrt(r2);
double r0Norm = rNorm;
double maxrx = rNorm;
double maxrr = rNorm;
double delta = param.delta;
// this parameter determines how many consective reliable update
// reisudal increases we tolerate before terminating the solver,
// i.e., how long do we want to keep trying to converge
int maxResIncrease = 0; // 0 means we have no tolerance
profile.Stop(QUDA_PROFILE_PREAMBLE);
profile.Start(QUDA_PROFILE_COMPUTE);
blas_flops = 0;
int k=0;
PrintStats("CG", k, r2, b2, heavy_quark_res);
int steps_since_reliable = 1;
while ( !convergence(r2, heavy_quark_res, stop, param.tol_hq) &&
k < param.maxiter) {
matSloppy(Ap, p, tmp, tmp2); // tmp as tmp
double sigma;
bool breakdown = false;
if (param.pipeline) {
double3 triplet = tripleCGReductionCuda(rSloppy, Ap, p);
r2 = triplet.x; double Ap2 = triplet.y; pAp = triplet.z;
r2_old = r2;
alpha = r2 / pAp;
sigma = alpha*(alpha * Ap2 - pAp);
if (sigma < 0.0 || steps_since_reliable==0) { // sigma condition has broken down
r2 = axpyNormCuda(-alpha, Ap, rSloppy);
sigma = r2;
breakdown = true;
}
r2 = sigma;
} else {
r2_old = r2;
pAp = reDotProductCuda(p, Ap);
alpha = r2 / pAp;
// here we are deploying the alternative beta computation
Complex cg_norm = axpyCGNormCuda(-alpha, Ap, rSloppy);
r2 = real(cg_norm); // (r_new, r_new)
sigma = imag(cg_norm) >= 0.0 ? imag(cg_norm) : r2; // use r2 if (r_k+1, r_k+1-r_k) breaks
}
// reliable update conditions
rNorm = sqrt(r2);
if (rNorm > maxrx) maxrx = rNorm;
if (rNorm > maxrr) maxrr = rNorm;
int updateX = (rNorm < delta*r0Norm && r0Norm <= maxrx) ? 1 : 0;
int updateR = ((rNorm < delta*maxrr && r0Norm <= maxrr) || updateX) ? 1 : 0;
// force a reliable update if we are within target tolerance (only if doing reliable updates)
if ( convergence(r2, heavy_quark_res, stop, param.tol_hq) && delta >= param.tol) updateX = 1;
if ( !(updateR || updateX)) {
//beta = r2 / r2_old;
beta = sigma / r2_old; // use the alternative beta computation
if (param.pipeline && !breakdown) tripleCGUpdateCuda(alpha, beta, Ap, rSloppy, xSloppy, p);
else axpyZpbxCuda(alpha, p, xSloppy, rSloppy, beta);
if (use_heavy_quark_res && k%heavy_quark_check==0) {
copyCuda(tmp,y);
heavy_quark_res = sqrt(xpyHeavyQuarkResidualNormCuda(xSloppy, tmp, rSloppy).z);
}
steps_since_reliable++;
} else {
axpyCuda(alpha, p, xSloppy);
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(x, y); // swap these around?
mat(r, y, x); // here we can use x as tmp
r2 = xmyNormCuda(b, r);
if (x.Precision() != rSloppy.Precision()) copyCuda(rSloppy, r);
zeroCuda(xSloppy);
// break-out check if we have reached the limit of the precision
static int resIncrease = 0;
if (sqrt(r2) > r0Norm && updateX) { // reuse r0Norm for this
warningQuda("CG: new reliable residual norm %e is greater than previous reliable residual norm %e", sqrt(r2), r0Norm);
k++;
rUpdate++;
if (++resIncrease > maxResIncrease) break;
} else {
resIncrease = 0;
}
rNorm = sqrt(r2);
maxrr = rNorm;
maxrx = rNorm;
r0Norm = rNorm;
rUpdate++;
// explicitly restore the orthogonality of the gradient vector
double rp = reDotProductCuda(rSloppy, p) / (r2);
axpyCuda(-rp, rSloppy, p);
beta = r2 / r2_old;
xpayCuda(rSloppy, beta, p);
if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(y,r).z);
steps_since_reliable = 0;
}
breakdown = false;
k++;
PrintStats("CG", k, r2, b2, heavy_quark_res);
}
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(y, x);
profile.Stop(QUDA_PROFILE_COMPUTE);
profile.Start(QUDA_PROFILE_EPILOGUE);
param.secs = profile.Last(QUDA_PROFILE_COMPUTE);
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9;
reduceDouble(gflops);
param.gflops = gflops;
param.iter += k;
if (k==param.maxiter)
warningQuda("Exceeded maximum iterations %d", param.maxiter);
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("CG: Reliable updates = %d\n", rUpdate);
// compute the true residuals
mat(r, x, y);
param.true_res = sqrt(xmyNormCuda(b, r) / b2);
#if (__COMPUTE_CAPABILITY__ >= 200)
param.true_res_hq = sqrt(HeavyQuarkResidualNormCuda(x,r).z);
#else
param.true_res_hq = 0.0;
#endif
PrintSummary("CG", k, r2, b2);
// reset the flops counters
quda::blas_flops = 0;
mat.flops();
matSloppy.flops();
profile.Stop(QUDA_PROFILE_EPILOGUE);
profile.Start(QUDA_PROFILE_FREE);
if (&tmp2 != &tmp) delete tmp2_p;
if (param.precision_sloppy != x.Precision()) {
delete r_sloppy;
delete x_sloppy;
}
profile.Stop(QUDA_PROFILE_FREE);
return;
}
void PreconCG::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b)
{
profile.Start(QUDA_PROFILE_INIT);
// Check to see that we're not trying to invert on a zero-field source
const double b2 = norm2(b);
if(b2 == 0){
profile.Stop(QUDA_PROFILE_INIT);
printfQuda("Warning: inverting on zero-field source\n");
x=b;
param.true_res = 0.0;
param.true_res_hq = 0.0;
}
int k=0;
int rUpdate=0;
cudaColorSpinorField* minvrPre;
cudaColorSpinorField* rPre;
cudaColorSpinorField* minvr;
cudaColorSpinorField* minvrSloppy;
cudaColorSpinorField* p;
ColorSpinorParam csParam(b);
cudaColorSpinorField r(b);
if(K) minvr = new cudaColorSpinorField(b);
csParam.create = QUDA_ZERO_FIELD_CREATE;
cudaColorSpinorField y(b,csParam);
mat(r, x, y); // => r = A*x;
double r2 = xmyNormCuda(b,r);
csParam.setPrecision(param.precision_sloppy);
cudaColorSpinorField tmpSloppy(x,csParam);
cudaColorSpinorField Ap(x,csParam);
cudaColorSpinorField *r_sloppy;
if(param.precision_sloppy == x.Precision())
{
r_sloppy = &r;
minvrSloppy = minvr;
}else{
csParam.create = QUDA_COPY_FIELD_CREATE;
r_sloppy = new cudaColorSpinorField(r,csParam);
if(K) minvrSloppy = new cudaColorSpinorField(*minvr,csParam);
}
cudaColorSpinorField *x_sloppy;
if(param.precision_sloppy == x.Precision() ||
!param.use_sloppy_partial_accumulator) {
csParam.create = QUDA_REFERENCE_FIELD_CREATE;
x_sloppy = &x;
}else{
csParam.create = QUDA_COPY_FIELD_CREATE;
x_sloppy = new cudaColorSpinorField(x,csParam);
}
cudaColorSpinorField &xSloppy = *x_sloppy;
cudaColorSpinorField &rSloppy = *r_sloppy;
if(&x != &xSloppy){
copyCuda(y, x); // copy x to y
zeroCuda(xSloppy);
}else{
zeroCuda(y); // no reliable updates // NB: check this
}
const bool use_heavy_quark_res = (param.residual_type & QUDA_HEAVY_QUARK_RESIDUAL) ? true : false;
if(K){
csParam.create = QUDA_COPY_FIELD_CREATE;
csParam.setPrecision(param.precision_precondition);
rPre = new cudaColorSpinorField(rSloppy,csParam);
// Create minvrPre
minvrPre = new cudaColorSpinorField(*rPre);
globalReduce = false;
(*K)(*minvrPre, *rPre);
globalReduce = true;
*minvrSloppy = *minvrPre;
p = new cudaColorSpinorField(*minvrSloppy);
}else{
p = new cudaColorSpinorField(rSloppy);
}
profile.Stop(QUDA_PROFILE_INIT);
profile.Start(QUDA_PROFILE_PREAMBLE);
double stop = stopping(param.tol, b2, param.residual_type); // stopping condition of solver
double heavy_quark_res = 0.0; // heavy quark residual
if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(x,r).z);
int heavy_quark_check = 10; // how often to check the heavy quark residual
double alpha = 0.0, beta=0.0;
double pAp;
double rMinvr = 0;
double rMinvr_old = 0.0;
double r_new_Minvr_old = 0.0;
double r2_old = 0;
r2 = norm2(r);
double rNorm = sqrt(r2);
double r0Norm = rNorm;
double maxrx = rNorm;
double maxrr = rNorm;
double delta = param.delta;
if(K) rMinvr = reDotProductCuda(rSloppy,*minvrSloppy);
profile.Stop(QUDA_PROFILE_PREAMBLE);
profile.Start(QUDA_PROFILE_COMPUTE);
quda::blas_flops = 0;
int steps_since_reliable = 1;
const int maxResIncrease = 0;
while(!convergence(r2, heavy_quark_res, stop, param.tol_hq) && k < param.maxiter){
matSloppy(Ap, *p, tmpSloppy);
double sigma;
bool breakdown = false;
pAp = reDotProductCuda(*p,Ap);
alpha = (K) ? rMinvr/pAp : r2/pAp;
Complex cg_norm = axpyCGNormCuda(-alpha, Ap, rSloppy);
// r --> r - alpha*A*p
r2_old = r2;
r2 = real(cg_norm);
sigma = imag(cg_norm) >= 0.0 ? imag(cg_norm) : r2; // use r2 if (r_k+1, r_k-1 - r_k) breaks
if(K) rMinvr_old = rMinvr;
rNorm = sqrt(r2);
if(rNorm > maxrx) maxrx = rNorm;
if(rNorm > maxrr) maxrr = rNorm;
int updateX = (rNorm < delta*r0Norm && r0Norm <= maxrx) ? 1 : 0;
int updateR = ((rNorm < delta*maxrr && r0Norm <= maxrr) || updateX) ? 1 : 0;
// force a reliable update if we are within target tolerance (only if doing reliable updates)
if( convergence(r2, heavy_quark_res, stop, param.tol_hq) && delta >= param.tol) updateX = 1;
if( !(updateR || updateX) ){
if(K){
r_new_Minvr_old = reDotProductCuda(rSloppy,*minvrSloppy);
*rPre = rSloppy;
globalReduce = false;
(*K)(*minvrPre, *rPre);
globalReduce = true;
*minvrSloppy = *minvrPre;
rMinvr = reDotProductCuda(rSloppy,*minvrSloppy);
beta = (rMinvr - r_new_Minvr_old)/rMinvr_old;
axpyZpbxCuda(alpha, *p, xSloppy, *minvrSloppy, beta);
}else{
beta = sigma/r2_old; // use the alternative beta computation
axpyZpbxCuda(alpha, *p, xSloppy, rSloppy, beta);
}
} else { // reliable update
axpyCuda(alpha, *p, xSloppy); // xSloppy += alpha*p
copyCuda(x, xSloppy);
xpyCuda(x, y); // y += x
// Now compute r
mat(r, y, x); // x is just a temporary here
r2 = xmyNormCuda(b, r);
copyCuda(rSloppy, r); // copy r to rSloppy
zeroCuda(xSloppy);
// break-out check if we have reached the limit of the precision
static int resIncrease = 0;
if(sqrt(r2) > r0Norm && updateX) { // reuse r0Norm for this
warningQuda("PCG: new reliable residual norm %e is greater than previous reliable residual norm %e", sqrt(r2), r0Norm);
k++;
rUpdate++;
if(++resIncrease > maxResIncrease) break;
}else{
resIncrease = 0;
}
rNorm = sqrt(r2);
maxrr = rNorm;
maxrx = rNorm;
r0Norm = rNorm;
++rUpdate;
if(K){
*rPre = rSloppy;
globalReduce = false;
(*K)(*minvrPre, *rPre);
globalReduce = true;
*minvrSloppy = *minvrPre;
rMinvr = reDotProductCuda(rSloppy,*minvrSloppy);
beta = rMinvr/rMinvr_old;
xpayCuda(*minvrSloppy, beta, *p); // p = minvrSloppy + beta*p
}else{ // standard CG - no preconditioning
// explicitly restore the orthogonality of the gradient vector
double rp = reDotProductCuda(rSloppy, *p)/(r2);
axpyCuda(-rp, rSloppy, *p);
beta = r2/r2_old;
xpayCuda(rSloppy, beta, *p);
steps_since_reliable = 0;
}
}
breakdown = false;
++k;
PrintStats("PCG", k, r2, b2, heavy_quark_res);
}
profile.Stop(QUDA_PROFILE_COMPUTE);
profile.Start(QUDA_PROFILE_EPILOGUE);
if(x.Precision() != param.precision_sloppy) copyCuda(x, xSloppy);
xpyCuda(y, x); // x += y
param.secs = profile.Last(QUDA_PROFILE_COMPUTE);
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops() + matPrecon.flops())*1e-9;
reduceDouble(gflops);
param.gflops = gflops;
param.iter += k;
if (k==param.maxiter)
warningQuda("Exceeded maximum iterations %d", param.maxiter);
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("CG: Reliable updates = %d\n", rUpdate);
// compute the true residual
mat(r, x, y);
double true_res = xmyNormCuda(b, r);
param.true_res = sqrt(true_res / b2);
// reset the flops counters
quda::blas_flops = 0;
mat.flops();
matSloppy.flops();
matPrecon.flops();
profile.Stop(QUDA_PROFILE_EPILOGUE);
profile.Start(QUDA_PROFILE_FREE);
if(K){ // These are only needed if preconditioning is used
delete minvrPre;
delete rPre;
delete minvr;
if(x.Precision() != param.precision_sloppy) delete minvrSloppy;
}
delete p;
if(x.Precision() != param.precision_sloppy){
delete x_sloppy;
delete r_sloppy;
}
profile.Stop(QUDA_PROFILE_FREE);
return;
}
