void MultiShiftCG::operator()(cudaColorSpinorField **x, cudaColorSpinorField &b)
{
int num_offset = invParam.num_offset;
double *offset = invParam.offset;
double *residue_sq = invParam.tol_offset;
if (num_offset == 0) return;
int *finished = new int [num_offset];
double *zeta_i = new double[num_offset];
double *zeta_im1 = new double[num_offset];
double *zeta_ip1 = new double[num_offset];
double *beta_i = new double[num_offset];
double *beta_im1 = new double[num_offset];
double *alpha = new double[num_offset];
int i, j;
int j_low = 0;
int num_offset_now = num_offset;
for (i=0; i<num_offset; i++) {
finished[i] = 0;
zeta_im1[i] = zeta_i[i] = 1.0;
beta_im1[i] = -1.0;
alpha[i] = 0.0;
}
//double msq_x4 = offset[0];
cudaColorSpinorField *r = new cudaColorSpinorField(b);
cudaColorSpinorField **x_sloppy = new cudaColorSpinorField*[num_offset], *r_sloppy;
ColorSpinorParam param;
param.create = QUDA_ZERO_FIELD_CREATE;
param.precision = invParam.cuda_prec_sloppy;
if (invParam.cuda_prec_sloppy == x[0]->Precision()) {
for (i=0; i<num_offset; i++){
x_sloppy[i] = x[i];
zeroCuda(*x_sloppy[i]);
}
r_sloppy = r;
} else {
for (i=0; i<num_offset; i++) {
x_sloppy[i] = new cudaColorSpinorField(*x[i], param);
}
param.create = QUDA_COPY_FIELD_CREATE;
r_sloppy = new cudaColorSpinorField(*r, param);
}
cudaColorSpinorField **p = new cudaColorSpinorField*[num_offset];
for(i=0;i < num_offset;i++){
p[i]= new cudaColorSpinorField(*r_sloppy);
}
param.create = QUDA_ZERO_FIELD_CREATE;
param.precision = invParam.cuda_prec_sloppy;
cudaColorSpinorField* Ap = new cudaColorSpinorField(*r_sloppy, param);
double b2 = 0.0;
b2 = normCuda(b);
double r2 = b2;
double r2_old;
double stop = r2*invParam.tol*invParam.tol; // stopping condition of solver
double pAp;
int k = 0;
stopwatchStart();
while (r2 > stop && k < invParam.maxiter) {
//dslashCuda_st(tmp_sloppy, fatlinkSloppy, longlinkSloppy, p[0], 1 - oddBit, 0);
//dslashAxpyCuda(Ap, fatlinkSloppy, longlinkSloppy, tmp_sloppy, oddBit, 0, p[0], msq_x4);
matSloppy(*Ap, *p[0]);
if (invParam.dslash_type != QUDA_ASQTAD_DSLASH){
axpyCuda(offset[0], *p[0], *Ap);
}
pAp = reDotProductCuda(*p[0], *Ap);
beta_i[0] = r2 / pAp;
zeta_ip1[0] = 1.0;
for (j=1; j<num_offset_now; j++) {
zeta_ip1[j] = zeta_i[j] * zeta_im1[j] * beta_im1[j_low];
double c1 = beta_i[j_low] * alpha[j_low] * (zeta_im1[j]-zeta_i[j]);
double c2 = zeta_im1[j] * beta_im1[j_low] * (1.0+(offset[j]-offset[0])*beta_i[j_low]);
/*THISBLOWSUP
zeta_ip1[j] /= c1 + c2;
beta_i[j] = beta_i[j_low] * zeta_ip1[j] / zeta_i[j];
*/
/*TRYTHIS*/
if( (c1+c2) != 0.0 )
zeta_ip1[j] /= (c1 + c2);
else {
zeta_ip1[j] = 0.0;
finished[j] = 1;
}
if( zeta_i[j] != 0.0) {
beta_i[j] = beta_i[j_low] * zeta_ip1[j] / zeta_i[j];
} else {
zeta_ip1[j] = 0.0;
beta_i[j] = 0.0;
finished[j] = 1;
if (invParam.verbosity >= QUDA_VERBOSE)
printfQuda("SETTING A ZERO, j=%d, num_offset_now=%d\n",j,num_offset_now);
//if(j==num_offset_now-1)node0_PRINTF("REDUCING OFFSET\n");
if(j==num_offset_now-1) num_offset_now--;
// don't work any more on finished solutions
// this only works if largest offsets are last, otherwise
// just wastes time multiplying by zero
}
}
r2_old = r2;
r2 = axpyNormCuda(-beta_i[j_low], *Ap, *r_sloppy);
alpha[0] = r2 / r2_old;
for (j=1; j<num_offset_now; j++) {
/*THISBLOWSUP
alpha[j] = alpha[j_low] * zeta_ip1[j] * beta_i[j] /
(zeta_i[j] * beta_i[j_low]);
*/
/*TRYTHIS*/
if( zeta_i[j] * beta_i[j_low] != 0.0)
alpha[j] = alpha[j_low] * zeta_ip1[j] * beta_i[j] /
(zeta_i[j] * beta_i[j_low]);
else {
alpha[j] = 0.0;
finished[j] = 1;
}
}
axpyZpbxCuda(beta_i[0], *p[0], *x_sloppy[0], *r_sloppy, alpha[0]);
for (j=1; j<num_offset_now; j++) {
axpyBzpcxCuda(beta_i[j], *p[j], *x_sloppy[j], zeta_ip1[j], *r_sloppy, alpha[j]);
}
for (j=0; j<num_offset_now; j++) {
beta_im1[j] = beta_i[j];
zeta_im1[j] = zeta_i[j];
zeta_i[j] = zeta_ip1[j];
}
k++;
if (invParam.verbosity >= QUDA_VERBOSE){
printfQuda("Multimass CG: %d iterations, r2 = %e\n", k, r2);
}
}
if (x[0]->Precision() != x_sloppy[0]->Precision()) {
for(i=0;i < num_offset; i++){
copyCuda(*x[i], *x_sloppy[i]);
}
}
*residue_sq = r2;
invParam.secs = stopwatchReadSeconds();
if (k==invParam.maxiter) {
warningQuda("Exceeded maximum iterations %d\n", invParam.maxiter);
}
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9;
reduceDouble(gflops);
invParam.gflops = gflops;
invParam.iter = k;
// Calculate the true residual of the system with the smallest shift
mat(*r, *x[0]);
axpyCuda(offset[0],*x[0], *r); // Offset it.
double true_res = xmyNormCuda(b, *r);
if (invParam.verbosity >= QUDA_SUMMARIZE){
printfQuda("MultiShift CG: Converged after %d iterations, r2 = %e, relative true_r2 = %e\n",
k,r2, (true_res / b2));
}
if (invParam.verbosity >= QUDA_VERBOSE){
printfQuda("MultiShift CG: Converged after %d iterations\n", k);
printfQuda(" shift=0 resid_rel=%e\n", sqrt(true_res/b2));
for(int i=1; i < num_offset; i++) {
mat(*r, *x[i]);
axpyCuda(offset[i],*x[i], *r); // Offset it.
true_res = xmyNormCuda(b, *r);
printfQuda(" shift=%d resid_rel=%e\n",i, sqrt(true_res/b2));
}
}
delete r;
for(i=0;i < num_offset; i++){
delete p[i];
}
delete p;
delete Ap;
if (invParam.cuda_prec_sloppy != x[0]->Precision()) {
for(i=0;i < num_offset;i++){
delete x_sloppy[i];
}
delete r_sloppy;
}
delete x_sloppy;
delete []finished;
delete []zeta_i;
delete []zeta_im1;
delete []zeta_ip1;
delete []beta_i;
delete []beta_im1;
delete []alpha;
}
void CG::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b)
{
int k=0;
int rUpdate = 0;
cudaColorSpinorField r(b);
ColorSpinorParam param(x);
param.create = QUDA_ZERO_FIELD_CREATE;
cudaColorSpinorField y(b, param);
mat(r, x, y);
zeroCuda(y);
double r2 = xmyNormCuda(b, r);
rUpdate ++;
param.precision = invParam.cuda_prec_sloppy;
cudaColorSpinorField Ap(x, param);
cudaColorSpinorField tmp(x, param);
cudaColorSpinorField tmp2(x, param); // only needed for clover and twisted mass
cudaColorSpinorField *x_sloppy, *r_sloppy;
if (invParam.cuda_prec_sloppy == x.Precision()) {
param.create = QUDA_REFERENCE_FIELD_CREATE;
x_sloppy = &x;
r_sloppy = &r;
} else {
param.create = QUDA_COPY_FIELD_CREATE;
x_sloppy = new cudaColorSpinorField(x, param);
r_sloppy = new cudaColorSpinorField(r, param);
}
cudaColorSpinorField &xSloppy = *x_sloppy;
cudaColorSpinorField &rSloppy = *r_sloppy;
cudaColorSpinorField p(rSloppy);
double r2_old;
double src_norm = norm2(b);
double stop = src_norm*invParam.tol*invParam.tol; // stopping condition of solver
double alpha, beta;
double pAp;
double rNorm = sqrt(r2);
double r0Norm = rNorm;
double maxrx = rNorm;
double maxrr = rNorm;
double delta = invParam.reliable_delta;
if (invParam.verbosity >= QUDA_VERBOSE) printfQuda("CG: %d iterations, r2 = %e\n", k, r2);
quda::blas_flops = 0;
stopwatchStart();
while (r2 > stop && k<invParam.maxiter) {
matSloppy(Ap, p, tmp, tmp2); // tmp as tmp
pAp = reDotProductCuda(p, Ap);
alpha = r2 / pAp;
r2_old = r2;
r2 = axpyNormCuda(-alpha, Ap, rSloppy);
// reliable update conditions
rNorm = sqrt(r2);
if (rNorm > maxrx) maxrx = rNorm;
if (rNorm > maxrr) maxrr = rNorm;
int updateX = (rNorm < delta*r0Norm && r0Norm <= maxrx) ? 1 : 0;
int updateR = ((rNorm < delta*maxrr && r0Norm <= maxrr) || updateX) ? 1 : 0;
if (!(updateR || updateX)) {
beta = r2 / r2_old;
axpyZpbxCuda(alpha, p, xSloppy, rSloppy, beta);
} else {
axpyCuda(alpha, p, xSloppy);
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(x, y); // swap these around?
mat(r, y, x); // here we can use x as tmp
r2 = xmyNormCuda(b, r);
if (x.Precision() != rSloppy.Precision()) copyCuda(rSloppy, r);
zeroCuda(xSloppy);
rNorm = sqrt(r2);
maxrr = rNorm;
maxrx = rNorm;
r0Norm = rNorm;
rUpdate++;
beta = r2 / r2_old;
xpayCuda(rSloppy, beta, p);
}
k++;
if (invParam.verbosity >= QUDA_VERBOSE)
printfQuda("CG: %d iterations, r2 = %e\n", k, r2);
}
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(y, x);
invParam.secs = stopwatchReadSeconds();
if (k==invParam.maxiter)
warningQuda("Exceeded maximum iterations %d", invParam.maxiter);
if (invParam.verbosity >= QUDA_SUMMARIZE)
printfQuda("CG: Reliable updates = %d\n", rUpdate);
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9;
reduceDouble(gflops);
// printfQuda("%f gflops\n", gflops / stopwatchReadSeconds());
invParam.gflops = gflops;
invParam.iter = k;
quda::blas_flops = 0;
if (invParam.verbosity >= QUDA_SUMMARIZE){
mat(r, x, y);
double true_res = xmyNormCuda(b, r);
printfQuda("CG: Converged after %d iterations, relative residua: iterated = %e, true = %e\n",
k, sqrt(r2/src_norm), sqrt(true_res / src_norm));
}
if (invParam.cuda_prec_sloppy != x.Precision()) {
delete r_sloppy;
delete x_sloppy;
}
return;
}
void CG::operator()(cudaColorSpinorField &x, cudaColorSpinorField &b)
{
profile.Start(QUDA_PROFILE_INIT);
// Check to see that we're not trying to invert on a zero-field source
const double b2 = norm2(b);
if(b2 == 0){
profile.Stop(QUDA_PROFILE_INIT);
printfQuda("Warning: inverting on zero-field source\n");
x=b;
param.true_res = 0.0;
param.true_res_hq = 0.0;
return;
}
cudaColorSpinorField r(b);
ColorSpinorParam csParam(x);
csParam.create = QUDA_ZERO_FIELD_CREATE;
cudaColorSpinorField y(b, csParam);
mat(r, x, y);
// zeroCuda(y);
double r2 = xmyNormCuda(b, r);
csParam.setPrecision(param.precision_sloppy);
cudaColorSpinorField Ap(x, csParam);
cudaColorSpinorField tmp(x, csParam);
cudaColorSpinorField *tmp2_p = &tmp;
// tmp only needed for multi-gpu Wilson-like kernels
if (mat.Type() != typeid(DiracStaggeredPC).name() &&
mat.Type() != typeid(DiracStaggered).name()) {
tmp2_p = new cudaColorSpinorField(x, csParam);
}
cudaColorSpinorField &tmp2 = *tmp2_p;
cudaColorSpinorField *x_sloppy, *r_sloppy;
if (param.precision_sloppy == x.Precision()) {
csParam.create = QUDA_REFERENCE_FIELD_CREATE;
x_sloppy = &x;
r_sloppy = &r;
} else {
csParam.create = QUDA_COPY_FIELD_CREATE;
x_sloppy = new cudaColorSpinorField(x, csParam);
r_sloppy = new cudaColorSpinorField(r, csParam);
}
cudaColorSpinorField &xSloppy = *x_sloppy;
cudaColorSpinorField &rSloppy = *r_sloppy;
cudaColorSpinorField p(rSloppy);
if(&x != &xSloppy){
copyCuda(y,x);
zeroCuda(xSloppy);
}else{
zeroCuda(y);
}
const bool use_heavy_quark_res =
(param.residual_type & QUDA_HEAVY_QUARK_RESIDUAL) ? true : false;
profile.Stop(QUDA_PROFILE_INIT);
profile.Start(QUDA_PROFILE_PREAMBLE);
double r2_old;
double stop = b2*param.tol*param.tol; // stopping condition of solver
double heavy_quark_res = 0.0; // heavy quark residual
if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(x,r).z);
int heavy_quark_check = 10; // how often to check the heavy quark residual
double alpha=0.0, beta=0.0;
double pAp;
int rUpdate = 0;
double rNorm = sqrt(r2);
double r0Norm = rNorm;
double maxrx = rNorm;
double maxrr = rNorm;
double delta = param.delta;
// this parameter determines how many consective reliable update
// reisudal increases we tolerate before terminating the solver,
// i.e., how long do we want to keep trying to converge
int maxResIncrease = 0; // 0 means we have no tolerance
profile.Stop(QUDA_PROFILE_PREAMBLE);
profile.Start(QUDA_PROFILE_COMPUTE);
blas_flops = 0;
int k=0;
PrintStats("CG", k, r2, b2, heavy_quark_res);
int steps_since_reliable = 1;
while ( !convergence(r2, heavy_quark_res, stop, param.tol_hq) &&
k < param.maxiter) {
matSloppy(Ap, p, tmp, tmp2); // tmp as tmp
double sigma;
bool breakdown = false;
if (param.pipeline) {
double3 triplet = tripleCGReductionCuda(rSloppy, Ap, p);
r2 = triplet.x; double Ap2 = triplet.y; pAp = triplet.z;
r2_old = r2;
alpha = r2 / pAp;
sigma = alpha*(alpha * Ap2 - pAp);
if (sigma < 0.0 || steps_since_reliable==0) { // sigma condition has broken down
r2 = axpyNormCuda(-alpha, Ap, rSloppy);
sigma = r2;
breakdown = true;
}
r2 = sigma;
} else {
r2_old = r2;
pAp = reDotProductCuda(p, Ap);
alpha = r2 / pAp;
// here we are deploying the alternative beta computation
Complex cg_norm = axpyCGNormCuda(-alpha, Ap, rSloppy);
r2 = real(cg_norm); // (r_new, r_new)
sigma = imag(cg_norm) >= 0.0 ? imag(cg_norm) : r2; // use r2 if (r_k+1, r_k+1-r_k) breaks
}
// reliable update conditions
rNorm = sqrt(r2);
if (rNorm > maxrx) maxrx = rNorm;
if (rNorm > maxrr) maxrr = rNorm;
int updateX = (rNorm < delta*r0Norm && r0Norm <= maxrx) ? 1 : 0;
int updateR = ((rNorm < delta*maxrr && r0Norm <= maxrr) || updateX) ? 1 : 0;
// force a reliable update if we are within target tolerance (only if doing reliable updates)
if ( convergence(r2, heavy_quark_res, stop, param.tol_hq) && delta >= param.tol) updateX = 1;
if ( !(updateR || updateX)) {
//beta = r2 / r2_old;
beta = sigma / r2_old; // use the alternative beta computation
if (param.pipeline && !breakdown) tripleCGUpdateCuda(alpha, beta, Ap, rSloppy, xSloppy, p);
else axpyZpbxCuda(alpha, p, xSloppy, rSloppy, beta);
if (use_heavy_quark_res && k%heavy_quark_check==0) {
copyCuda(tmp,y);
heavy_quark_res = sqrt(xpyHeavyQuarkResidualNormCuda(xSloppy, tmp, rSloppy).z);
}
steps_since_reliable++;
} else {
axpyCuda(alpha, p, xSloppy);
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(x, y); // swap these around?
mat(r, y, x); // here we can use x as tmp
r2 = xmyNormCuda(b, r);
if (x.Precision() != rSloppy.Precision()) copyCuda(rSloppy, r);
zeroCuda(xSloppy);
// break-out check if we have reached the limit of the precision
static int resIncrease = 0;
if (sqrt(r2) > r0Norm && updateX) { // reuse r0Norm for this
warningQuda("CG: new reliable residual norm %e is greater than previous reliable residual norm %e", sqrt(r2), r0Norm);
k++;
rUpdate++;
if (++resIncrease > maxResIncrease) break;
} else {
resIncrease = 0;
}
rNorm = sqrt(r2);
maxrr = rNorm;
maxrx = rNorm;
r0Norm = rNorm;
rUpdate++;
// explicitly restore the orthogonality of the gradient vector
double rp = reDotProductCuda(rSloppy, p) / (r2);
axpyCuda(-rp, rSloppy, p);
beta = r2 / r2_old;
xpayCuda(rSloppy, beta, p);
if(use_heavy_quark_res) heavy_quark_res = sqrt(HeavyQuarkResidualNormCuda(y,r).z);
steps_since_reliable = 0;
}
breakdown = false;
k++;
PrintStats("CG", k, r2, b2, heavy_quark_res);
}
if (x.Precision() != xSloppy.Precision()) copyCuda(x, xSloppy);
xpyCuda(y, x);
profile.Stop(QUDA_PROFILE_COMPUTE);
profile.Start(QUDA_PROFILE_EPILOGUE);
param.secs = profile.Last(QUDA_PROFILE_COMPUTE);
double gflops = (quda::blas_flops + mat.flops() + matSloppy.flops())*1e-9;
reduceDouble(gflops);
param.gflops = gflops;
param.iter += k;
if (k==param.maxiter)
warningQuda("Exceeded maximum iterations %d", param.maxiter);
if (getVerbosity() >= QUDA_VERBOSE)
printfQuda("CG: Reliable updates = %d\n", rUpdate);
// compute the true residuals
mat(r, x, y);
param.true_res = sqrt(xmyNormCuda(b, r) / b2);
#if (__COMPUTE_CAPABILITY__ >= 200)
param.true_res_hq = sqrt(HeavyQuarkResidualNormCuda(x,r).z);
#else
param.true_res_hq = 0.0;
#endif
PrintSummary("CG", k, r2, b2);
// reset the flops counters
quda::blas_flops = 0;
mat.flops();
matSloppy.flops();
profile.Stop(QUDA_PROFILE_EPILOGUE);
profile.Start(QUDA_PROFILE_FREE);
if (&tmp2 != &tmp) delete tmp2_p;
if (param.precision_sloppy != x.Precision()) {
delete r_sloppy;
delete x_sloppy;
}
profile.Stop(QUDA_PROFILE_FREE);
return;
}
