void DiracStaggered::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
if (!initDslash){
initDslashConstants(*fatGauge, in.Stride());
initStaggeredConstants(*fatGauge, *longGauge);
}
bool reset = newTmp(&tmp1, in);
cudaColorSpinorField* mytmp = dynamic_cast<cudaColorSpinorField*>(&(tmp1->Even()));
cudaColorSpinorField* ineven = dynamic_cast<cudaColorSpinorField*>(&(in.Even()));
cudaColorSpinorField* inodd = dynamic_cast<cudaColorSpinorField*>(&(in.Odd()));
cudaColorSpinorField* outeven = dynamic_cast<cudaColorSpinorField*>(&(out.Even()));
cudaColorSpinorField* outodd = dynamic_cast<cudaColorSpinorField*>(&(out.Odd()));
//even
Dslash(*mytmp, *ineven, QUDA_ODD_PARITY);
DslashXpay(*outeven, *mytmp, QUDA_EVEN_PARITY, *ineven, 4*mass*mass);
//odd
Dslash(*mytmp, *inodd, QUDA_EVEN_PARITY);
DslashXpay(*outodd, *mytmp, QUDA_ODD_PARITY, *inodd, 4*mass*mass);
deleteTmp(&tmp1, reset);
}
void DiracStaggeredPC::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
if (!initDslash){
initDslashConstants(*fatGauge, in.Stride());
initStaggeredConstants(*fatGauge, *longGauge);
}
bool reset = newTmp(&tmp1, in);
QudaParity parity = QUDA_INVALID_PARITY;
QudaParity other_parity = QUDA_INVALID_PARITY;
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
parity = QUDA_EVEN_PARITY;
other_parity = QUDA_ODD_PARITY;
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
parity = QUDA_ODD_PARITY;
other_parity = QUDA_EVEN_PARITY;
} else {
errorQuda("Invalid matpcType(%d) in function\n", matpcType);
}
Dslash(*tmp1, in, other_parity);
DslashXpay(out, *tmp1, parity, in, 4*mass*mass);
deleteTmp(&tmp1, reset);
}
void DiracCloverPC::reconstruct(cudaColorSpinorField &x, const cudaColorSpinorField &b,
const QudaSolutionType solType) const
{
if (solType == QUDA_MATPC_SOLUTION || solType == QUDA_MATPCDAG_MATPC_SOLUTION) {
return;
}
checkFullSpinor(x, b);
bool reset = newTmp(&tmp1, b.Even());
// create full solution
if (matpcType == QUDA_MATPC_EVEN_EVEN ||
matpcType == QUDA_MATPC_EVEN_EVEN_ASYMMETRIC) {
// x_o = A_oo^-1 (b_o + k D_oe x_e)
DiracWilson::DslashXpay(*tmp1, x.Even(), QUDA_ODD_PARITY, b.Odd(), kappa);
CloverInv(x.Odd(), *tmp1, QUDA_ODD_PARITY);
} else if (matpcType == QUDA_MATPC_ODD_ODD ||
matpcType == QUDA_MATPC_ODD_ODD_ASYMMETRIC) {
// x_e = A_ee^-1 (b_e + k D_eo x_o)
DiracWilson::DslashXpay(*tmp1, x.Odd(), QUDA_EVEN_PARITY, b.Even(), kappa);
CloverInv(x.Even(), *tmp1, QUDA_EVEN_PARITY);
} else {
errorQuda("MatPCType %d not valid for DiracCloverPC", matpcType);
}
deleteTmp(&tmp1, reset);
}
// Full staggered operator
void DiracStaggered::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
bool reset = newTmp(&tmp1, in.Even());
DslashXpay(out.Even(), in.Odd(), QUDA_EVEN_PARITY, *tmp1, 2*mass);
DslashXpay(out.Odd(), in.Even(), QUDA_ODD_PARITY, *tmp1, 2*mass);
deleteTmp(&tmp1, reset);
}
void DiracCloverPC::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
// need extra temporary because of symmetric preconditioning dagger
// and for multi-gpu the input and output fields cannot alias
bool reset = newTmp(&tmp2, in);
M(*tmp2, in);
Mdag(out, *tmp2);
deleteTmp(&tmp2, reset);
}
void DiracDomainWallPC::MdagM(ColorSpinorField &out, const ColorSpinorField &in) const
{
//M(out, in);
//Mdag(out, out);
bool reset = newTmp(&tmp2, in);
M(*tmp2, in);
Mdag(out, *tmp2);
deleteTmp(&tmp2, reset);
}
void DiracCloverPC::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
// need extra temporary because of symmetric preconditioning dagger
bool reset = newTmp(&tmp2, in);
M(*tmp2, in);
Mdag(out, *tmp2);
deleteTmp(&tmp2, reset);
}
// Apply the even-odd preconditioned clover-improved Dirac operator
void DiracCloverPC::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
double kappa2 = -kappa*kappa;
// FIXME: For asymmetric, a "DslashCxpay" kernel would improve performance.
bool reset = newTmp(&tmp1, in);
if (matpcType == QUDA_MATPC_EVEN_EVEN_ASYMMETRIC) {
bool reset = newTmp(&tmp2, in);
// DiracCloverPC::Dslash applies A^{-1}Dslash
Dslash(*tmp1, in, QUDA_ODD_PARITY);
Clover(*tmp2, in, QUDA_EVEN_PARITY);
// DiracWilson::Dslash applies only Dslash
DiracWilson::DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, *tmp2, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD_ASYMMETRIC) {
// FIXME: It would be nice if I could do something like: cudaColorSpinorField tmp3( in.param() );
// to save copying the data from 'in'
bool reset = newTmp(&tmp2, in);
// DiracCloverPC::Dslash applies A^{-1}Dslash
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
Clover(*tmp2, in, QUDA_ODD_PARITY);
// DiracWilson::Dslash applies only Dslash
DiracWilson::DslashXpay(out, *tmp1, QUDA_ODD_PARITY, *tmp2, kappa2);
} else if (!dagger) { // symmetric preconditioning
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
Dslash(*tmp1, in, QUDA_ODD_PARITY);
DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("Invalid matpcType");
}
} else { // symmetric preconditioning, dagger
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
CloverInv(out, in, QUDA_EVEN_PARITY);
Dslash(*tmp1, out, QUDA_ODD_PARITY);
DiracWilson::DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
CloverInv(out, in, QUDA_ODD_PARITY);
Dslash(*tmp1, out, QUDA_EVEN_PARITY);
DiracWilson::DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("MatPCType %d not valid for DiracCloverPC", matpcType);
}
}
deleteTmp(&tmp1, reset);
}
void DiracDomainWall::MdagM(ColorSpinorField &out, const ColorSpinorField &in) const
{
checkFullSpinor(out, in);
bool reset = newTmp(&tmp1, in);
M(*tmp1, in);
Mdag(out, *tmp1);
deleteTmp(&tmp1, reset);
}
void DiracWilson::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
checkFullSpinor(out, in);
bool reset = newTmp(&tmp1, in);
checkFullSpinor(*tmp1, in);
M(*tmp1, in);
Mdag(out, *tmp1);
deleteTmp(&tmp1, reset);
}
void DiracWilsonPC::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
#ifdef MULTI_GPU
bool reset = newTmp(&tmp2, in);
M(*tmp2, in);
Mdag(out, *tmp2);
deleteTmp(&tmp2, reset);
#else
M(out, in);
Mdag(out, out);
#endif
}
// Apply the even-odd preconditioned clover-improved Dirac operator
void DiracCloverPC::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
double kappa2 = -kappa*kappa;
// FIXME: For asymmetric, a "DslashCxpay" kernel would improve performance.
bool reset = newTmp(&tmp1, in);
if (matpcType == QUDA_MATPC_EVEN_EVEN_ASYMMETRIC) {
Dslash(*tmp1, in, QUDA_ODD_PARITY);
Clover(out, in, QUDA_EVEN_PARITY);
#ifdef MULTI_GPU // not safe to alias because of partial updates
cudaColorSpinorField tmp3(in);
#else // safe since out is not read after writing
cudaColorSpinorField &tmp3 = out;
#endif
DiracWilson::DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, tmp3, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD_ASYMMETRIC) {
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
Clover(out, in, QUDA_ODD_PARITY);
#ifdef MULTI_GPU // not safe to alias because of partial updates
cudaColorSpinorField tmp3(in);
#else // safe since out is not read after writing
cudaColorSpinorField &tmp3 = out;
#endif
DiracWilson::DslashXpay(out, *tmp1, QUDA_ODD_PARITY, tmp3, kappa2);
} else if (!dagger) { // symmetric preconditioning
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
Dslash(*tmp1, in, QUDA_ODD_PARITY);
DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("Invalid matpcType");
}
} else { // symmetric preconditioning, dagger
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
CloverInv(out, in, QUDA_EVEN_PARITY);
Dslash(*tmp1, out, QUDA_ODD_PARITY);
DiracWilson::DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
CloverInv(out, in, QUDA_ODD_PARITY);
Dslash(*tmp1, out, QUDA_EVEN_PARITY);
DiracWilson::DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("MatPCType %d not valid for DiracCloverPC", matpcType);
}
}
deleteTmp(&tmp1, reset);
}
void DiracStaggered::MdagM(ColorSpinorField &out, const ColorSpinorField &in) const
{
bool reset = newTmp(&tmp1, in);
//even
Dslash(tmp1->Even(), in.Even(), QUDA_ODD_PARITY);
DslashXpay(out.Even(), tmp1->Even(), QUDA_EVEN_PARITY, in.Even(), 4*mass*mass);
//odd
Dslash(tmp1->Even(), in.Odd(), QUDA_EVEN_PARITY);
DslashXpay(out.Odd(), tmp1->Even(), QUDA_ODD_PARITY, in.Odd(), 4*mass*mass);
deleteTmp(&tmp1, reset);
}
// Full staggered operator
void DiracStaggered::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
if (!initDslash){
initDslashConstants(*fatGauge, in.Stride());
initStaggeredConstants(*fatGauge, *longGauge);
}
bool reset = newTmp(&tmp1, in.Even());
DslashXpay(out.Even(), in.Odd(), QUDA_EVEN_PARITY, *tmp1, 2*mass);
DslashXpay(out.Odd(), in.Even(), QUDA_ODD_PARITY, *tmp1, 2*mass);
deleteTmp(&tmp1, reset);
}
void DiracCloverPC::prepare(cudaColorSpinorField* &src, cudaColorSpinorField* &sol,
cudaColorSpinorField &x, cudaColorSpinorField &b,
const QudaSolutionType solType) const
{
// we desire solution to preconditioned system
if (solType == QUDA_MATPC_SOLUTION || solType == QUDA_MATPCDAG_MATPC_SOLUTION) {
src = &b;
sol = &x;
return;
}
bool reset = newTmp(&tmp1, b.Even());
// we desire solution to full system
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
// src = A_ee^-1 (b_e + k D_eo A_oo^-1 b_o)
src = &(x.Odd());
CloverInv(*src, b.Odd(), QUDA_ODD_PARITY);
DiracWilson::DslashXpay(*tmp1, *src, QUDA_EVEN_PARITY, b.Even(), kappa);
CloverInv(*src, *tmp1, QUDA_EVEN_PARITY);
sol = &(x.Even());
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
// src = A_oo^-1 (b_o + k D_oe A_ee^-1 b_e)
src = &(x.Even());
CloverInv(*src, b.Even(), QUDA_EVEN_PARITY);
DiracWilson::DslashXpay(*tmp1, *src, QUDA_ODD_PARITY, b.Odd(), kappa);
CloverInv(*src, *tmp1, QUDA_ODD_PARITY);
sol = &(x.Odd());
} else if (matpcType == QUDA_MATPC_EVEN_EVEN_ASYMMETRIC) {
// src = b_e + k D_eo A_oo^-1 b_o
src = &(x.Odd());
CloverInv(*tmp1, b.Odd(), QUDA_ODD_PARITY); // safe even when *tmp1 = b.odd
DiracWilson::DslashXpay(*src, *tmp1, QUDA_EVEN_PARITY, b.Even(), kappa);
sol = &(x.Even());
} else if (matpcType == QUDA_MATPC_ODD_ODD_ASYMMETRIC) {
// src = b_o + k D_oe A_ee^-1 b_e
src = &(x.Even());
CloverInv(*tmp1, b.Even(), QUDA_EVEN_PARITY); // safe even when *tmp1 = b.even
DiracWilson::DslashXpay(*src, *tmp1, QUDA_ODD_PARITY, b.Odd(), kappa);
sol = &(x.Odd());
} else {
errorQuda("MatPCType %d not valid for DiracCloverPC", matpcType);
}
// here we use final solution to store parity solution and parity source
// b is now up for grabs if we want
deleteTmp(&tmp1, reset);
}
// FIXME: create kernel to eliminate tmp
void DiracClover::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
checkFullSpinor(out, in);
cudaColorSpinorField *tmp=0; // this hack allows for tmp2 to be full or parity field
if (tmp2) {
if (tmp2->SiteSubset() == QUDA_FULL_SITE_SUBSET) tmp = &(tmp2->Even());
else tmp = tmp2;
}
bool reset = newTmp(&tmp, in.Even());
Clover(*tmp, in.Odd(), QUDA_ODD_PARITY);
DslashXpay(out.Odd(), in.Even(), QUDA_ODD_PARITY, *tmp, -kappa);
Clover(*tmp, in.Even(), QUDA_EVEN_PARITY);
DslashXpay(out.Even(), in.Odd(), QUDA_EVEN_PARITY, *tmp, -kappa);
deleteTmp(&tmp, reset);
}
void DiracWilsonPC::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
double kappa2 = -kappa*kappa;
bool reset = newTmp(&tmp1, in);
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
Dslash(*tmp1, in, QUDA_ODD_PARITY);
DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("MatPCType %d not valid for DiracWilsonPC", matpcType);
}
deleteTmp(&tmp1, reset);
}
// Apply the even-odd preconditioned clover-improved Dirac operator
void DiracDomainWallPC::M(ColorSpinorField &out, const ColorSpinorField &in) const
{
if ( in.Ndim() != 5 || out.Ndim() != 5) errorQuda("Wrong number of dimensions\n");
double kappa2 = -kappa5*kappa5;
bool reset = newTmp(&tmp1, in);
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
Dslash(*tmp1, in, QUDA_ODD_PARITY);
DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("MatPCType %d not valid for DiracDomainWallPC", matpcType);
}
deleteTmp(&tmp1, reset);
}
void DiracStaggeredPC::MdagM(ColorSpinorField &out, const ColorSpinorField &in) const
{
bool reset = newTmp(&tmp1, in);
QudaParity parity = QUDA_INVALID_PARITY;
QudaParity other_parity = QUDA_INVALID_PARITY;
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
parity = QUDA_EVEN_PARITY;
other_parity = QUDA_ODD_PARITY;
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
parity = QUDA_ODD_PARITY;
other_parity = QUDA_EVEN_PARITY;
} else {
errorQuda("Invalid matpcType(%d) in function\n", matpcType);
}
Dslash(*tmp1, in, other_parity);
DslashXpay(out, *tmp1, parity, in, 4*mass*mass);
deleteTmp(&tmp1, reset);
}
// Apply the even-odd preconditioned clover-improved Dirac operator
void DiracCloverPC::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
double kappa2 = -kappa*kappa;
bool reset1 = newTmp(&tmp1, in);
if (matpcType == QUDA_MATPC_EVEN_EVEN_ASYMMETRIC) {
// DiracCloverPC::Dslash applies A^{-1}Dslash
Dslash(*tmp1, in, QUDA_ODD_PARITY);
// DiracClover::DslashXpay applies (A - kappa^2 D)
DiracClover::DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD_ASYMMETRIC) {
// DiracCloverPC::Dslash applies A^{-1}Dslash
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
// DiracClover::DslashXpay applies (A - kappa^2 D)
DiracClover::DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else if (!dagger) { // symmetric preconditioning
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
Dslash(*tmp1, in, QUDA_ODD_PARITY);
DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
Dslash(*tmp1, in, QUDA_EVEN_PARITY);
DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("Invalid matpcType");
}
} else { // symmetric preconditioning, dagger
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
CloverInv(out, in, QUDA_EVEN_PARITY);
Dslash(*tmp1, out, QUDA_ODD_PARITY);
DiracWilson::DslashXpay(out, *tmp1, QUDA_EVEN_PARITY, in, kappa2);
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
CloverInv(out, in, QUDA_ODD_PARITY);
Dslash(*tmp1, out, QUDA_EVEN_PARITY);
DiracWilson::DslashXpay(out, *tmp1, QUDA_ODD_PARITY, in, kappa2);
} else {
errorQuda("MatPCType %d not valid for DiracCloverPC", matpcType);
}
}
deleteTmp(&tmp1, reset1);
}