void DiracStaggered::checkParitySpinor(const cudaColorSpinorField &in, const cudaColorSpinorField &out) const
{
if (in.Precision() != out.Precision()) {
errorQuda("Input and output spinor precisions don't match in dslash_quda");
}
if (in.Stride() != out.Stride()) {
errorQuda("Input %d and output %d spinor strides don't match in dslash_quda", in.Stride(), out.Stride());
}
if (in.SiteSubset() != QUDA_PARITY_SITE_SUBSET || out.SiteSubset() != QUDA_PARITY_SITE_SUBSET) {
errorQuda("ColorSpinorFields are not single parity, in = %d, out = %d",
in.SiteSubset(), out.SiteSubset());
}
}
void DiracStaggeredPC::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
if (!initDslash){
initDslashConstants(*fatGauge, in.Stride());
initStaggeredConstants(*fatGauge, *longGauge);
}
bool reset = newTmp(&tmp1, in);
QudaParity parity = QUDA_INVALID_PARITY;
QudaParity other_parity = QUDA_INVALID_PARITY;
if (matpcType == QUDA_MATPC_EVEN_EVEN) {
parity = QUDA_EVEN_PARITY;
other_parity = QUDA_ODD_PARITY;
} else if (matpcType == QUDA_MATPC_ODD_ODD) {
parity = QUDA_ODD_PARITY;
other_parity = QUDA_EVEN_PARITY;
} else {
errorQuda("Invalid matpcType(%d) in function\n", matpcType);
}
Dslash(*tmp1, in, other_parity);
DslashXpay(out, *tmp1, parity, in, 4*mass*mass);
deleteTmp(&tmp1, reset);
}
void DiracStaggered::MdagM(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
if (!initDslash){
initDslashConstants(*fatGauge, in.Stride());
initStaggeredConstants(*fatGauge, *longGauge);
}
bool reset = newTmp(&tmp1, in);
cudaColorSpinorField* mytmp = dynamic_cast<cudaColorSpinorField*>(&(tmp1->Even()));
cudaColorSpinorField* ineven = dynamic_cast<cudaColorSpinorField*>(&(in.Even()));
cudaColorSpinorField* inodd = dynamic_cast<cudaColorSpinorField*>(&(in.Odd()));
cudaColorSpinorField* outeven = dynamic_cast<cudaColorSpinorField*>(&(out.Even()));
cudaColorSpinorField* outodd = dynamic_cast<cudaColorSpinorField*>(&(out.Odd()));
//even
Dslash(*mytmp, *ineven, QUDA_ODD_PARITY);
DslashXpay(*outeven, *mytmp, QUDA_EVEN_PARITY, *ineven, 4*mass*mass);
//odd
Dslash(*mytmp, *inodd, QUDA_EVEN_PARITY);
DslashXpay(*outodd, *mytmp, QUDA_ODD_PARITY, *inodd, 4*mass*mass);
deleteTmp(&tmp1, reset);
}
void DiracStaggered::checkParitySpinor(const cudaColorSpinorField &in, const cudaColorSpinorField &out) const
{
if (in.Precision() != out.Precision()) {
errorQuda("Input and output spinor precisions don't match in dslash_quda");
}
if (in.Stride() != out.Stride()) {
errorQuda("Input %d and output %d spinor strides don't match in dslash_quda", in.Stride(), out.Stride());
}
if (in.SiteSubset() != QUDA_PARITY_SITE_SUBSET || out.SiteSubset() != QUDA_PARITY_SITE_SUBSET) {
errorQuda("ColorSpinorFields are not single parity, in = %d, out = %d",
in.SiteSubset(), out.SiteSubset());
}
if ((out.Volume() != 2*fatGauge->VolumeCB() && out.SiteSubset() == QUDA_FULL_SITE_SUBSET) ||
(out.Volume() != fatGauge->VolumeCB() && out.SiteSubset() == QUDA_PARITY_SITE_SUBSET) ) {
errorQuda("Spinor volume %d doesn't match gauge volume %d", out.Volume(), fatGauge->VolumeCB());
}
}
// Public method to apply the clover term only
void DiracClover::Clover(cudaColorSpinorField &out, const cudaColorSpinorField &in, const QudaParity parity) const
{
if (!initDslash) initDslashConstants(gauge, in.Stride());
if (!initClover) initCloverConstants(clover.Stride());
checkParitySpinor(in, out, clover);
// regular clover term
FullClover cs;
cs.even = clover.even; cs.odd = clover.odd; cs.evenNorm = clover.evenNorm; cs.oddNorm = clover.oddNorm;
cs.precision = clover.precision; cs.bytes = clover.bytes, cs.norm_bytes = clover.norm_bytes;
cloverCuda(&out, gauge, cs, &in, parity);
flops += 504*in.Volume();
}
// Full staggered operator
void DiracStaggered::M(cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
if (!initDslash){
initDslashConstants(*fatGauge, in.Stride());
initStaggeredConstants(*fatGauge, *longGauge);
}
bool reset = newTmp(&tmp1, in.Even());
DslashXpay(out.Even(), in.Odd(), QUDA_EVEN_PARITY, *tmp1, 2*mass);
DslashXpay(out.Odd(), in.Even(), QUDA_ODD_PARITY, *tmp1, 2*mass);
deleteTmp(&tmp1, reset);
}
void DiracStaggered::Dslash(cudaColorSpinorField &out, const cudaColorSpinorField &in,
const QudaParity parity) const
{
if (!initDslash) {
initDslashConstants(*fatGauge, in.Stride());
initStaggeredConstants(*fatGauge, *longGauge);
}
checkParitySpinor(in, out);
setFace(face); // FIXME: temporary hack maintain C linkage for dslashCuda
staggeredDslashCuda(&out, *fatGauge, *longGauge, &in, parity, dagger, 0, 0, commDim);
flops += 1146*in.Volume();
}
void Dirac::checkParitySpinor(const cudaColorSpinorField &out, const cudaColorSpinorField &in) const
{
if (in.GammaBasis() != QUDA_UKQCD_GAMMA_BASIS ||
out.GammaBasis() != QUDA_UKQCD_GAMMA_BASIS) {
errorQuda("CUDA Dirac operator requires UKQCD basis, out = %d, in = %d",
out.GammaBasis(), in.GammaBasis());
}
if (in.Precision() != out.Precision()) {
errorQuda("Input precision %d and output spinor precision %d don't match in dslash_quda",
in.Precision(), out.Precision());
}
if (in.Stride() != out.Stride()) {
errorQuda("Input %d and output %d spinor strides don't match in dslash_quda",
in.Stride(), out.Stride());
}
if (in.SiteSubset() != QUDA_PARITY_SITE_SUBSET || out.SiteSubset() != QUDA_PARITY_SITE_SUBSET) {
errorQuda("ColorSpinorFields are not single parity: in = %d, out = %d",
in.SiteSubset(), out.SiteSubset());
}
if (out.Ndim() != 5) {
if ((out.Volume() != gauge.Volume() && out.SiteSubset() == QUDA_FULL_SITE_SUBSET) ||
(out.Volume() != gauge.VolumeCB() && out.SiteSubset() == QUDA_PARITY_SITE_SUBSET) ) {
errorQuda("Spinor volume %d doesn't match gauge volume %d", out.Volume(), gauge.VolumeCB());
}
} else {
// Domain wall fermions, compare 4d volumes not 5d
if ((out.Volume()/out.X(4) != gauge.Volume() && out.SiteSubset() == QUDA_FULL_SITE_SUBSET) ||
(out.Volume()/out.X(4) != gauge.VolumeCB() && out.SiteSubset() == QUDA_PARITY_SITE_SUBSET) ) {
errorQuda("Spinor volume %d doesn't match gauge volume %d", out.Volume(), gauge.VolumeCB());
}
}
}
// Public method
void DiracCloverPC::CloverInv(cudaColorSpinorField &out, const cudaColorSpinorField &in,
const QudaParity parity) const
{
if (!initDslash) initDslashConstants(gauge, in.Stride());
if (!initClover) initCloverConstants(clover.Stride());
checkParitySpinor(in, out, clover);
// needs to be cloverinv
FullClover cs;
cs.even = clover.evenInv; cs.odd = clover.oddInv; cs.evenNorm = clover.evenInvNorm; cs.oddNorm = clover.oddInvNorm;
cs.precision = clover.precision; cs.bytes = clover.bytes, cs.norm_bytes = clover.norm_bytes;
cloverCuda(&out, gauge, cs, &in, parity, tuneClover);
flops += 504*in.Volume();
}
void DiracDomainWall::Dslash(cudaColorSpinorField &out, const cudaColorSpinorField &in,
const QudaParity parity) const
{
if ( in.Ndim() != 5 || out.Ndim() != 5) errorQuda("Wrong number of dimensions\n");
if (!initDslash) initDslashConstants(gauge, in.Stride());
if (!initDomainWall) initDomainWallConstants(in.X(4));
checkParitySpinor(in, out);
checkSpinorAlias(in, out);
domainWallDslashCuda(&out, gauge, &in, parity, dagger, 0, mass, 0, tuneDslash);
long long Ls = in.X(4);
long long bulk = (Ls-2)*(in.Volume()/Ls);
long long wall = 2*in.Volume()/Ls;
flops += 1320LL*(long long)in.Volume() + 96LL*bulk + 120LL*wall;
}
// apply hopping term, then clover: (A_ee^-1 D_eo) or (A_oo^-1 D_oe),
// and likewise for dagger: (A_ee^-1 D^dagger_eo) or (A_oo^-1 D^dagger_oe)
// NOTE - this isn't Dslash dagger since order should be reversed!
void DiracCloverPC::Dslash(cudaColorSpinorField &out, const cudaColorSpinorField &in,
const QudaParity parity) const
{
if (!initDslash) initDslashConstants(gauge, in.Stride());
if (!initClover) initCloverConstants(clover.Stride());
checkParitySpinor(in, out, clover);
checkSpinorAlias(in, out);
setFace(face); // FIXME: temporary hack maintain C linkage for dslashCuda
FullClover cs;
cs.even = clover.evenInv; cs.odd = clover.oddInv; cs.evenNorm = clover.evenInvNorm; cs.oddNorm = clover.oddInvNorm;
cs.precision = clover.precision; cs.bytes = clover.bytes, cs.norm_bytes = clover.norm_bytes;
cloverDslashCuda(&out, gauge, cs, &in, parity, dagger, 0, 0.0, commDim);
flops += (1320+504)*in.Volume();
}