void d_plaquette(double *ss_plaq,double *st_plaq) {
register int i,dir1,dir2;
register site *s;
register su3_matrix *m1,*m4;
double ss_sum,st_sum;
msg_tag *mtag0,*mtag1;
ss_sum = st_sum = 0.0;
for(dir1=YUP;dir1<=TUP;dir1++){
for(dir2=XUP;dir2<dir1;dir2++){
mtag0 = start_gather_site( F_OFFSET(link[dir2]), sizeof(su3_matrix),
dir1, EVENANDODD, gen_pt[0] );
mtag1 = start_gather_site( F_OFFSET(link[dir1]), sizeof(su3_matrix),
dir2, EVENANDODD, gen_pt[1] );
FORALLSITES(i,s){
m1 = &(s->link[dir1]);
m4 = &(s->link[dir2]);
mult_su3_an(m4,m1,&(s->tempmat1));
}
wait_gather(mtag0);
FORALLSITES(i,s){
#ifdef SCHROED_FUN
if(dir1==TUP ){
if(s->t==(nt-1)){
mult_su3_nn( &(s->tempmat1),
&(s->boundary[dir2]), &(s->staple));
}
else{
mult_su3_nn( &(s->tempmat1),
(su3_matrix *)(gen_pt[0][i]), &(s->staple));
}
}
else if(s->t > 0){
mult_su3_nn( &(s->tempmat1), (su3_matrix *)(gen_pt[0][i]),
&(s->staple));
}
#else
mult_su3_nn( &(s->tempmat1),(su3_matrix *)(gen_pt[0][i]),
&(s->staple) );
#endif
}
wait_gather(mtag1);
FORALLSITES(i,s){
if(dir1==TUP )st_sum += (double)
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&(s->staple) );
#ifdef SCHROED_FUN
else if(s->t > 0) ss_sum += (double)
#else
else ss_sum += (double)
#endif
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&(s->staple) );
}
cleanup_gather(mtag0);
cleanup_gather(mtag1);
}
}
// -----------------------------------------------------------------
void directional_staple(int dir1, int dir2, field_offset lnk1,
field_offset lnk2, su3_matrix *stp) {
register int i;
register site *s;
msg_tag *tag0, *tag1, *tag2;
su3_matrix tmat1, tmat2;
// Get blocked_link[dir2] from direction dir1
tag0 = start_gather_site(lnk2, sizeof(su3_matrix), dir1,
EVENANDODD, gen_pt[0]);
// Get blocked_link[dir1] from direction dir2
tag1 = start_gather_site(lnk1, sizeof(su3_matrix), dir2,
EVENANDODD, gen_pt[1]);
// Start working on the lower staple while we wait for the gathers
// The lower staple is prepared at x-dir2 and stored in tempmat1,
// then gathered to x
FORALLSITES(i, s)
mult_su3_an((su3_matrix*)F_PT(s,lnk2), (su3_matrix*)F_PT(s,lnk1),
tempmat1 + i);
wait_gather(tag0);
wait_gather(tag1);
// Finish lower staple
FORALLSITES(i, s) {
mult_su3_nn(tempmat1 + i, (su3_matrix *)gen_pt[0][i], &tmat1);
su3mat_copy(&tmat1, tempmat1 + i);
}
void fuz_prop(field_offset fprop, int r0)
{
register int i;
register site *s;
int dir, k;
msg_tag *tag0, *tag1;
/* Save unfuzzed propagator in xxx */
copy_latvec( fprop, F_OFFSET(xxx), EVENANDODD);
/* Give central value weight two */
scalar_mult_latvec( F_OFFSET(xxx), 2.0, fprop, EVENANDODD);
if (r0 > 0){
for(dir=XUP;dir<=ZUP;dir++){
/* Start gathering for 'backward' link-product */
tag0 = start_gather_site(F_OFFSET(xxx), sizeof(su3_vector),
dir, EVENANDODD, gen_pt[0]);
/* Start 'forward' link-product */
FORALLSITES(i,s) {
mult_adj_su3_mat_vec(&(s->link[dir]), &(s->xxx), &(s->ttt));
}
tag1 = start_gather_site(F_OFFSET(ttt), sizeof(su3_vector),
OPP_DIR(dir), EVENANDODD, gen_pt[1]);
for(k=1;k<r0;k++) {
wait_gather(tag0);
/*
copy_latvec( (field_offset)gen_pt[0], F_OFFSET(resid),
EVENANDODD);
*/
FORALLSITES(i,s) {
su3vec_copy((su3_vector *)gen_pt[0][i], &(s->resid));
}
FORALLSITES(i,s) {
mult_su3_mat_vec(&(s->link[dir]), &(s->resid), &(s->cg_p));
}
if(k==1) {
cleanup_gather(tag0);
tag0 = start_gather_site(F_OFFSET(cg_p), sizeof(su3_vector),
dir, EVENANDODD, gen_pt[0]);
}
else {
restart_gather_site(F_OFFSET(cg_p), sizeof(su3_vector),
dir, EVENANDODD, gen_pt[0], tag0);
}
wait_gather(tag1);
/*
copy_latvec( (field_offset)gen_pt[1], F_OFFSET(resid),
EVENANDODD);
*/
FORALLSITES(i,s) {
su3vec_copy((su3_vector *)gen_pt[1][i], &(s->resid));
}
/* update the momenta with the gauge force */
void gauge_force(Real eps) {
register int i,dir1,dir2;
register site *st;
msg_tag *tag0,*tag1,*tag2;
int start;
su3_matrix tmat1,tmat2;
register Real eb3;
/**double dtime,dclock();
dtime = -dclock();**/
eb3 = eps*beta/3.0;
/* Loop over directions, update mom[dir1] */
for(dir1=XUP; dir1<=TUP; dir1++){
/* Loop over other directions, computing force from plaquettes in
the dir1,dir2 plane */
start=1; /* indicates staple sum not initialized */
for(dir2=XUP;dir2<=TUP;dir2++)if(dir2 != dir1){
/* get link[dir2] from direction dir1 */
tag0 = start_gather_site( F_OFFSET(link[dir2]), sizeof(su3_matrix),
dir1, EVENANDODD, gen_pt[0] );
/* Start gather for the "upper staple" */
tag2 = start_gather_site( F_OFFSET(link[dir1]), sizeof(su3_matrix),
dir2, EVENANDODD, gen_pt[2] );
/* begin the computation "at the dir2DOWN point", we will
later gather the intermediate result "to the home point" */
wait_gather(tag0);
FORALLSITES(i,st){
mult_su3_an( &(st->link[dir2]), &(st->link[dir1]), &tmat1 );
mult_su3_nn( &tmat1, (su3_matrix *)gen_pt[0][i],
&(st->tempmat1) );
}
/* Gather this partial result "up to home site" */
tag1 = start_gather_site( F_OFFSET(tempmat1), sizeof(su3_matrix),
OPP_DIR(dir2), EVENANDODD, gen_pt[1] );
/* begin the computation of the "upper" staple. Note that
one of the links has already been gathered, since it
was used in computing the "lower" staple of the site
above us (in dir2) */
wait_gather(tag2);
if(start){ /* this is the first contribution to staple */
FORALLSITES(i,st){
mult_su3_nn( &(st->link[dir2]), (su3_matrix *)gen_pt[2][i],
&tmat1);
mult_su3_na( &tmat1, (su3_matrix *)gen_pt[0][i],
&(st->staple) );
}
start=0;
}
static void remap_data(int index, ft_data *ftd){
msg_tag *mtag;
char *temp;
int i;
double dtime = start_timing();
temp = (char *)malloc(sites_on_node*ftd->size);
if(temp==NULL){
printf("remap_data: No room\n");
terminate(1);
}
mtag = start_gather_field(ftd->data, ftd->size, index, EVENANDODD,
gen_pt[0]);
wait_gather(mtag);
/* First copy gathered data to temporary */
for(i = 0; i < sites_on_node; i++)
memcpy(temp + ftd->size*i, gen_pt[0][i], ftd->size);
cleanup_gather(mtag);
/* Then copy temp back to field */
memcpy((char *)ftd->data, temp, sites_on_node*ftd->size);
free(temp);
print_timing(dtime, "REMAP FFTW remap");
}
complex u1ploop(void)
{
int d[4];
Real sum;
complex plp;
Real *tempmat1;
int i,t;
site *s;
msg_tag *mtag;
tempmat1 = create_r_field();
sum = 0.0;
d[XUP]=d[YUP]=d[ZUP]=0;
mtag = declare_strided_gather(u1_A+TUP, 4*sizeof(Real), sizeof(Real),
TUP, EVEN, gen_pt[0]);
prepare_gather(mtag);
do_gather(mtag);
// mtag=start_gather_site(F_OFFSET(u1link[TUP]),sizeof(complex),
// TUP,EVEN,gen_pt[0]);
wait_gather(mtag);
FOREVENSITES(i,s){
// l1=&(s->u1link[TUP]);
// l2 = (complex *)gen_pt[0][i];
tempmat1[i] = u1_A[4*i+TUP] + *((Real *)gen_pt[0][i]);
}
void
dslash_fn_dir(su3_vector *src, su3_vector *dest, int parity,
fn_links_t *fn, int dir, int fb,
Real wtfat, Real wtlong)
{
register int i ;
site *s;
msg_tag *tag[2];
su3_matrix *fat = get_fatlinks(fn);
su3_matrix *lng = get_lnglinks(fn);
su3_vector tmp;
int do_long = (lng != NULL) && (wtlong != 0.);
char myname[] = "fn_shift";
if(fat == NULL) {
printf("%s(%d): fat or lng member is null\n", myname, this_node);
terminate(1);
}
if(fb > 0){
/* Shift from forward direction */
tag[0] = start_gather_field( src, sizeof(su3_vector), dir,
parity, gen_pt[0] );
if(do_long)
tag[1] = start_gather_field( src, sizeof(su3_vector), DIR3(dir),
parity, gen_pt[1] );
wait_gather(tag[0]);
if(do_long)
wait_gather(tag[1]);
FORSOMEPARITYDOMAIN(i,s,parity)
{
mult_su3_mat_vec( fat+4*i+dir, (su3_vector *)gen_pt[0][i], &tmp );
scalar_mult_add_su3_vector( dest+i, &tmp, wtfat, dest+i ) ;
if(do_long){
mult_su3_mat_vec( lng+4*i+dir, (su3_vector *)gen_pt[1][i], &tmp );
scalar_mult_add_su3_vector( dest+i, &tmp, wtlong, dest+i ) ;
}
} END_LOOP
void path(int *dir,int *sign,int length)
{
register int i;
register site *s;
msg_tag *mtag0, *mtag1;
int j;
/* j=0 */
if(sign[0]>0) {
mtag0 = start_gather_site( F_OFFSET(link[dir[0]]), sizeof(su3_matrix),
OPP_DIR(dir[0]), EVENANDODD, gen_pt[0] );
wait_gather(mtag0);
FORALLSITES(i,s){
su3mat_copy((su3_matrix *)(gen_pt[0][i]),&(s->tempmat1) );
}
void static_prop()
{
register int i;
register site *st;
msg_tag *tag;
int tloop ;
int nthalf = nt/2 ;
/*************---------**********-------------************/
/* Initialise the gauge part of the propagator ***/
setup_static_prop() ;
/*
* Calculate the static propagator for positive time
*
* W(t+1) = W(t) U_4(t)
*/
for(tloop=1 ; tloop <= nthalf ; ++tloop)
{
/* The smear_w_line[0] object is used as work space ***/
FORALLSITES(i,st)
{
mult_su3_nn(&(st->w_line), &(st->link[TUP]), &(st->smear_w_line[0]));
}
/* Pull the w(t)*u(t) from the previous time slice ***/
tag=start_gather_site( F_OFFSET(smear_w_line[0]), sizeof(su3_matrix),
TDOWN, EVENANDODD, gen_pt[0] );
wait_gather(tag);
FORALLSITES(i,st)
{
if( st-> t == tloop )
su3mat_copy((su3_matrix *) gen_pt[0][i], &(st->w_line));
}
cleanup_gather(tag);
} /* end the loop over time slice ***/
void gauge_trans(field_offset G)
{
register int i,mu;
site *s;
su3_matrix tmp;
msg_tag *tag[4];
FORALLUPDIR(mu)
tag[mu] = start_gather_site(G,sizeof(su3_matrix),mu,EVENANDODD,
gen_pt[mu]);
FORALLUPDIR(mu) {
wait_gather(tag[mu]);
FORALLSITES(i,s) {
mult_su3_an((su3_matrix *)F_PT(s,G), &(s->link[mu]), &tmp);
mult_su3_nn(&tmp, (su3_matrix *)gen_pt[mu][i],
&(s->link[mu]));
}
cleanup_gather(tag[mu]);
}
int nl_spectrum( Real vmass, field_offset temp1, field_offset temp2 ) {
/* return the C.G. iteration number */
double *piprop,*pi2prop,*rhoprop,*rho2prop,*barprop;
double *nlpiprop,*nlpi2prop,*ckpiprop,*ckpi2prop;
double *delprop,*ckbarprop;
Real vmass_x2;
site* s;
register complex cc;
Real finalrsq;
register int i,x,y,z,t,icol,cgn;
register int t_source,t_off;
int dir,isrc;
msg_tag *mtag[16];
piprop = (double *)malloc( nt*sizeof(double) );
pi2prop = (double *)malloc( nt*sizeof(double) );
rhoprop = (double *)malloc( nt*sizeof(double) );
rho2prop = (double *)malloc( nt*sizeof(double) );
barprop = (double *)malloc( nt*sizeof(double) );
nlpiprop = (double *)malloc( nt*sizeof(double) );
nlpi2prop = (double *)malloc( nt*sizeof(double) );
ckpiprop = (double *)malloc( nt*sizeof(double) );
ckpi2prop = (double *)malloc( nt*sizeof(double) );
delprop = (double *)malloc( nt*sizeof(double) );
ckbarprop = (double *)malloc( nt*sizeof(double) );
for( t=0; t<nt; t++ ){
piprop[t]=0.0; pi2prop[t]=0.0; rhoprop[t]=0.0; rho2prop[t]=0.0;
nlpiprop[t]=0.0; nlpi2prop[t]=0.0;
ckpiprop[t]=0.0; ckpi2prop[t]=0.0;
barprop[t]=0.0; delprop[t]=0.0; ckbarprop[t]=0.0;
}
vmass_x2 = 2.*vmass;
cgn=0;
/* Fix TUP Coulomb gauge - gauge links only*/
rephase( OFF );
gaugefix(TUP,(Real)1.8,500,(Real)GAUGE_FIX_TOL);
rephase( ON );
#ifdef FN
invalidate_all_ferm_links(&fn_links);
#endif
/* Unlike spectrum.c, here we calculate only with wall sources */
for(t_source=source_start, isrc=0; t_source<2*nt && isrc < n_sources;
++isrc, t_source += source_inc ) {
/* Only work for even source slices */
if( t_source%2 != 0 ){
printf("DUMMY: Use even time slices for nl_spectrum()\n");
terminate(0);
}
/* Compute propagator from even wall sites */
/* Sources are normalized to 1/8 to make them comparable to */
/* propagators from a wall with ones on the cube origin. */
/* Put result in propmat */
for(icol=0; icol<3; icol++) {
/* initialize temp1 and temp2 */
clear_latvec( temp1, EVEN);
clear_latvec( temp2, EVEN);
for(x=0;x<nx;x++)for(y=0;y<ny;y++)for(z=0;z<nz;z++) {
if((x+y+z) % 2 == 0) {
if( node_number(x,y,z,t_source) != mynode() )continue;
i=node_index(x,y,z,t_source);
((su3_vector *)(F_PT(&lattice[i],temp1)))->c[icol].real =
-0.25;
}
}
/* do a C.G. */
load_ferm_links(&fn_links);
cgn += congrad(niter,rsqprop,EVEN,&finalrsq, &fn_links);
/* Multiply by -Madjoint */
dslash_site( temp2, temp2, ODD, &fn_links);
scalar_mult_latvec( temp2, -vmass_x2, temp2, EVEN);
/* fill the hadron matrix */
copy_latvec( temp2, F_OFFSET(propmat[icol]), EVENANDODD);
} /* end loop on icol */
/* Compute propagator from odd wall sites */
/* Put result in propmat2 */
for(icol=0; icol<3; icol++) {
/* initialize temp1 and temp2 */
clear_latvec( temp1, ODD);
clear_latvec( temp2, ODD);
for(x=0;x<nx;x++)for(y=0;y<ny;y++)for(z=0;z<nz;z++) {
if((x+y+z) % 2 == 1) {
if( node_number(x,y,z,t_source) != mynode() )continue;
i=node_index(x,y,z,t_source);
((su3_vector *)(F_PT(&lattice[i],temp1)))->c[icol].real =
-0.25;
}
}
/* do a C.G. */
load_ferm_links(&fn_links);
cgn += congrad(niter,rsqprop,ODD,&finalrsq,&fn_links);
/* Multiply by -Madjoint */
dslash_site( temp2, temp2, EVEN, &fn_links);
scalar_mult_latvec( temp2, -vmass_x2, temp2, ODD);
/* fill the hadron matrix */
copy_latvec( temp2, F_OFFSET(propmat2[icol]), EVENANDODD);
} /* end loop on icol */
/* cgn now gives the sum for both inversions */
/* measure the meson propagator for the E wall source */
for(t=0; t<nt; t++) {
/* define the time value offset t from t_source */
t_off = (t+t_source)%nt;
for(x=0;x<nx;x++)for(y=0;y<ny;y++)for(z=0;z<nz;z++)
for(icol=0;icol<3;icol++) {
if( node_number(x,y,z,t_off) != mynode() )continue;
i=node_index(x,y,z,t_off);
cc = su3_dot( &lattice[i].propmat[icol],
&lattice[i].propmat[icol] );
piprop[t] += cc.real;
/* (rhoprop and rho2prop are not generated by this source) */
if( (x+y+z)%2==0)pi2prop[t] += cc.real;
else pi2prop[t] -= cc.real;
}
} /* nt-loop */
/* measure the baryon propagator for the E wall source */
for(t=0; t<nt; t++) {
/* define the time value offset t from t_source */
t_off = (t+t_source)%nt;
for(x=0;x<nx;x+=2)for(y=0;y<ny;y+=2)for(z=0;z<nz;z+=2) {
if( node_number(x,y,z,t_off) != mynode() )continue;
i=node_index(x,y,z,t_off);
cc = det_su3( (su3_matrix *)(lattice[i].propmat) );
barprop[t] += cc.real;
}
/* must get sign right. This looks to see if we have
wrapped around the lattice. "t" is the distance
from the source to the measurement, so we are
trying to find out if t_source+t is greater than
or equal to nt. */
if( (((t+t_source)/nt-t_source/nt)%2) == 1 )barprop[t] *= -1.0;
/* change sign because antiperiodic b.c. sink point
should really be in a copy of the lattice */
} /* nt-loop */
/* Measure nonlocal (and some local for checking) propagators */
/* These propagators include the delta and some nonlocal mesons */
/* The method for the delta is described in M.F.L. Golterman and */
/* J. Smit, Nucl. Phys. B 255, 328 (1985) */
/* Equation (6.3) defines the sink operator for the delta */
/* The method for the mesons is described in M.F.L. Golterman */
/* Nucl. Phys. B 273, 663 (1986) */
/* The treatment of the source wall is described in Gupta, */
/* Guralnik, Kilcup, and Sharpe, (GGKS) NSF-ITP-90-172 (1990) */
/* To get the delta propagator, we take the "q" propagator */
/* matrices for each of the wall colors and antisymmetrize over */
/* wall color as well as s */
/* First construct the "q" and "o" propagators */
/* Put q = E + O in propmat and o = E - O in propmat2 */
FORALLSITES(i,s) {
for(icol=0; icol<3; icol++) {
add_su3_vector (&(s->propmat[icol]), &(s->propmat2[icol]),
(su3_vector *)(s->tempmat1.e[icol]) );
sub_su3_vector (&(s->propmat[icol]), &(s->propmat2[icol]),
&(s->propmat2[icol]) );
su3vec_copy( (su3_vector *)(s->tempmat1.e[icol]),
&(s->propmat[icol]) );
}
}
/* Next gather the propagators in preparation for calculating */
/* shifted propagators Dq and Do */
FORALLUPDIRBUT(TUP,dir) {
/* Start bringing "q" = propmat from forward sites */
mtag[dir] = start_gather_site(F_OFFSET(propmat[0]),
sizeof(su3_matrix), dir, EVENANDODD, gen_pt[dir]);
/* Start bringing "q" from backward neighbors */
mtag[dir+4] = start_gather_site(F_OFFSET(propmat[0]),
sizeof(su3_matrix), OPP_DIR(dir), EVENANDODD,
gen_pt[dir+4]);
wait_gather(mtag[dir]);
wait_gather(mtag[dir+4]);
/* Start bringing "o" = propmat2 from forward sites */
mtag[8+dir] = start_gather_site(F_OFFSET(propmat2[0]),
sizeof(su3_matrix), dir, EVENANDODD, gen_pt[8+dir]);
/* Start bringing "o" from backward neighbors */
mtag[8+dir+4] = start_gather_site(F_OFFSET(propmat2[0]),
sizeof(su3_matrix), OPP_DIR(dir), EVENANDODD,
gen_pt[8+dir+4]);
wait_gather(mtag[8+dir]);
wait_gather(mtag[8+dir+4]);
}
/* Calculate and dump delta propagator */
for(t=0; t<nt; t++) {
/* define the time value offset t from t_source */
t_off = (t+t_source)%nt;
/* Calculate contribution for each permutation of source color */
delta_prop (0,1,2, 1, t_off, &delprop[t]);
delta_prop (1,2,0, 1, t_off, &delprop[t]);
delta_prop (2,0,1, 1, t_off, &delprop[t]);
delta_prop (1,0,2,-1, t_off, &delprop[t]);
delta_prop (0,2,1,-1, t_off, &delprop[t]);
delta_prop (2,1,0,-1, t_off, &delprop[t]);
if( (((t+t_source)/nt-t_source/nt)%2) == 1 ) delprop[t] *= -1;
} /* nt-loop */
/* Calculate the "q" source nucleon as a check */
/* Calculate and dump nucleon check propagator */
for(t=0; t<nt; t++) {
/* define the time value offset t from t_source */
t_off = (t+t_source)%nt;
for(x=0;x<nx;x+=2)for(y=0;y<ny;y+=2)for(z=0;z<nz;z+=2) {
if( node_number(x,y,z,t_off) != mynode() )continue;
i=node_index(x,y,z,t_off);
/* The q propagator is in propmat */
cc = det_su3( (su3_matrix *)(lattice[i].propmat) );
ckbarprop[t] += cc.real;
}
if( (((t+t_source)/nt-t_source/nt)%2) == 1 )ckbarprop[t]*= -1.0;
/* change sign because antiperiodic b.c. sink point
should really be in a copy of the lattice */
} /* nt-loop */
/* Calculate nonlocal meson propagators and local check */
for(t=0; t<nt; t++) {
/* Calculate two nonlocal pion propagators */
/* These are pi_1 and pi_1 tilde of Gupta et al */
/* Also calculate two local propagators as a check */
/* define the time value offset t from t_source */
t_off = (t+t_source)%nt;
nl_meson_prop(t_off,&nlpiprop[t],&nlpi2prop[t],&ckpiprop[t],
&ckpi2prop[t]);
} /* nt-loop */
/* Clean up gathers */
FORALLUPDIRBUT(TUP,dir) {
cleanup_gather(mtag[dir]);
cleanup_gather(mtag[dir+4]);
cleanup_gather(mtag[8+dir]);
cleanup_gather(mtag[8+dir+4]);
}
/* Special dslash for use by congrad. Uses restart_gather_field() when
possible. Next to last argument is an array of message tags, to be set
if this is the first use, otherwise reused. If start=1,use
start_gather_field, otherwise use restart_gather_field.
The calling program must clean up the gathers and temps! */
void dslash_fn_field_special(su3_vector *src, su3_vector *dest,
int parity, msg_tag **tag, int start,
fn_links_t *fn){
register int i;
register site *s;
register int dir,otherparity=0;
register su3_matrix *fat4;
su3_matrix *t_fatlink;
#ifndef NO_LONG_LINKS
register su3_matrix *long4;
su3_matrix *t_longlink;
#endif
/* allocate temporary work space only if not already allocated */
if(temp_not_allocated)
{
for( dir=XUP; dir<=TUP; dir++ ){
temp[dir] =(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
temp[dir+4]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
}
temp[8]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
temp_not_allocated = 0 ;
}
/* load fatlinks and longlinks */
if(fn == NULL){
printf("dslash_fn_field_special: invalid fn links!\n");
terminate(1);
}
#ifndef NO_LONG_LINKS
t_longlink = get_lnglinks(fn);
#endif
t_fatlink = get_fatlinks(fn);
switch(parity)
{
case EVEN: otherparity=ODD; break;
case ODD: otherparity=EVEN; break;
case EVENANDODD: otherparity=EVENANDODD; break;
}
/* Start gathers from positive directions */
/* And start the 3-step gather too */
for( dir=XUP; dir<=TUP; dir++ ){
if(start==1)
{
tag[dir] = start_gather_field( src, sizeof(su3_vector),
dir, parity,gen_pt[dir] );
#ifndef NO_LONG_LINKS
tag[DIR3(dir)] = start_gather_field(src, sizeof(su3_vector),
DIR3(dir),parity,
gen_pt[DIR3(dir)] );
#endif
}
else
{
restart_gather_field( src, sizeof(su3_vector),
dir, parity,gen_pt[dir], tag[dir]);
#ifndef NO_LONG_LINKS
restart_gather_field(src, sizeof(su3_vector), DIR3(dir), parity,
gen_pt[DIR3(dir)], tag[DIR3(dir)]);
#endif
}
}
/* Multiply by adjoint matrix at other sites */
/* Use fat link for single link transport */
FORSOMEPARITYDOMAIN_OMP( i, s, otherparity, private(fat4,long4) ){
//NOPRE if( i < loopend-FETCH_UP ){
//NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_V(&(src[i+FETCH_UP]));
//NOPRE prefetch_4MVVVV(
//NOPRE fat4,
//NOPRE &(temp[0][i+FETCH_UP]),
//NOPRE &(temp[1][i+FETCH_UP]),
//NOPRE &(temp[2][i+FETCH_UP]),
//NOPRE &(temp[3][i+FETCH_UP]) );
#ifndef NO_LONG_LINKS
//NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_4MVVVV(
//NOPRE long4,
//NOPRE &(temp[4][i+FETCH_UP]),
//NOPRE &(temp[5][i+FETCH_UP]),
//NOPRE &(temp[6][i+FETCH_UP]),
//NOPRE &(temp[7][i+FETCH_UP]) );
#endif
//NOPRE }
fat4 = &(t_fatlink[4*i]);
mult_adj_su3_mat_4vec( fat4, &(src[i]), &(temp[0][i]),
&(temp[1][i]), &(temp[2][i]), &(temp[3][i]) );
#ifndef NO_LONG_LINKS
/* multiply by 3-link matrices too */
long4 = &(t_longlink[4*i]);
mult_adj_su3_mat_4vec( long4, &(src[i]),&(temp[4][i]),
&(temp[5][i]), &(temp[6][i]), &(temp[7][i]) );
#endif
} END_LOOP_OMP
/* Start gathers from negative directions */
for( dir=XUP; dir <= TUP; dir++){
if (start==1) tag[OPP_DIR(dir)] = start_gather_field( temp[dir],
sizeof(su3_vector), OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)] );
else restart_gather_field( temp[dir], sizeof(su3_vector),
OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)], tag[OPP_DIR(dir)] );
}
/* Start 3-neighbour gathers from negative directions */
for( dir=X3UP; dir <= T3UP; dir++){
if (start==1) tag[OPP_3_DIR(dir)]=start_gather_field(
temp[INDEX_3RD(dir)+4], sizeof(su3_vector),
OPP_3_DIR( dir), parity, gen_pt[OPP_3_DIR(dir)] );
else restart_gather_field(temp[INDEX_3RD(dir)+4],
sizeof(su3_vector), OPP_3_DIR( dir),parity,
gen_pt[OPP_3_DIR(dir)], tag[OPP_3_DIR(dir)] );
}
/* Wait gathers from positive directions, multiply by matrix and
accumulate */
/* wait for the 3-neighbours from positive directions, multiply */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[dir]);
#ifndef NO_LONG_LINKS
wait_gather(tag[DIR3(dir)]);
#endif
}
FORSOMEPARITYDOMAIN_OMP(i,s,parity, private(fat4,long4) ){
//NOPRE if( i < loopend-FETCH_UP ){
//NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_4MVVVV(
//NOPRE fat4,
//NOPRE (su3_vector *)gen_pt[XUP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[YUP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[ZUP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[TUP][i+FETCH_UP] );
//NOPRE prefetch_VVVV(
//NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
#ifndef NO_LONG_LINKS
//NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_4MVVVV(
//NOPRE long4,
//NOPRE (su3_vector *)gen_pt[X3UP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
//NOPRE prefetch_VVVV(
//NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
#endif
//NOPRE }
fat4 = &(t_fatlink[4*i]);
mult_su3_mat_vec_sum_4dir( fat4,
(su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
(su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
&(dest[i]) );
#ifndef NO_LONG_LINKS
long4 = &(t_longlink[4*i]);
mult_su3_mat_vec_sum_4dir( long4,
(su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
(su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
&(temp[8][i]));
#endif
} END_LOOP_OMP
/* Wait gathers from negative directions, accumulate (negative) */
/* and the same for the negative 3-rd neighbours */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[OPP_DIR(dir)]);
}
for(dir=X3UP; dir<=T3UP; dir++){
wait_gather(tag[OPP_3_DIR(dir)]);
}
FORSOMEPARITYDOMAIN_OMP(i,s,parity, ){
//NOPRE if( i < loopend-FETCH_UP ){
//NOPRE prefetch_VVVVV(
//NOPRE &(dest[i+FETCH_UP]),
//NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
//NOPRE prefetch_VVVVV(
//NOPRE &(temp[8][i+FETCH_UP]),
//NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
//NOPRE }
sub_four_su3_vecs( &(dest[i]),
(su3_vector *)(gen_pt[XDOWN][i]),
(su3_vector *)(gen_pt[YDOWN][i]),
(su3_vector *)(gen_pt[ZDOWN][i]),
(su3_vector *)(gen_pt[TDOWN][i]) );
sub_four_su3_vecs( &(temp[8][i]),
(su3_vector *)(gen_pt[X3DOWN][i]),
(su3_vector *)(gen_pt[Y3DOWN][i]),
(su3_vector *)(gen_pt[Z3DOWN][i]),
(su3_vector *)(gen_pt[T3DOWN][i]) );
/* Now need to add these things together */
add_su3_vector(&(dest[i]), &(temp[8][i]),&(dest[i]));
} END_LOOP_OMP
}
void d_plaquette_field_hist(su3_matrix **U_field,
int Npowers, int *Nhist, double **hist,
double **hist_bounds,
double *ss_plaq, double *st_plaq) {
/* su3mat is scratch space of size su3_matrix */
su3_matrix *su3mat;
register int i,dir1,dir2;
register int ipower,ihist;
register site *s;
register su3_matrix *m1,*m4;
double *plaq_power, *step_hist;
su3_matrix mtmp;
double ss_sum,st_sum;
double rtrace, rtrace3;
msg_tag *mtag0,*mtag1;
ss_sum = st_sum = 0.0;
#ifdef HISQ_DUMP_PLAQ_INTO_FILE
FILE *fp;
char plaq_file_name[300];
#endif /* HISQ_DUMP_PLAQ_INTO_FILE */
su3mat = (su3_matrix *)malloc(sizeof(su3_matrix)*sites_on_node);
if(su3mat == NULL)
{
printf("plaquette: can't malloc su3mat\n");
fflush(stdout); terminate(1);
}
/* zero out the histogram */
for(ipower=0;ipower<Npowers;ipower++) {
for(ihist=0;ihist<Nhist[ipower];ihist++) {
hist[ipower][ihist]=0.0;
}
}
/* array with powers of (3-plaquette) */
plaq_power=(double*)malloc(sizeof(double)*Npowers);
/* array with step sizes */
step_hist=(double*)malloc(sizeof(double)*Npowers);
for(ipower=0;ipower<Npowers;ipower++) {
step_hist[ipower]=
(hist_bounds[ipower][1]-hist_bounds[ipower][0])/Nhist[ipower];
}
#ifdef HISQ_DUMP_PLAQ_INTO_FILE
sprintf( plaq_file_name, "plaq_W_node%04d.dat", this_node );
fp = fopen( plaq_file_name, "wt" );
#endif /* HISQ_DUMP_PLAQ_INTO_FILE */
for(dir1=YUP;dir1<=TUP;dir1++){
for(dir2=XUP;dir2<dir1;dir2++){
mtag0 = start_gather_field( U_field[dir2], sizeof(su3_matrix),
dir1, EVENANDODD, gen_pt[0] );
mtag1 = start_gather_field( U_field[dir1], sizeof(su3_matrix),
dir2, EVENANDODD, gen_pt[1] );
FORALLSITES(i,s){
m1 = &(U_field[dir1][i]);
m4 = &(U_field[dir2][i]);
mult_su3_an(m4,m1,&su3mat[i]);
}
wait_gather(mtag0);
wait_gather(mtag1);
FORALLSITES(i,s){
mult_su3_nn( &su3mat[i], (su3_matrix *)(gen_pt[0][i]),
&mtmp);
if(dir1==TUP ) {
rtrace = (double)
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&mtmp);
st_sum += rtrace;
}
else {
rtrace = (double)
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&mtmp);
ss_sum += rtrace;
}
// printf("Plaq i=%d, dir1=%d, dir2=%d: %f %f\n",
// i,dir1,dir2,rtrace_s,rtrace_t);
#ifdef HISQ_DUMP_PLAQ_INTO_FILE
fprintf( fp, "%18.12g\n", rtrace );
#endif /* HISQ_DUMP_PLAQ_INTO_FILE */
/* powers of (3-plaquette) */
rtrace3=3.0-rtrace;
plaq_power[0]=rtrace3;
for(ipower=1;ipower<Npowers;ipower++) {
plaq_power[ipower]=plaq_power[ipower-1]*rtrace3;
}
/* find histogram entry */
for(ipower=0;ipower<Npowers;ipower++) {
if( (plaq_power[ipower]>hist_bounds[ipower][0])
&& (plaq_power[ipower]<hist_bounds[ipower][1]) ) {
ihist=(int)(
(plaq_power[ipower]-hist_bounds[ipower][0])/
step_hist[ipower] );
hist[ipower][ihist]+=1.0;
}
}
}
cleanup_gather(mtag0);
cleanup_gather(mtag1);
}
void d_plaquette_minmax(double *ss_plaq,double *st_plaq,
double *ss_plaq_min, double *st_plaq_min,
double *ss_plaq_max, double *st_plaq_max) {
/* su3mat is scratch space of size su3_matrix */
su3_matrix *su3mat;
register int i,dir1,dir2;
register site *s;
register int first_pass_s,first_pass_t;
register su3_matrix *m1,*m4;
su3_matrix mtmp;
double ss_sum,st_sum;
double rtrace_s, rtrace_t, ss_min, st_min, ss_max, st_max;
msg_tag *mtag0,*mtag1;
ss_sum = st_sum = 0.0;
first_pass_s=1;
first_pass_t=1;
su3mat = (su3_matrix *)malloc(sizeof(su3_matrix)*sites_on_node);
if(su3mat == NULL)
{
printf("plaquette: can't malloc su3mat\n");
fflush(stdout); terminate(1);
}
for(dir1=YUP;dir1<=TUP;dir1++){
for(dir2=XUP;dir2<dir1;dir2++){
mtag0 = start_gather_site( F_OFFSET(link[dir2]), sizeof(su3_matrix),
dir1, EVENANDODD, gen_pt[0] );
mtag1 = start_gather_site( F_OFFSET(link[dir1]), sizeof(su3_matrix),
dir2, EVENANDODD, gen_pt[1] );
FORALLSITES(i,s){
m1 = &(s->link[dir1]);
m4 = &(s->link[dir2]);
mult_su3_an(m4,m1,&su3mat[i]);
}
wait_gather(mtag0);
wait_gather(mtag1);
FORALLSITES(i,s){
#ifdef SCHROED_FUN
if(dir1==TUP ){
if(s->t==(nt-1)){
mult_su3_nn( &su3mat[i],
&(s->boundary[dir2]), &mtmp);
}
else{
mult_su3_nn( &su3mat[i],
(su3_matrix *)(gen_pt[0][i]), &mtmp);
}
rtrace_t =
realtrace_su3((su3_matrix *)(gen_pt[1][i]), &mtmp);
st_sum += rtrace_t;
}
else if(s->t > 0){
mult_su3_nn( &su3mat[i], (su3_matrix *)(gen_pt[0][i]),
&mtmp);
rtrace_s =
realtrace_su3((su3_matrix *)(gen_pt[1][i]), &mtmp);
ss_sum += rtrace_s;
}
#else
mult_su3_nn( &su3mat[i], (su3_matrix *)(gen_pt[0][i]),
&mtmp);
if(dir1==TUP ) {
rtrace_t = (double)
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&mtmp);
st_sum += rtrace_t;
}
else {
rtrace_s = (double)
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&mtmp);
ss_sum += rtrace_s;
}
#endif
// printf("Plaq i=%d, dir1=%d, dir2=%d: %f %f\n",
// i,dir1,dir2,rtrace_s,rtrace_t);
/* set min and max values on the first pass */
if( dir1==TUP ) {
if( 1==first_pass_t ) {
st_min = rtrace_t;
st_max = rtrace_t;
first_pass_t = 0;
}
else {
if( rtrace_t < st_min ) st_min = rtrace_t;
if( rtrace_t > st_max ) st_max = rtrace_t;
}
}
else {
if( 1==first_pass_s ) {
ss_min = rtrace_s;
ss_max = rtrace_s;
first_pass_s = 0;
}
else {
if( rtrace_s < ss_min ) ss_min = rtrace_s;
if( rtrace_s > ss_max ) ss_max = rtrace_s;
}
}
}
cleanup_gather(mtag0);
cleanup_gather(mtag1);
}
}
void hvy_pot(int do_det) {
register int i;
register site *s;
int t_dist, x_dist;
double wloop;
complex tc;
matrix tmat, tmat2;
msg_tag *mtag = NULL;
node0_printf("hvy_pot: MAX_T = %d, MAX_X = %d\n", MAX_T, MAX_X);
// Use staple to hold product of t_dist links at each point
for (t_dist = 1; t_dist <= MAX_T; t_dist++) {
if (t_dist == 1) {
FORALLSITES(i, s)
mat_copy(&(s->link[TUP]), &(staple[i]));
}
else {
mtag = start_gather_field(staple, sizeof(matrix),
goffset[TUP], EVENANDODD, gen_pt[0]);
// Be careful about overwriting staple;
// gen_pt may just point to it for on-node "gathers"
wait_gather(mtag);
FORALLSITES(i, s)
mult_nn(&(s->link[TUP]), (matrix *)gen_pt[0][i], &(tempmat2[i]));
cleanup_gather(mtag);
FORALLSITES(i, s)
mat_copy(&(tempmat2[i]), &(staple[i]));
}
// Copy staple to tempmat
// Will shoft at end of loop
FORALLSITES(i, s)
mat_copy(&(staple[i]), &(tempmat[i]));
for (x_dist = 0; x_dist <= MAX_X; x_dist++) {
// Evaluate potential at this separation
wloop = 0.0;
FORALLSITES(i, s) {
// Compute the actual Coulomb gauge Wilson loop product
mult_na(&(staple[i]), &(tempmat[i]), &tmat);
if (do_det == 1)
det_project(&tmat, &tmat2);
else
mat_copy(&tmat, &tmat2);
tc = trace(&tmat2);
wloop += tc.real;
}
g_doublesum(&wloop);
if (do_det == 1) { // Braces fix compiler error
node0_printf("D_LOOP ");
}
else
node0_printf("POT_LOOP ");
node0_printf("%d %d %.6g\n", x_dist, t_dist, wloop / volume);
// As we increment x, shift in x direction
shiftmat(tempmat, tempmat2, goffset[XUP]);
} // x_dist
} // t_dist
/* do measurements: load density, ploop, etc. and phases onto lattice */
void measure() {
register int i,j,k, c;
register site *s;
int dx,dy,dz; /* separation for correlated observables */
int dir; /* direction of separation */
msg_tag *tag;
register complex cc,dd; /*scratch*/
complex ztr, zcof, znum, zdet, TC, zd, density, zphase;
complex p[4]; /* probabilities of n quarks at a site */
complex np[4]; /* probabilities at neighbor site */
complex pp[4][4]; /* joint probabilities of n here and m there */
complex zplp, plp;
Real locphase, phase;
/* First make T (= timelike P-loop) from s->ploop_t
T stored in s->tempmat1
*/
ploop_less_slice(nt-1,EVEN);
ploop_less_slice(nt-1,ODD);
phase = 0.;
density = plp = cmplx(0.0, 0.0);
for(j=0;j<4;j++){
p[j]=cmplx(0.0,0.0);
for(k=0;k<4;k++)pp[j][k]=cmplx(0.0,0.0);
}
FORALLSITES(i,s) {
if(s->t != nt-1) continue;
mult_su3_nn(&(s->link[TUP]), &(s->ploop_t), &(s->tempmat1));
zplp = trace_su3(&(s->tempmat1));
CSUM(plp, zplp);
ztr = trace_su3(&(s->tempmat1));
CONJG(ztr, zcof);
for(c=0; c<3; ++c) s->tempmat1.e[c][c].real += C;
zdet = det_su3(&(s->tempmat1));
znum = numer(C, ztr, zcof);
CDIV(znum, zdet, zd);
CSUM(density, zd);
/* store n_quark probabilities at this site in lattice variable
qprob[], accumulate sum over lattice in p[] */
cc = cmplx(C*C*C,0.0); CDIV(cc,zdet,s->qprob[0]); CSUM(p[0],s->qprob[0]);
CMULREAL(ztr,C*C,cc); CDIV(cc,zdet,s->qprob[1]); CSUM(p[1],s->qprob[1]);
CMULREAL(zcof,C,cc); CDIV(cc,zdet,s->qprob[2]); CSUM(p[2],s->qprob[2]);
cc = cmplx(1.0,0.0); CDIV(cc,zdet,s->qprob[3]); CSUM(p[3],s->qprob[3]);
locphase = carg(&zdet);
phase += locphase;
}
g_floatsum( &phase );
g_complexsum( &density );
g_complexsum( &p[0] );
g_complexsum( &p[1] );
g_complexsum( &p[2] );
g_complexsum( &p[3] );
g_complexsum( &plp );
CDIVREAL(density,(Real)(nx*ny*nz),density);
CDIVREAL(p[0],(Real)(nx*ny*nz),p[0]);
CDIVREAL(p[1],(Real)(nx*ny*nz),p[1]);
CDIVREAL(p[2],(Real)(nx*ny*nz),p[2]);
CDIVREAL(p[3],(Real)(nx*ny*nz),p[3]);
CDIVREAL(plp,(Real)(nx*ny*nz),plp);
zphase = ce_itheta(phase);
if(this_node == 0) {
printf("ZMES\t%e\t%e\t%e\t%e\t%e\t%e\n", zphase.real, zphase.imag,
density.real, density.imag,
plp.real, plp.imag);
printf("PMES\t%e\t%e\t%e\t%e\t%e\t%e\t%e\t%e\n",
p[0].real, p[0].imag, p[1].real, p[1].imag,
p[2].real, p[2].imag, p[3].real, p[3].imag );
}
#ifdef PPCORR
dx=1; dy=0; dz=0; /* Temporary - right now we just do nearest neighbor */
for(dir=XUP;dir<=ZUP;dir++){
tag = start_gather_site( F_OFFSET(qprob[0]), 4*sizeof(complex), dir,
EVENANDODD, gen_pt[0] );
wait_gather(tag);
FORALLSITES(i,s)if(s->t==nt-1){
for(j=0;j<4;j++)for(k=0;k<4;k++){
CMUL( (s->qprob)[j],((complex *)gen_pt[0][i])[k],cc);
CSUM(pp[j][k],cc);
}
}
cleanup_gather(tag);
}
/* density correlation format:
PP dx dy dz n1 n2 real imag */
for(j=0;j<4;j++)for(k=0;k<4;k++){
g_complexsum( &pp[j][k] );
CDIVREAL(pp[j][k],(Real)(3*nx*ny*nz),pp[j][k]);
if(this_node==0)
printf("PP %d %d %d %d %d %e %e\n",dx,dy,dz,j,k,
pp[j][k].real,pp[j][k].imag);
}
#endif /*PPCORR*/
}
void generate_hqet_prop(field_offset hqet_prop, int tstart, int tend , int tcurrent,
int v1, int v2)
{
register int i,j,t;
register site *s;
msg_tag *tag[8];
int tloop ;
int nthalf = nt/2 ;
int dir ;
int jj ;
register su3_vector *a,*b1,*b2,*b3,*b4;
complex unorm[4] ;
/*************---------**********-------------************/
assert( tend > tstart ) ;
/** check to see whether to a two or three point function ***/
if( v1 == v2 )
{
tcurrent = tend + 4 ;
}
/** set up the renormalized velocity ****/
unorm[TUP].real =0.0 ; unorm[TUP].imag = 0.0 ;
for(jj =XUP ; jj <= ZUP ; ++jj)
{
unorm[jj].real =0.0 ;
unorm[jj].imag = velocity[v1][jj] /( 2.0 * velocity[v1][TUP] );
}
++tstart ; /*** tloop is the time slice of the required propagator ****/
for(tloop=tstart ; tloop <= tend ; ++tloop)
{
if( tloop > tcurrent )
{
for(jj =XUP ; jj <= ZUP ; ++jj)
{
unorm[jj].real =0.0 ;
unorm[jj].imag = velocity[v2][jj] /( 2.0 * velocity[v2][TUP] ) ;
}
}
/***----- spatial parts of the evolution equation ------***/
/* Start gathers of the propagator from positive directions */
for(dir=XUP; dir<=ZUP; dir++)
{
tag[dir] = start_gather_site( hqet_prop, sizeof(su3_matrix), dir, EVENANDODD,
gen_pt[dir] );
}
/*
* tmpvec[i] = U_{\mu}^{\dagger}(x-i,t) s(x-i)
*/
/* Multiply by adjoint matrix at other sites */
FORALLSITES(i,s)
{
for(dir=XUP; dir<=ZUP; dir++)
{
mult_su3_an_z(unorm[dir] , &(s->link[dir]),
(su3_matrix *)F_PT(s,hqet_prop), &(s->tempvec[dir]) );
}
}
/* Start gathers from negative directions */
for( dir=XUP; dir <= ZUP; dir++)
{
tag[OPP_DIR(dir)] = start_gather_site( F_OFFSET(tempvec[dir]),
sizeof(su3_matrix), OPP_DIR( dir), EVENANDODD ,
gen_pt[OPP_DIR(dir)] );
}
/* Wait gathers from positive directions *******/
for(dir=XUP; dir<=ZUP; dir++)
{
wait_gather(tag[dir]);
}
FORALLSITES(i,s)
{
su3mat_copy((su3_matrix *)F_PT(s,hqet_prop) , &(s->tempvec[ TUP ]) );
for(dir=XUP; dir<=ZUP; dir++)
{
mult_su3_nn_z_inc( unorm[dir] ,&(s->link[dir]),(su3_matrix *)(gen_pt[dir][i]),
&(s->tempvec[ TUP ]) );
}
}
/* Wait gathers from negative directions */
for(dir=XUP; dir<=ZUP; dir++)
{
wait_gather(tag[OPP_DIR(dir)]);
}
FORALLSITES(i,s)
{
sub3_su3_matrix((su3_matrix *)(gen_pt[XDOWN][i]) ,
(su3_matrix *)(gen_pt[YDOWN][i]) ,
(su3_matrix *)(gen_pt[ZDOWN][i]) ,
&(s->tempvec[ TUP ])) ;
}
/* Special dslash_site for use by congrad. Uses restart_gather_site() when
possible. Third to last argument is an array of message tags, to be set
if this is the first use, otherwise reused. If start=1,use
start_gather_site, otherwise use restart_gather_site.
The calling program must clean up the gathers! */
void dslash_fn_site_special( field_offset src, field_offset dest,
int parity, msg_tag **tag, int start,
fn_links_t *fn){
register int i;
register site *s;
register int dir,otherparity=0;
register su3_matrix *fat4;
su3_matrix *t_fatlink;
#ifndef NO_LONG_LINKS
register su3_matrix *long4;
su3_matrix *t_longlink;
su3_vector *templongvec, *templongv1;
#endif
su3_vector *tempvec;
char myname[] = "dslash_fn_site_special";
if(fn == NULL){
printf("dslash_fn_site_special: invalid fn links!\n");
terminate(1);
}
#ifndef NO_LONG_LINKS
t_longlink = get_lnglinks(fn);
#endif
t_fatlink = get_fatlinks(fn);
tempvec = (su3_vector *) malloc(sizeof(su3_vector)*4*sites_on_node);
if(tempvec == NULL){
printf("%s(%d)No room for temporary\n",myname, this_node);
terminate(1);
}
#ifndef NO_LONG_LINKS
templongvec = (su3_vector *) malloc(sizeof(su3_vector)*4*sites_on_node);
if(templongvec == NULL){
printf("%s(%d)No room for temporary\n",myname, this_node);
terminate(1);
}
templongv1 = create_v_field();
#endif
switch(parity){
case EVEN: otherparity=ODD; break;
case ODD: otherparity=EVEN; break;
case EVENANDODD: otherparity=EVENANDODD; break;
}
/* Start gathers from positive directions */
for(dir=XUP; dir<=TUP; dir++){
if(start==1) tag[dir] = start_gather_site( src, sizeof(su3_vector),
dir, parity, gen_pt[dir] );
else restart_gather_site( src, sizeof(su3_vector),
dir, parity, gen_pt[dir] , tag[dir] );
}
/* and start the 3rd neighbor gather */
for(dir=X3UP; dir<=T3UP; dir++){
if(start==1) tag[dir] = start_gather_site( src, sizeof(su3_vector),
dir, parity, gen_pt[dir] );
else restart_gather_site( src, sizeof(su3_vector),
dir, parity, gen_pt[dir] , tag[dir] );
}
/* Multiply by adjoint matrix at other sites */
FORSOMEPARITYDOMAIN_OMP(i,s,otherparity,private(fat4,long4)){
if( i < loopend-FETCH_UP ){
fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
prefetch_4MV4V(
fat4,
(su3_vector *)F_PT(s+FETCH_UP,src),
tempvec+4*i+FETCH_UP );
#ifndef NO_LONG_LINKS
long4 = &(t_longlink[4*(i+FETCH_UP)]);
prefetch_4MV4V(
long4,
(su3_vector *)F_PT(s+FETCH_UP,src),
templongvec+4*i+FETCH_UP );
#endif
}
fat4 = &(t_fatlink[4*i]);
#ifndef NO_LONG_LINKS
long4 = &(t_longlink[4*i]);
#endif
mult_adj_su3_mat_vec_4dir( fat4,
(su3_vector *)F_PT(s,src), (tempvec+4*i) );
/* multiply by 3-link matrices too */
#ifndef NO_LONG_LINKS
mult_adj_su3_mat_vec_4dir( long4,
(su3_vector *)F_PT(s,src), (templongvec+4*i) );
#endif
} END_LOOP_OMP
/* Start gathers from negative directions */
for( dir=XUP; dir <= TUP; dir++){
if (start==1){
/* We need the strided gather so we can pick off one of a
group of four vectors in tempvec */
tag[OPP_DIR(dir)] =
declare_strided_gather( (char *)(tempvec+dir), 4*sizeof(su3_vector),
sizeof(su3_vector), OPP_DIR( dir), parity,
gen_pt[OPP_DIR(dir)] );
prepare_gather(tag[OPP_DIR(dir)]);
do_gather(tag[OPP_DIR(dir)]);
} else {
do_gather(tag[OPP_DIR(dir)]);
}
}
#ifndef NO_LONG_LINKS
/* and 3rd neighbours */
for( dir=X3UP; dir <= T3UP; dir++){
/**printf("dslash_fn_site_special: down gathers, start=%d\n",start);**/
if (start==1){
tag[OPP_3_DIR(dir)] =
declare_strided_gather( (char *)(templongvec+INDEX_3RD(dir)),
4*sizeof(su3_vector),
sizeof(su3_vector), OPP_3_DIR(dir),
parity, gen_pt[OPP_3_DIR(dir)] );
prepare_gather(tag[OPP_3_DIR(dir)]);
do_gather(tag[OPP_3_DIR(dir)]);
} else {
do_gather(tag[OPP_3_DIR(dir)]);
}
}
#endif
/* Wait gathers from positive directions, multiply by matrix and
accumulate */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[dir]);
}
/* wait for the 3-neighbours from positive directions, multiply */
for(dir=X3UP; dir<=T3UP; dir++){
wait_gather(tag[dir]);
}
FORSOMEPARITYDOMAIN_OMP(i,s,parity, private(fat4,long4) ){
if( i < loopend-FETCH_UP ){
fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
prefetch_4MVVVV(
fat4,
(su3_vector *)gen_pt[XUP][i+FETCH_UP],
(su3_vector *)gen_pt[YUP][i+FETCH_UP],
(su3_vector *)gen_pt[ZUP][i+FETCH_UP],
(su3_vector *)gen_pt[TUP][i+FETCH_UP] );
#ifndef NO_LONG_LINKS
long4 = &(t_longlink[4*(i+FETCH_UP)]);
prefetch_VV(
(su3_vector *)F_PT(s+FETCH_UP,dest),
templongv1+i+FETCH_UP);
prefetch_4MVVVV(
long4,
(su3_vector *)gen_pt[X3UP][i+FETCH_UP],
(su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
(su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
(su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
#endif
}
fat4 = &(t_fatlink[4*i]);
mult_su3_mat_vec_sum_4dir( fat4,
(su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
(su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
(su3_vector *)F_PT(s,dest));
#ifndef NO_LONG_LINKS
long4 = &(t_longlink[4*i]);
mult_su3_mat_vec_sum_4dir( long4,
(su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
(su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
templongv1+i);
#endif
} END_LOOP_OMP
/* Wait gathers from negative directions, accumulate (negative) */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[OPP_DIR(dir)]);
}
/* and the same for the negative 3-rd neighbours */
for(dir=X3UP; dir<=T3UP; dir++){
wait_gather(tag[OPP_3_DIR(dir)]);
}
FORSOMEPARITYDOMAIN_OMP(i,s,parity, ){
if( i < loopend-FETCH_UP ){
#ifndef NO_LONG_LINKS
prefetch_VV(
(su3_vector *)F_PT(s+FETCH_UP,dest),
templongv1+i+FETCH_UP);
#endif
prefetch_VVVV(
(su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
prefetch_VVVV(
(su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
}
sub_four_su3_vecs( (su3_vector *)F_PT(s,dest),
(su3_vector *)(gen_pt[XDOWN][i]),
(su3_vector *)(gen_pt[YDOWN][i]),
(su3_vector *)(gen_pt[ZDOWN][i]),
(su3_vector *)(gen_pt[TDOWN][i]) );
#ifndef NO_LONG_LINKS
sub_four_su3_vecs( templongv1+i,
(su3_vector *)(gen_pt[X3DOWN][i]),
(su3_vector *)(gen_pt[Y3DOWN][i]),
(su3_vector *)(gen_pt[Z3DOWN][i]),
(su3_vector *)(gen_pt[T3DOWN][i]) );
/*** Now need to add these things together ***/
add_su3_vector((su3_vector *)F_PT(s,dest), templongv1+i,
(su3_vector *)F_PT(s,dest));
#endif
} END_LOOP_OMP
}
void d_plaquette(double *ss_plaq,double *st_plaq) {
/* su3mat is scratch space of size su3_matrix */
su3_matrix *su3mat;
register int i,dir1,dir2;
register site *s;
register su3_matrix *m1,*m4;
su3_matrix mtmp;
double ss_sum,st_sum;
msg_tag *mtag0,*mtag1;
ss_sum = st_sum = 0.0;
su3mat = (su3_matrix *)malloc(sizeof(su3_matrix)*sites_on_node);
if(su3mat == NULL)
{
printf("plaquette: can't malloc su3mat\n");
fflush(stdout); terminate(1);
}
for(dir1=YUP;dir1<=TUP;dir1++){
for(dir2=XUP;dir2<dir1;dir2++){
mtag0 = start_gather_site( F_OFFSET(link[dir2]), sizeof(su3_matrix),
dir1, EVENANDODD, gen_pt[0] );
mtag1 = start_gather_site( F_OFFSET(link[dir1]), sizeof(su3_matrix),
dir2, EVENANDODD, gen_pt[1] );
FORALLSITES(i,s){
m1 = &(s->link[dir1]);
m4 = &(s->link[dir2]);
mult_su3_an(m4,m1,&su3mat[i]);
}
wait_gather(mtag0);
wait_gather(mtag1);
FORALLSITES(i,s){
#ifdef SCHROED_FUN
if(dir1==TUP ){
if(s->t==(nt-1)){
mult_su3_nn( &su3mat[i],
&(s->boundary[dir2]), &mtmp);
}
else{
mult_su3_nn( &su3mat[i],
(su3_matrix *)(gen_pt[0][i]), &mtmp);
}
st_sum +=
realtrace_su3((su3_matrix *)(gen_pt[1][i]), &mtmp);
}
else if(s->t > 0){
mult_su3_nn( &su3mat[i], (su3_matrix *)(gen_pt[0][i]),
&mtmp);
ss_sum +=
realtrace_su3((su3_matrix *)(gen_pt[1][i]), &mtmp);
}
#else
mult_su3_nn( &su3mat[i], (su3_matrix *)(gen_pt[0][i]),
&mtmp);
if(dir1==TUP )st_sum += (double)
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&mtmp);
else ss_sum += (double)
realtrace_su3((su3_matrix *)(gen_pt[1][i]),&mtmp);
#endif
}
cleanup_gather(mtag0);
cleanup_gather(mtag1);
}
}
/* compute wiggly links in direction dir1 decorated in direction dir2 */
void hyp_block_stage1(register int dir1, register int dir2, int parity,
su3_matrix *U_link, su3_matrix *Wiggly_link, int dir_exclude,
hyp_coeffs_t *hc ) {
register int i;
register site *st;
msg_tag *tag0,*tag1,*tag2,*tag3,*tag4;
int start;
register int count,nWiggly;
su3_matrix tmat1,tmat2,fatq;
su3_matrix *tempmat1;
int disp[4]; /* displacement vector for general gather */
/* create temporary storage, one matrix per site */
tempmat1 = create_mn_special(1);
start=1; /* indicates staple sum not initialized */
// array size is fixed for 4D, in 3D not all entries are filled
nWiggly = 12;
count=3*dir1+dir2;
if(dir2>dir1)count=count-1;
/* displacement vector for link 2 sites away */
for(i=XUP;i<=TUP;i++)disp[i]=0;
disp[dir1] = 1;
disp[dir2] = -1;
/* get U_link[dir2] from direction dir1 */
tag0 = declare_strided_gather( U_link + dir2, 4*sizeof(su3_matrix),
sizeof(su3_matrix), dir1, parity, gen_pt[0] );
do_gather( tag0 );
/* get U_link[dir1] from direction dir2 */
tag1 = declare_strided_gather( U_link + dir1, 4*sizeof(su3_matrix),
sizeof(su3_matrix), dir2, parity, gen_pt[1] );
do_gather( tag1 );
/* get U_link[dir2] from direction -dir2 */
tag2 = declare_strided_gather( U_link + dir2, 4*sizeof(su3_matrix),
sizeof(su3_matrix), OPP_DIR(dir2), parity, gen_pt[2] );
do_gather( tag2 );
/* get U_link[dir1] from direction -dir2 */
tag3 = declare_strided_gather( U_link + dir1, 4*sizeof(su3_matrix),
sizeof(su3_matrix), OPP_DIR(dir2), parity, gen_pt[3] );
do_gather( tag3 );
/* get U_link[dir2] from displacement +dir1-dir2 */
tag4 = start_general_strided_gather( (char *)(U_link + dir2), 4*sizeof(su3_matrix),
sizeof(su3_matrix), disp, parity, gen_pt[4] );
/* Upper staple */
wait_gather(tag0);
wait_gather(tag1);
if(start){ /* this is the first contribution to staple */
FORSOMEPARITY(i,st,parity){
mult_su3_nn( &(U_link[4*i+dir2]), (su3_matrix *)gen_pt[1][i], &tmat1 );
mult_su3_na( &tmat1, (su3_matrix *)gen_pt[0][i], &(tempmat1[i]) );
}
start=0;
}
void make_field_strength(
field_offset link_src, /* field offset for su3_matrix[4] type
for the source link matrices */
field_offset field_dest /* field offset for su3_matrix[6] type
for the resulting field strength */
)
{
register int i,component,dir0=-99,dir1=-99;
register site *s;
int j;
su3_matrix tmat1,tmat2;
su3_matrix *temp1,*temp2;
complex cc;
msg_tag *mtag0,*mtag1;
/* Allocate temporary space for two su3_matrix fields */
temp1 = (su3_matrix *)malloc(sites_on_node*sizeof(su3_matrix));
if(temp1 == NULL){
printf("field_strength: No room for temp1\n");
terminate(1);
}
temp2 = (su3_matrix *)malloc(sites_on_node*sizeof(su3_matrix));
if(temp2 == NULL){
printf("field_strength: No room for temp2\n");
terminate(1);
}
for(component=FS_XY;component<=FS_ZT;component++){
switch(component){
case FS_XY: dir0=XUP; dir1=YUP; break;
case FS_XZ: dir0=XUP; dir1=ZUP; break;
case FS_YZ: dir0=YUP; dir1=ZUP; break;
case FS_XT: dir0=XUP; dir1=TUP; break;
case FS_YT: dir0=YUP; dir1=TUP; break;
case FS_ZT: dir0=ZUP; dir1=TUP; break;
}
/* Plaquette in +dir0 +dir1 direction */
mtag0 = start_gather_site( LINK_OFFSET(dir0), sizeof(su3_matrix),
dir1, EVENANDODD, gen_pt[0] );
mtag1 = start_gather_site( LINK_OFFSET(dir1), sizeof(su3_matrix),
dir0, EVENANDODD, gen_pt[1] );
wait_gather(mtag0);
wait_gather(mtag1);
FORALLSITES(i,s){
mult_su3_nn( &LINK(dir0),
(su3_matrix *)(gen_pt[1][i]), &tmat1 );
mult_su3_na( &tmat1, (su3_matrix *)(gen_pt[0][i]), &tmat2 );
mult_su3_na( &tmat2, &LINK(dir1), &tmat1 );
su3_adjoint( &tmat1, &tmat2 );
sub_su3_matrix( &tmat1, &tmat2, &FIELD_STRENGTH(component) );
}
/**cleanup_gather(mtag0); Use same gather in next plaquette**/
cleanup_gather(mtag1);
/* Plaquette in -dir0 +dir1 direction */
/**mtag0 = start_gather_site( LINK_OFFSET(dir0),
sizeof(su3_matrix), dir1, EVENANDODD, gen_pt[0] );
wait_gather(mtag0); Already gathered above**/
FORALLSITES(i,s){
mult_su3_an( &LINK(dir1),
&LINK(dir0), &tmat1 );
mult_su3_an( (su3_matrix *)(gen_pt[0][i]), &tmat1, &temp1[i] );
}
// -----------------------------------------------------------------
void meas_plaq() {
register int i, dir1, dir2;
register site *s;
register matrix *m1, *m4;
double plaq_ss = 0, plaq_e[5], plaq_o[5]; // st, x, y, z, a
matrix mtmp, *tmat; // Scratch space
msg_tag *mtag0,*mtag1;
tmat = (matrix *)malloc(sizeof(matrix) * sites_on_node);
if (tmat == NULL) {
node0_printf("ERROR: can't malloc tmat in plaq_diff()\n");
fflush(stdout);
terminate(1);
}
for (i = 0; i < 5; i++) {
plaq_e[i] = 0;
plaq_o[i] = 0;
}
for (dir1 = YUP; dir1 <= TUP; dir1++) {
for (dir2 = XUP; dir2 < dir1; dir2++) {
mtag0 = start_gather_site(F_OFFSET(link[dir2]), sizeof(matrix),
dir1, EVENANDODD, gen_pt[0]);
mtag1 = start_gather_site(F_OFFSET(link[dir1]), sizeof(matrix),
dir2, EVENANDODD, gen_pt[1]);
FORALLSITES(i, s) {
m1 = &(s->link[dir1]);
m4 = &(s->link[dir2]);
mult_an(m4, m1, &tmat[i]);
}
wait_gather(mtag0);
wait_gather(mtag1);
FORALLSITES(i, s) {
mult_nn(&tmat[i], (matrix *)(gen_pt[0][i]), &mtmp);
if (dir1 == TUP) {
if ((s->t) % 2 == 0)
plaq_e[0] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
else
plaq_o[0] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
}
else // Check
plaq_ss += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
if (dir1 == XUP || dir2 == XUP) {
if ((s->x) % 2 == 0)
plaq_e[1] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
else
plaq_o[1] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
}
if (dir1 == YUP || dir2 == YUP) {
if ((s->y) % 2 == 0)
plaq_e[2] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
else
plaq_o[2] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
}
if (dir1 == ZUP || dir2 == ZUP) {
if ((s->z) % 2 == 0)
plaq_e[3] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
else
plaq_o[3] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
}
// "a" is not really even/odd, but leave the labels for consistency
if ((s->t) % 2 == 1 && (s->x) % 2 == 1
&& (s->y) % 2 == 1 && (s->z) % 2 == 1)
plaq_e[4] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
if ((s->t) % 2 == 0 && (s->x) % 2 == 0
&& (s->y) % 2 == 0 && (s->z) % 2 == 0)
plaq_o[4] += (double)realtrace((matrix *)(gen_pt[1][i]),
&mtmp);
}
cleanup_gather(mtag0);
cleanup_gather(mtag1);
} // End loop over dir2
/* D_slash routine - sets dest. on each site equal to sum of
sources parallel transported to site, with minus sign for transport
from negative directions. Use "fatlinks" for one link transport,
"longlinks" for three link transport. */
void dslash_fn( field_offset src, field_offset dest, int parity ) {
register int i;
register site *s;
register int dir,otherparity;
register su3_matrix *fat4, *long4;
msg_tag *tag[16];
if(!valid_longlinks)load_longlinks();
if(!valid_fatlinks)load_fatlinks();
switch(parity){
case EVEN: otherparity=ODD; break;
case ODD: otherparity=EVEN; break;
case EVENANDODD: otherparity=EVENANDODD; break;
}
/* Start gathers from positive directions */
/* And start the 3-step gather too */
for( dir=XUP; dir<=TUP; dir++ ){
tag[dir] = start_gather( src, sizeof(su3_vector), dir, parity,
gen_pt[dir] );
tag[DIR3(dir)] = start_gather( src, sizeof(su3_vector), DIR3(dir),
parity, gen_pt[DIR3(dir)] );
}
/* Multiply by adjoint matrix at other sites */
/* Use fat link for single link transport */
FORSOMEPARITY( i, s, otherparity ){
if( i < loopend-FETCH_UP ){
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
long4 = &(t_longlink[4*(i+FETCH_UP)]);
#else
fat4 = (s+FETCH_UP)->fatlink;
long4 = (s+FETCH_UP)->longlink;
#endif
prefetch_4MV4V(
fat4,
(su3_vector *)F_PT(s+FETCH_UP,src),
(s+FETCH_UP)->tempvec );
prefetch_4MV4V(
long4,
(su3_vector *)F_PT(s+FETCH_UP,src),
(s+FETCH_UP)->templongvec );
}
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*i]);
long4 = &(t_longlink[4*i]);
#else
fat4 = s->fatlink;
long4 = s->longlink;
#endif
mult_adj_su3_mat_vec_4dir( fat4,
(su3_vector *)F_PT(s,src), s->tempvec );
/* multiply by 3-link matrices too */
mult_adj_su3_mat_vec_4dir( long4,
(su3_vector *)F_PT(s,src), s->templongvec );
} END_LOOP
/* Start gathers from negative directions */
for( dir=XUP; dir <= TUP; dir++){
tag[OPP_DIR(dir)] = start_gather( F_OFFSET(tempvec[dir]),
sizeof(su3_vector), OPP_DIR( dir), parity,
gen_pt[OPP_DIR(dir)] );
}
/* Start 3-neighbour gathers from negative directions */
for( dir=X3UP; dir <= T3UP; dir++){
tag[OPP_3_DIR(dir)]
= start_gather( F_OFFSET(templongvec[INDEX_3RD(dir)]),
sizeof(su3_vector), OPP_3_DIR( dir), parity,
gen_pt[OPP_3_DIR(dir)] );
}
/* Wait gathers from positive directions, multiply by matrix and
accumulate */
/* wait for the 3-neighbours from positive directions, multiply */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[dir]);
wait_gather(tag[DIR3(dir)]);
}
/* Wait gathers from negative directions, accumulate (negative) */
/* and the same for the negative 3-rd neighbours */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[OPP_DIR(dir)]);
}
for(dir=X3UP; dir<=T3UP; dir++){
wait_gather(tag[OPP_3_DIR(dir)]);
}
FORSOMEPARITY(i,s,parity){
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*i]);
long4 = &(t_longlink[4*i]);
#else
fat4 = s->fatlink;
long4 = s->longlink;
#endif
mult_su3_mat_vec_sum_4dir( fat4,
(su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
(su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
(su3_vector *)F_PT(s,dest));
mult_su3_mat_vec_sum_4dir( long4,
(su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
(su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
(su3_vector *) &(s->templongv1));
if( i < loopend-FETCH_UP ){
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
long4 = &(t_longlink[4*(i+FETCH_UP)]);
#else
fat4 = (s+FETCH_UP)->fatlink;
long4 = (s+FETCH_UP)->longlink;
#endif
prefetch_4MVVVV(
fat4,
(su3_vector *)gen_pt[XUP][i+FETCH_UP],
(su3_vector *)gen_pt[YUP][i+FETCH_UP],
(su3_vector *)gen_pt[ZUP][i+FETCH_UP],
(su3_vector *)gen_pt[TUP][i+FETCH_UP] );
prefetch_4MVVVV(
long4,
(su3_vector *)gen_pt[X3UP][i+FETCH_UP],
(su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
(su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
(su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
prefetch_VVVV(
(su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
prefetch_VVVV(
(su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
}
sub_four_su3_vecs( (su3_vector *)F_PT(s,dest),
(su3_vector *)(gen_pt[XDOWN][i]),
(su3_vector *)(gen_pt[YDOWN][i]),
(su3_vector *)(gen_pt[ZDOWN][i]),
(su3_vector *)(gen_pt[TDOWN][i]) );
sub_four_su3_vecs( &(s->templongv1),
(su3_vector *)(gen_pt[X3DOWN][i]),
(su3_vector *)(gen_pt[Y3DOWN][i]),
(su3_vector *)(gen_pt[Z3DOWN][i]),
(su3_vector *)(gen_pt[T3DOWN][i]) );
/* Now need to add these things together */
add_su3_vector((su3_vector *)F_PT(s,dest), & (s->templongv1),
(su3_vector *)F_PT(s,dest));
} END_LOOP
