void
dslash_fn_dir(su3_vector *src, su3_vector *dest, int parity,
fn_links_t *fn, int dir, int fb,
Real wtfat, Real wtlong)
{
register int i ;
site *s;
msg_tag *tag[2];
su3_matrix *fat = get_fatlinks(fn);
su3_matrix *lng = get_lnglinks(fn);
su3_vector tmp;
int do_long = (lng != NULL) && (wtlong != 0.);
char myname[] = "fn_shift";
if(fat == NULL) {
printf("%s(%d): fat or lng member is null\n", myname, this_node);
terminate(1);
}
if(fb > 0){
/* Shift from forward direction */
tag[0] = start_gather_field( src, sizeof(su3_vector), dir,
parity, gen_pt[0] );
if(do_long)
tag[1] = start_gather_field( src, sizeof(su3_vector), DIR3(dir),
parity, gen_pt[1] );
wait_gather(tag[0]);
if(do_long)
wait_gather(tag[1]);
FORSOMEPARITYDOMAIN(i,s,parity)
{
mult_su3_mat_vec( fat+4*i+dir, (su3_vector *)gen_pt[0][i], &tmp );
scalar_mult_add_su3_vector( dest+i, &tmp, wtfat, dest+i ) ;
if(do_long){
mult_su3_mat_vec( lng+4*i+dir, (su3_vector *)gen_pt[1][i], &tmp );
scalar_mult_add_su3_vector( dest+i, &tmp, wtlong, dest+i ) ;
}
} END_LOOP
/* D_slash routine - sets dest. on each site equal to sum of
sources parallel transported to site, with minus sign for transport
from negative directions. Use "fatlinks" for one link transport,
"longlinks" for three link transport. */
void dslash_fn( field_offset src, field_offset dest, int parity ) {
register int i;
register site *s;
register int dir,otherparity;
register su3_matrix *fat4, *long4;
msg_tag *tag[16];
if(!valid_longlinks)load_longlinks();
if(!valid_fatlinks)load_fatlinks();
switch(parity){
case EVEN: otherparity=ODD; break;
case ODD: otherparity=EVEN; break;
case EVENANDODD: otherparity=EVENANDODD; break;
}
/* Start gathers from positive directions */
/* And start the 3-step gather too */
for( dir=XUP; dir<=TUP; dir++ ){
tag[dir] = start_gather( src, sizeof(su3_vector), dir, parity,
gen_pt[dir] );
tag[DIR3(dir)] = start_gather( src, sizeof(su3_vector), DIR3(dir),
parity, gen_pt[DIR3(dir)] );
}
/* Multiply by adjoint matrix at other sites */
/* Use fat link for single link transport */
FORSOMEPARITY( i, s, otherparity ){
if( i < loopend-FETCH_UP ){
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
long4 = &(t_longlink[4*(i+FETCH_UP)]);
#else
fat4 = (s+FETCH_UP)->fatlink;
long4 = (s+FETCH_UP)->longlink;
#endif
prefetch_4MV4V(
fat4,
(su3_vector *)F_PT(s+FETCH_UP,src),
(s+FETCH_UP)->tempvec );
prefetch_4MV4V(
long4,
(su3_vector *)F_PT(s+FETCH_UP,src),
(s+FETCH_UP)->templongvec );
}
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*i]);
long4 = &(t_longlink[4*i]);
#else
fat4 = s->fatlink;
long4 = s->longlink;
#endif
mult_adj_su3_mat_vec_4dir( fat4,
(su3_vector *)F_PT(s,src), s->tempvec );
/* multiply by 3-link matrices too */
mult_adj_su3_mat_vec_4dir( long4,
(su3_vector *)F_PT(s,src), s->templongvec );
} END_LOOP
/* Start gathers from negative directions */
for( dir=XUP; dir <= TUP; dir++){
tag[OPP_DIR(dir)] = start_gather( F_OFFSET(tempvec[dir]),
sizeof(su3_vector), OPP_DIR( dir), parity,
gen_pt[OPP_DIR(dir)] );
}
/* Start 3-neighbour gathers from negative directions */
for( dir=X3UP; dir <= T3UP; dir++){
tag[OPP_3_DIR(dir)]
= start_gather( F_OFFSET(templongvec[INDEX_3RD(dir)]),
sizeof(su3_vector), OPP_3_DIR( dir), parity,
gen_pt[OPP_3_DIR(dir)] );
}
/* Wait gathers from positive directions, multiply by matrix and
accumulate */
/* wait for the 3-neighbours from positive directions, multiply */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[dir]);
wait_gather(tag[DIR3(dir)]);
}
/* Wait gathers from negative directions, accumulate (negative) */
/* and the same for the negative 3-rd neighbours */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[OPP_DIR(dir)]);
}
for(dir=X3UP; dir<=T3UP; dir++){
wait_gather(tag[OPP_3_DIR(dir)]);
}
FORSOMEPARITY(i,s,parity){
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*i]);
long4 = &(t_longlink[4*i]);
#else
fat4 = s->fatlink;
long4 = s->longlink;
#endif
mult_su3_mat_vec_sum_4dir( fat4,
(su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
(su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
(su3_vector *)F_PT(s,dest));
mult_su3_mat_vec_sum_4dir( long4,
(su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
(su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
(su3_vector *) &(s->templongv1));
if( i < loopend-FETCH_UP ){
#ifdef DSLASH_TMP_LINKS
fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
long4 = &(t_longlink[4*(i+FETCH_UP)]);
#else
fat4 = (s+FETCH_UP)->fatlink;
long4 = (s+FETCH_UP)->longlink;
#endif
prefetch_4MVVVV(
fat4,
(su3_vector *)gen_pt[XUP][i+FETCH_UP],
(su3_vector *)gen_pt[YUP][i+FETCH_UP],
(su3_vector *)gen_pt[ZUP][i+FETCH_UP],
(su3_vector *)gen_pt[TUP][i+FETCH_UP] );
prefetch_4MVVVV(
long4,
(su3_vector *)gen_pt[X3UP][i+FETCH_UP],
(su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
(su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
(su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
prefetch_VVVV(
(su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
(su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
prefetch_VVVV(
(su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
(su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
}
sub_four_su3_vecs( (su3_vector *)F_PT(s,dest),
(su3_vector *)(gen_pt[XDOWN][i]),
(su3_vector *)(gen_pt[YDOWN][i]),
(su3_vector *)(gen_pt[ZDOWN][i]),
(su3_vector *)(gen_pt[TDOWN][i]) );
sub_four_su3_vecs( &(s->templongv1),
(su3_vector *)(gen_pt[X3DOWN][i]),
(su3_vector *)(gen_pt[Y3DOWN][i]),
(su3_vector *)(gen_pt[Z3DOWN][i]),
(su3_vector *)(gen_pt[T3DOWN][i]) );
/* Now need to add these things together */
add_su3_vector((su3_vector *)F_PT(s,dest), & (s->templongv1),
(su3_vector *)F_PT(s,dest));
} END_LOOP
/* Special dslash for use by congrad. Uses restart_gather_field() when
possible. Next to last argument is an array of message tags, to be set
if this is the first use, otherwise reused. If start=1,use
start_gather_field, otherwise use restart_gather_field.
The calling program must clean up the gathers and temps! */
void dslash_fn_field_special(su3_vector *src, su3_vector *dest,
int parity, msg_tag **tag, int start,
fn_links_t *fn){
register int i;
register site *s;
register int dir,otherparity=0;
register su3_matrix *fat4;
su3_matrix *t_fatlink;
#ifndef NO_LONG_LINKS
register su3_matrix *long4;
su3_matrix *t_longlink;
#endif
/* allocate temporary work space only if not already allocated */
if(temp_not_allocated)
{
for( dir=XUP; dir<=TUP; dir++ ){
temp[dir] =(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
temp[dir+4]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
}
temp[8]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
temp_not_allocated = 0 ;
}
/* load fatlinks and longlinks */
if(fn == NULL){
printf("dslash_fn_field_special: invalid fn links!\n");
terminate(1);
}
#ifndef NO_LONG_LINKS
t_longlink = get_lnglinks(fn);
#endif
t_fatlink = get_fatlinks(fn);
switch(parity)
{
case EVEN: otherparity=ODD; break;
case ODD: otherparity=EVEN; break;
case EVENANDODD: otherparity=EVENANDODD; break;
}
/* Start gathers from positive directions */
/* And start the 3-step gather too */
for( dir=XUP; dir<=TUP; dir++ ){
if(start==1)
{
tag[dir] = start_gather_field( src, sizeof(su3_vector),
dir, parity,gen_pt[dir] );
#ifndef NO_LONG_LINKS
tag[DIR3(dir)] = start_gather_field(src, sizeof(su3_vector),
DIR3(dir),parity,
gen_pt[DIR3(dir)] );
#endif
}
else
{
restart_gather_field( src, sizeof(su3_vector),
dir, parity,gen_pt[dir], tag[dir]);
#ifndef NO_LONG_LINKS
restart_gather_field(src, sizeof(su3_vector), DIR3(dir), parity,
gen_pt[DIR3(dir)], tag[DIR3(dir)]);
#endif
}
}
/* Multiply by adjoint matrix at other sites */
/* Use fat link for single link transport */
FORSOMEPARITYDOMAIN_OMP( i, s, otherparity, private(fat4,long4) ){
//NOPRE if( i < loopend-FETCH_UP ){
//NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_V(&(src[i+FETCH_UP]));
//NOPRE prefetch_4MVVVV(
//NOPRE fat4,
//NOPRE &(temp[0][i+FETCH_UP]),
//NOPRE &(temp[1][i+FETCH_UP]),
//NOPRE &(temp[2][i+FETCH_UP]),
//NOPRE &(temp[3][i+FETCH_UP]) );
#ifndef NO_LONG_LINKS
//NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_4MVVVV(
//NOPRE long4,
//NOPRE &(temp[4][i+FETCH_UP]),
//NOPRE &(temp[5][i+FETCH_UP]),
//NOPRE &(temp[6][i+FETCH_UP]),
//NOPRE &(temp[7][i+FETCH_UP]) );
#endif
//NOPRE }
fat4 = &(t_fatlink[4*i]);
mult_adj_su3_mat_4vec( fat4, &(src[i]), &(temp[0][i]),
&(temp[1][i]), &(temp[2][i]), &(temp[3][i]) );
#ifndef NO_LONG_LINKS
/* multiply by 3-link matrices too */
long4 = &(t_longlink[4*i]);
mult_adj_su3_mat_4vec( long4, &(src[i]),&(temp[4][i]),
&(temp[5][i]), &(temp[6][i]), &(temp[7][i]) );
#endif
} END_LOOP_OMP
/* Start gathers from negative directions */
for( dir=XUP; dir <= TUP; dir++){
if (start==1) tag[OPP_DIR(dir)] = start_gather_field( temp[dir],
sizeof(su3_vector), OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)] );
else restart_gather_field( temp[dir], sizeof(su3_vector),
OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)], tag[OPP_DIR(dir)] );
}
/* Start 3-neighbour gathers from negative directions */
for( dir=X3UP; dir <= T3UP; dir++){
if (start==1) tag[OPP_3_DIR(dir)]=start_gather_field(
temp[INDEX_3RD(dir)+4], sizeof(su3_vector),
OPP_3_DIR( dir), parity, gen_pt[OPP_3_DIR(dir)] );
else restart_gather_field(temp[INDEX_3RD(dir)+4],
sizeof(su3_vector), OPP_3_DIR( dir),parity,
gen_pt[OPP_3_DIR(dir)], tag[OPP_3_DIR(dir)] );
}
/* Wait gathers from positive directions, multiply by matrix and
accumulate */
/* wait for the 3-neighbours from positive directions, multiply */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[dir]);
#ifndef NO_LONG_LINKS
wait_gather(tag[DIR3(dir)]);
#endif
}
FORSOMEPARITYDOMAIN_OMP(i,s,parity, private(fat4,long4) ){
//NOPRE if( i < loopend-FETCH_UP ){
//NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_4MVVVV(
//NOPRE fat4,
//NOPRE (su3_vector *)gen_pt[XUP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[YUP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[ZUP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[TUP][i+FETCH_UP] );
//NOPRE prefetch_VVVV(
//NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
#ifndef NO_LONG_LINKS
//NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]);
//NOPRE prefetch_4MVVVV(
//NOPRE long4,
//NOPRE (su3_vector *)gen_pt[X3UP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
//NOPRE prefetch_VVVV(
//NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
#endif
//NOPRE }
fat4 = &(t_fatlink[4*i]);
mult_su3_mat_vec_sum_4dir( fat4,
(su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
(su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
&(dest[i]) );
#ifndef NO_LONG_LINKS
long4 = &(t_longlink[4*i]);
mult_su3_mat_vec_sum_4dir( long4,
(su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
(su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
&(temp[8][i]));
#endif
} END_LOOP_OMP
/* Wait gathers from negative directions, accumulate (negative) */
/* and the same for the negative 3-rd neighbours */
for(dir=XUP; dir<=TUP; dir++){
wait_gather(tag[OPP_DIR(dir)]);
}
for(dir=X3UP; dir<=T3UP; dir++){
wait_gather(tag[OPP_3_DIR(dir)]);
}
FORSOMEPARITYDOMAIN_OMP(i,s,parity, ){
//NOPRE if( i < loopend-FETCH_UP ){
//NOPRE prefetch_VVVVV(
//NOPRE &(dest[i+FETCH_UP]),
//NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
//NOPRE prefetch_VVVVV(
//NOPRE &(temp[8][i+FETCH_UP]),
//NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
//NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
//NOPRE }
sub_four_su3_vecs( &(dest[i]),
(su3_vector *)(gen_pt[XDOWN][i]),
(su3_vector *)(gen_pt[YDOWN][i]),
(su3_vector *)(gen_pt[ZDOWN][i]),
(su3_vector *)(gen_pt[TDOWN][i]) );
sub_four_su3_vecs( &(temp[8][i]),
(su3_vector *)(gen_pt[X3DOWN][i]),
(su3_vector *)(gen_pt[Y3DOWN][i]),
(su3_vector *)(gen_pt[Z3DOWN][i]),
(su3_vector *)(gen_pt[T3DOWN][i]) );
/* Now need to add these things together */
add_su3_vector(&(dest[i]), &(temp[8][i]),&(dest[i]));
} END_LOOP_OMP
}
/* Smearing level 0 */
static void
QOP_hisq_force_multi_smearing0_fnmat(QOP_info_t *info,
REAL *residues,
QDP_ColorVector *x[],
int nterms,
QDP_ColorMatrix *force_accum[4],
QDP_ColorMatrix *force_accum_naik[4])
{
int term;
int i,k;
int dir;
REAL coeff;
QDP_ColorMatrix *tmat;
QDP_ColorMatrix *oprod_along_path[MAX_PATH_LENGTH+1];
QDP_ColorMatrix *mat_tmp0;
QDP_ColorVector *tsrc[2], *vec_tmp[2];
size_t nflops = 0;
if( nterms==0 )return;
mat_tmp0 = QDP_create_M();
tmat = QDP_create_M();
tsrc[0] = QDP_create_V();
tsrc[1] = QDP_create_V();
vec_tmp[0] = QDP_create_V();
vec_tmp[1] = QDP_create_V();
for(i=0;i<=MAX_PATH_LENGTH;i++){
oprod_along_path[i] = QDP_create_M();
}
// clear force accumulators
for(dir=XUP;dir<=TUP;dir++)
QDP_M_eq_zero(force_accum[dir], QDP_all);
for(dir=XUP;dir<=TUP;dir++){ //AB loop on directions, path table is not needed
k=0; // which vec_tmp we are using (0 or 1)
QDP_V_eq_V(tsrc[k], x[0], QDP_all);
QDP_V_eq_sV(vec_tmp[k], tsrc[k],
fnshift(OPP_DIR(dir)), fndir(OPP_DIR(dir)), QDP_all);
QDP_M_eq_zero(oprod_along_path[0], QDP_all);
for(term=0;term<nterms;term++){
if(term<nterms-1) {
QDP_V_eq_V(tsrc[1-k], x[term+1], QDP_all);
QDP_V_eq_sV(vec_tmp[1-k], tsrc[1-k],
fnshift(OPP_DIR(dir)), fndir(OPP_DIR(dir)), QDP_all);
}
//QDP_M_eq_V_times_Va(tmat, x[term], vec_tmp[k], QDP_all);
QDP_M_eq_V_times_Va(tmat, tsrc[k], vec_tmp[k], QDP_all);
nflops += 54;
QDP_discard_V(vec_tmp[k]);
QDP_M_peq_r_times_M(oprod_along_path[0], &residues[term], tmat,
QDP_all);
nflops += 36;
k=1-k; // swap 0 and 1
} // end loop over terms in rational function expansion
link_gather_connection_qdp(oprod_along_path[1], oprod_along_path[0], tmat,
dir );
coeff = 1.;
QDP_M_peq_r_times_M(force_accum[dir],&coeff,oprod_along_path[1],QDP_all);
nflops += 36;
} // end of loop on directions //
// *** Naik part *** /
// clear force accumulators
for(dir=XUP;dir<=TUP;dir++)
QDP_M_eq_zero(force_accum_naik[dir], QDP_all);
for(dir=XUP;dir<=TUP;dir++){ //AB loop on directions, path table is not needed
k=0; // which vec_tmp we are using (0 or 1)
QDP_V_eq_V(tsrc[k], x[0], QDP_all);
QDP_V_eq_sV(vec_tmp[k], tsrc[k], fnshift(OPP_3_DIR( DIR3(dir) )),
fndir(OPP_3_DIR( DIR3(dir) )), QDP_all);
QDP_M_eq_zero(oprod_along_path[0], QDP_all);
for(term=0;term<nterms;term++){
if(term<nterms-1) {
QDP_V_eq_V(tsrc[1-k], x[term+1], QDP_all);
QDP_V_eq_sV(vec_tmp[1-k], tsrc[1-k], fnshift(OPP_3_DIR( DIR3(dir) )),
fndir(OPP_3_DIR( DIR3(dir) )), QDP_all);
}
//QDP_M_eq_V_times_Va(tmat, x[term], vec_tmp[k], QDP_all);
QDP_M_eq_V_times_Va(tmat, tsrc[k], vec_tmp[k], QDP_all);
nflops += 54;
QDP_discard_V(vec_tmp[k]);
QDP_M_peq_r_times_M(oprod_along_path[0], &residues[term], tmat, QDP_all);
nflops += 36;
k=1-k; // swap 0 and 1
} // end loop over terms in rational function expansion
link_gather_connection_qdp(oprod_along_path[1], oprod_along_path[0], tmat,
DIR3(dir) );
coeff = 1; // fermion_eps is outside this routine in "wrapper" routine
QDP_M_peq_r_times_M(force_accum_naik[dir],&coeff,
oprod_along_path[1],QDP_all);
nflops += 36;
} // end of loop on directions
QDP_destroy_V( tsrc[0] );
QDP_destroy_V( tsrc[1] );
QDP_destroy_V( vec_tmp[0] );
QDP_destroy_V( vec_tmp[1] );
QDP_destroy_M( mat_tmp0 );
QDP_destroy_M( tmat );
for(i=0;i<=MAX_PATH_LENGTH;i++){
QDP_destroy_M( oprod_along_path[i] );
}
info->final_flop = ((double)nflops)*QDP_sites_on_node;
return;
} //hisq_force_multi_smearing0_fnmat
