void dslash_fn_dir(su3_vector *src, su3_vector *dest, int parity, fn_links_t *fn, int dir, int fb, Real wtfat, Real wtlong) { register int i ; site *s; msg_tag *tag[2]; su3_matrix *fat = get_fatlinks(fn); su3_matrix *lng = get_lnglinks(fn); su3_vector tmp; int do_long = (lng != NULL) && (wtlong != 0.); char myname[] = "fn_shift"; if(fat == NULL) { printf("%s(%d): fat or lng member is null\n", myname, this_node); terminate(1); } if(fb > 0){ /* Shift from forward direction */ tag[0] = start_gather_field( src, sizeof(su3_vector), dir, parity, gen_pt[0] ); if(do_long) tag[1] = start_gather_field( src, sizeof(su3_vector), DIR3(dir), parity, gen_pt[1] ); wait_gather(tag[0]); if(do_long) wait_gather(tag[1]); FORSOMEPARITYDOMAIN(i,s,parity) { mult_su3_mat_vec( fat+4*i+dir, (su3_vector *)gen_pt[0][i], &tmp ); scalar_mult_add_su3_vector( dest+i, &tmp, wtfat, dest+i ) ; if(do_long){ mult_su3_mat_vec( lng+4*i+dir, (su3_vector *)gen_pt[1][i], &tmp ); scalar_mult_add_su3_vector( dest+i, &tmp, wtlong, dest+i ) ; } } END_LOOP
/* D_slash routine - sets dest. on each site equal to sum of sources parallel transported to site, with minus sign for transport from negative directions. Use "fatlinks" for one link transport, "longlinks" for three link transport. */ void dslash_fn( field_offset src, field_offset dest, int parity ) { register int i; register site *s; register int dir,otherparity; register su3_matrix *fat4, *long4; msg_tag *tag[16]; if(!valid_longlinks)load_longlinks(); if(!valid_fatlinks)load_fatlinks(); switch(parity){ case EVEN: otherparity=ODD; break; case ODD: otherparity=EVEN; break; case EVENANDODD: otherparity=EVENANDODD; break; } /* Start gathers from positive directions */ /* And start the 3-step gather too */ for( dir=XUP; dir<=TUP; dir++ ){ tag[dir] = start_gather( src, sizeof(su3_vector), dir, parity, gen_pt[dir] ); tag[DIR3(dir)] = start_gather( src, sizeof(su3_vector), DIR3(dir), parity, gen_pt[DIR3(dir)] ); } /* Multiply by adjoint matrix at other sites */ /* Use fat link for single link transport */ FORSOMEPARITY( i, s, otherparity ){ if( i < loopend-FETCH_UP ){ #ifdef DSLASH_TMP_LINKS fat4 = &(t_fatlink[4*(i+FETCH_UP)]); long4 = &(t_longlink[4*(i+FETCH_UP)]); #else fat4 = (s+FETCH_UP)->fatlink; long4 = (s+FETCH_UP)->longlink; #endif prefetch_4MV4V( fat4, (su3_vector *)F_PT(s+FETCH_UP,src), (s+FETCH_UP)->tempvec ); prefetch_4MV4V( long4, (su3_vector *)F_PT(s+FETCH_UP,src), (s+FETCH_UP)->templongvec ); } #ifdef DSLASH_TMP_LINKS fat4 = &(t_fatlink[4*i]); long4 = &(t_longlink[4*i]); #else fat4 = s->fatlink; long4 = s->longlink; #endif mult_adj_su3_mat_vec_4dir( fat4, (su3_vector *)F_PT(s,src), s->tempvec ); /* multiply by 3-link matrices too */ mult_adj_su3_mat_vec_4dir( long4, (su3_vector *)F_PT(s,src), s->templongvec ); } END_LOOP /* Start gathers from negative directions */ for( dir=XUP; dir <= TUP; dir++){ tag[OPP_DIR(dir)] = start_gather( F_OFFSET(tempvec[dir]), sizeof(su3_vector), OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)] ); } /* Start 3-neighbour gathers from negative directions */ for( dir=X3UP; dir <= T3UP; dir++){ tag[OPP_3_DIR(dir)] = start_gather( F_OFFSET(templongvec[INDEX_3RD(dir)]), sizeof(su3_vector), OPP_3_DIR( dir), parity, gen_pt[OPP_3_DIR(dir)] ); } /* Wait gathers from positive directions, multiply by matrix and accumulate */ /* wait for the 3-neighbours from positive directions, multiply */ for(dir=XUP; dir<=TUP; dir++){ wait_gather(tag[dir]); wait_gather(tag[DIR3(dir)]); } /* Wait gathers from negative directions, accumulate (negative) */ /* and the same for the negative 3-rd neighbours */ for(dir=XUP; dir<=TUP; dir++){ wait_gather(tag[OPP_DIR(dir)]); } for(dir=X3UP; dir<=T3UP; dir++){ wait_gather(tag[OPP_3_DIR(dir)]); } FORSOMEPARITY(i,s,parity){ #ifdef DSLASH_TMP_LINKS fat4 = &(t_fatlink[4*i]); long4 = &(t_longlink[4*i]); #else fat4 = s->fatlink; long4 = s->longlink; #endif mult_su3_mat_vec_sum_4dir( fat4, (su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i], (su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i], (su3_vector *)F_PT(s,dest)); mult_su3_mat_vec_sum_4dir( long4, (su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i], (su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i], (su3_vector *) &(s->templongv1)); if( i < loopend-FETCH_UP ){ #ifdef DSLASH_TMP_LINKS fat4 = &(t_fatlink[4*(i+FETCH_UP)]); long4 = &(t_longlink[4*(i+FETCH_UP)]); #else fat4 = (s+FETCH_UP)->fatlink; long4 = (s+FETCH_UP)->longlink; #endif prefetch_4MVVVV( fat4, (su3_vector *)gen_pt[XUP][i+FETCH_UP], (su3_vector *)gen_pt[YUP][i+FETCH_UP], (su3_vector *)gen_pt[ZUP][i+FETCH_UP], (su3_vector *)gen_pt[TUP][i+FETCH_UP] ); prefetch_4MVVVV( long4, (su3_vector *)gen_pt[X3UP][i+FETCH_UP], (su3_vector *)gen_pt[Y3UP][i+FETCH_UP], (su3_vector *)gen_pt[Z3UP][i+FETCH_UP], (su3_vector *)gen_pt[T3UP][i+FETCH_UP] ); prefetch_VVVV( (su3_vector *)gen_pt[XDOWN][i+FETCH_UP], (su3_vector *)gen_pt[YDOWN][i+FETCH_UP], (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP], (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] ); prefetch_VVVV( (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP], (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP], (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP], (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] ); } sub_four_su3_vecs( (su3_vector *)F_PT(s,dest), (su3_vector *)(gen_pt[XDOWN][i]), (su3_vector *)(gen_pt[YDOWN][i]), (su3_vector *)(gen_pt[ZDOWN][i]), (su3_vector *)(gen_pt[TDOWN][i]) ); sub_four_su3_vecs( &(s->templongv1), (su3_vector *)(gen_pt[X3DOWN][i]), (su3_vector *)(gen_pt[Y3DOWN][i]), (su3_vector *)(gen_pt[Z3DOWN][i]), (su3_vector *)(gen_pt[T3DOWN][i]) ); /* Now need to add these things together */ add_su3_vector((su3_vector *)F_PT(s,dest), & (s->templongv1), (su3_vector *)F_PT(s,dest)); } END_LOOP
/* Special dslash for use by congrad. Uses restart_gather_field() when possible. Next to last argument is an array of message tags, to be set if this is the first use, otherwise reused. If start=1,use start_gather_field, otherwise use restart_gather_field. The calling program must clean up the gathers and temps! */ void dslash_fn_field_special(su3_vector *src, su3_vector *dest, int parity, msg_tag **tag, int start, fn_links_t *fn){ register int i; register site *s; register int dir,otherparity=0; register su3_matrix *fat4; su3_matrix *t_fatlink; #ifndef NO_LONG_LINKS register su3_matrix *long4; su3_matrix *t_longlink; #endif /* allocate temporary work space only if not already allocated */ if(temp_not_allocated) { for( dir=XUP; dir<=TUP; dir++ ){ temp[dir] =(su3_vector *)malloc(sites_on_node*sizeof(su3_vector)); temp[dir+4]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector)); } temp[8]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector)); temp_not_allocated = 0 ; } /* load fatlinks and longlinks */ if(fn == NULL){ printf("dslash_fn_field_special: invalid fn links!\n"); terminate(1); } #ifndef NO_LONG_LINKS t_longlink = get_lnglinks(fn); #endif t_fatlink = get_fatlinks(fn); switch(parity) { case EVEN: otherparity=ODD; break; case ODD: otherparity=EVEN; break; case EVENANDODD: otherparity=EVENANDODD; break; } /* Start gathers from positive directions */ /* And start the 3-step gather too */ for( dir=XUP; dir<=TUP; dir++ ){ if(start==1) { tag[dir] = start_gather_field( src, sizeof(su3_vector), dir, parity,gen_pt[dir] ); #ifndef NO_LONG_LINKS tag[DIR3(dir)] = start_gather_field(src, sizeof(su3_vector), DIR3(dir),parity, gen_pt[DIR3(dir)] ); #endif } else { restart_gather_field( src, sizeof(su3_vector), dir, parity,gen_pt[dir], tag[dir]); #ifndef NO_LONG_LINKS restart_gather_field(src, sizeof(su3_vector), DIR3(dir), parity, gen_pt[DIR3(dir)], tag[DIR3(dir)]); #endif } } /* Multiply by adjoint matrix at other sites */ /* Use fat link for single link transport */ FORSOMEPARITYDOMAIN_OMP( i, s, otherparity, private(fat4,long4) ){ //NOPRE if( i < loopend-FETCH_UP ){ //NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]); //NOPRE prefetch_V(&(src[i+FETCH_UP])); //NOPRE prefetch_4MVVVV( //NOPRE fat4, //NOPRE &(temp[0][i+FETCH_UP]), //NOPRE &(temp[1][i+FETCH_UP]), //NOPRE &(temp[2][i+FETCH_UP]), //NOPRE &(temp[3][i+FETCH_UP]) ); #ifndef NO_LONG_LINKS //NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]); //NOPRE prefetch_4MVVVV( //NOPRE long4, //NOPRE &(temp[4][i+FETCH_UP]), //NOPRE &(temp[5][i+FETCH_UP]), //NOPRE &(temp[6][i+FETCH_UP]), //NOPRE &(temp[7][i+FETCH_UP]) ); #endif //NOPRE } fat4 = &(t_fatlink[4*i]); mult_adj_su3_mat_4vec( fat4, &(src[i]), &(temp[0][i]), &(temp[1][i]), &(temp[2][i]), &(temp[3][i]) ); #ifndef NO_LONG_LINKS /* multiply by 3-link matrices too */ long4 = &(t_longlink[4*i]); mult_adj_su3_mat_4vec( long4, &(src[i]),&(temp[4][i]), &(temp[5][i]), &(temp[6][i]), &(temp[7][i]) ); #endif } END_LOOP_OMP /* Start gathers from negative directions */ for( dir=XUP; dir <= TUP; dir++){ if (start==1) tag[OPP_DIR(dir)] = start_gather_field( temp[dir], sizeof(su3_vector), OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)] ); else restart_gather_field( temp[dir], sizeof(su3_vector), OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)], tag[OPP_DIR(dir)] ); } /* Start 3-neighbour gathers from negative directions */ for( dir=X3UP; dir <= T3UP; dir++){ if (start==1) tag[OPP_3_DIR(dir)]=start_gather_field( temp[INDEX_3RD(dir)+4], sizeof(su3_vector), OPP_3_DIR( dir), parity, gen_pt[OPP_3_DIR(dir)] ); else restart_gather_field(temp[INDEX_3RD(dir)+4], sizeof(su3_vector), OPP_3_DIR( dir),parity, gen_pt[OPP_3_DIR(dir)], tag[OPP_3_DIR(dir)] ); } /* Wait gathers from positive directions, multiply by matrix and accumulate */ /* wait for the 3-neighbours from positive directions, multiply */ for(dir=XUP; dir<=TUP; dir++){ wait_gather(tag[dir]); #ifndef NO_LONG_LINKS wait_gather(tag[DIR3(dir)]); #endif } FORSOMEPARITYDOMAIN_OMP(i,s,parity, private(fat4,long4) ){ //NOPRE if( i < loopend-FETCH_UP ){ //NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]); //NOPRE prefetch_4MVVVV( //NOPRE fat4, //NOPRE (su3_vector *)gen_pt[XUP][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[YUP][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[ZUP][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[TUP][i+FETCH_UP] ); //NOPRE prefetch_VVVV( //NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] ); #ifndef NO_LONG_LINKS //NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]); //NOPRE prefetch_4MVVVV( //NOPRE long4, //NOPRE (su3_vector *)gen_pt[X3UP][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[Y3UP][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[Z3UP][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[T3UP][i+FETCH_UP] ); //NOPRE prefetch_VVVV( //NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] ); #endif //NOPRE } fat4 = &(t_fatlink[4*i]); mult_su3_mat_vec_sum_4dir( fat4, (su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i], (su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i], &(dest[i]) ); #ifndef NO_LONG_LINKS long4 = &(t_longlink[4*i]); mult_su3_mat_vec_sum_4dir( long4, (su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i], (su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i], &(temp[8][i])); #endif } END_LOOP_OMP /* Wait gathers from negative directions, accumulate (negative) */ /* and the same for the negative 3-rd neighbours */ for(dir=XUP; dir<=TUP; dir++){ wait_gather(tag[OPP_DIR(dir)]); } for(dir=X3UP; dir<=T3UP; dir++){ wait_gather(tag[OPP_3_DIR(dir)]); } FORSOMEPARITYDOMAIN_OMP(i,s,parity, ){ //NOPRE if( i < loopend-FETCH_UP ){ //NOPRE prefetch_VVVVV( //NOPRE &(dest[i+FETCH_UP]), //NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] ); //NOPRE prefetch_VVVVV( //NOPRE &(temp[8][i+FETCH_UP]), //NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP], //NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] ); //NOPRE } sub_four_su3_vecs( &(dest[i]), (su3_vector *)(gen_pt[XDOWN][i]), (su3_vector *)(gen_pt[YDOWN][i]), (su3_vector *)(gen_pt[ZDOWN][i]), (su3_vector *)(gen_pt[TDOWN][i]) ); sub_four_su3_vecs( &(temp[8][i]), (su3_vector *)(gen_pt[X3DOWN][i]), (su3_vector *)(gen_pt[Y3DOWN][i]), (su3_vector *)(gen_pt[Z3DOWN][i]), (su3_vector *)(gen_pt[T3DOWN][i]) ); /* Now need to add these things together */ add_su3_vector(&(dest[i]), &(temp[8][i]),&(dest[i])); } END_LOOP_OMP }
/* Smearing level 0 */ static void QOP_hisq_force_multi_smearing0_fnmat(QOP_info_t *info, REAL *residues, QDP_ColorVector *x[], int nterms, QDP_ColorMatrix *force_accum[4], QDP_ColorMatrix *force_accum_naik[4]) { int term; int i,k; int dir; REAL coeff; QDP_ColorMatrix *tmat; QDP_ColorMatrix *oprod_along_path[MAX_PATH_LENGTH+1]; QDP_ColorMatrix *mat_tmp0; QDP_ColorVector *tsrc[2], *vec_tmp[2]; size_t nflops = 0; if( nterms==0 )return; mat_tmp0 = QDP_create_M(); tmat = QDP_create_M(); tsrc[0] = QDP_create_V(); tsrc[1] = QDP_create_V(); vec_tmp[0] = QDP_create_V(); vec_tmp[1] = QDP_create_V(); for(i=0;i<=MAX_PATH_LENGTH;i++){ oprod_along_path[i] = QDP_create_M(); } // clear force accumulators for(dir=XUP;dir<=TUP;dir++) QDP_M_eq_zero(force_accum[dir], QDP_all); for(dir=XUP;dir<=TUP;dir++){ //AB loop on directions, path table is not needed k=0; // which vec_tmp we are using (0 or 1) QDP_V_eq_V(tsrc[k], x[0], QDP_all); QDP_V_eq_sV(vec_tmp[k], tsrc[k], fnshift(OPP_DIR(dir)), fndir(OPP_DIR(dir)), QDP_all); QDP_M_eq_zero(oprod_along_path[0], QDP_all); for(term=0;term<nterms;term++){ if(term<nterms-1) { QDP_V_eq_V(tsrc[1-k], x[term+1], QDP_all); QDP_V_eq_sV(vec_tmp[1-k], tsrc[1-k], fnshift(OPP_DIR(dir)), fndir(OPP_DIR(dir)), QDP_all); } //QDP_M_eq_V_times_Va(tmat, x[term], vec_tmp[k], QDP_all); QDP_M_eq_V_times_Va(tmat, tsrc[k], vec_tmp[k], QDP_all); nflops += 54; QDP_discard_V(vec_tmp[k]); QDP_M_peq_r_times_M(oprod_along_path[0], &residues[term], tmat, QDP_all); nflops += 36; k=1-k; // swap 0 and 1 } // end loop over terms in rational function expansion link_gather_connection_qdp(oprod_along_path[1], oprod_along_path[0], tmat, dir ); coeff = 1.; QDP_M_peq_r_times_M(force_accum[dir],&coeff,oprod_along_path[1],QDP_all); nflops += 36; } // end of loop on directions // // *** Naik part *** / // clear force accumulators for(dir=XUP;dir<=TUP;dir++) QDP_M_eq_zero(force_accum_naik[dir], QDP_all); for(dir=XUP;dir<=TUP;dir++){ //AB loop on directions, path table is not needed k=0; // which vec_tmp we are using (0 or 1) QDP_V_eq_V(tsrc[k], x[0], QDP_all); QDP_V_eq_sV(vec_tmp[k], tsrc[k], fnshift(OPP_3_DIR( DIR3(dir) )), fndir(OPP_3_DIR( DIR3(dir) )), QDP_all); QDP_M_eq_zero(oprod_along_path[0], QDP_all); for(term=0;term<nterms;term++){ if(term<nterms-1) { QDP_V_eq_V(tsrc[1-k], x[term+1], QDP_all); QDP_V_eq_sV(vec_tmp[1-k], tsrc[1-k], fnshift(OPP_3_DIR( DIR3(dir) )), fndir(OPP_3_DIR( DIR3(dir) )), QDP_all); } //QDP_M_eq_V_times_Va(tmat, x[term], vec_tmp[k], QDP_all); QDP_M_eq_V_times_Va(tmat, tsrc[k], vec_tmp[k], QDP_all); nflops += 54; QDP_discard_V(vec_tmp[k]); QDP_M_peq_r_times_M(oprod_along_path[0], &residues[term], tmat, QDP_all); nflops += 36; k=1-k; // swap 0 and 1 } // end loop over terms in rational function expansion link_gather_connection_qdp(oprod_along_path[1], oprod_along_path[0], tmat, DIR3(dir) ); coeff = 1; // fermion_eps is outside this routine in "wrapper" routine QDP_M_peq_r_times_M(force_accum_naik[dir],&coeff, oprod_along_path[1],QDP_all); nflops += 36; } // end of loop on directions QDP_destroy_V( tsrc[0] ); QDP_destroy_V( tsrc[1] ); QDP_destroy_V( vec_tmp[0] ); QDP_destroy_V( vec_tmp[1] ); QDP_destroy_M( mat_tmp0 ); QDP_destroy_M( tmat ); for(i=0;i<=MAX_PATH_LENGTH;i++){ QDP_destroy_M( oprod_along_path[i] ); } info->final_flop = ((double)nflops)*QDP_sites_on_node; return; } //hisq_force_multi_smearing0_fnmat