Beispiel #1
0
/* Special dslash for use by congrad.  Uses restart_gather_field() when
  possible. Next to last argument is an array of message tags, to be set
  if this is the first use, otherwise reused. If start=1,use
  start_gather_field, otherwise use restart_gather_field. 
  The calling program must clean up the gathers and temps! */
void dslash_fn_field_special(su3_vector *src, su3_vector *dest,
			     int parity, msg_tag **tag, int start,
			     fn_links_t *fn){
  register int i;
  register site *s;
  register int dir,otherparity=0;
  register su3_matrix *fat4;
  su3_matrix *t_fatlink;
#ifndef NO_LONG_LINKS
  register su3_matrix *long4;
  su3_matrix *t_longlink;
#endif
  
  /* allocate temporary work space only if not already allocated */
  if(temp_not_allocated)
    {
      for( dir=XUP; dir<=TUP; dir++ ){
	temp[dir]  =(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
	temp[dir+4]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
      }
      temp[8]=(su3_vector *)malloc(sites_on_node*sizeof(su3_vector));
      temp_not_allocated = 0 ;
    }
  
  /* load fatlinks and longlinks */
  if(fn == NULL){
    printf("dslash_fn_field_special: invalid fn links!\n");
    terminate(1);
  }
#ifndef NO_LONG_LINKS
  t_longlink = get_lnglinks(fn);
#endif
  t_fatlink = get_fatlinks(fn);

  switch(parity)
    {
    case EVEN:	otherparity=ODD; break;
    case ODD:	otherparity=EVEN; break;
    case EVENANDODD:	otherparity=EVENANDODD; break;
    }
  
  /* Start gathers from positive directions */
  /* And start the 3-step gather too */
  for( dir=XUP; dir<=TUP; dir++ ){
    if(start==1)
      {
	tag[dir] = start_gather_field( src, sizeof(su3_vector), 
					   dir, parity,gen_pt[dir] );
#ifndef NO_LONG_LINKS
	tag[DIR3(dir)] = start_gather_field(src, sizeof(su3_vector),
						DIR3(dir),parity, 
						gen_pt[DIR3(dir)] );
#endif
      }
    else
      {
	restart_gather_field( src, sizeof(su3_vector), 
				  dir, parity,gen_pt[dir], tag[dir]);
#ifndef NO_LONG_LINKS
	restart_gather_field(src, sizeof(su3_vector), DIR3(dir), parity, 
				 gen_pt[DIR3(dir)], tag[DIR3(dir)]);
#endif
      }
  }
  
  /* Multiply by adjoint matrix at other sites */
  /* Use fat link for single link transport */
  FORSOMEPARITYDOMAIN_OMP( i, s, otherparity, private(fat4,long4) ){
    //NOPRE if( i < loopend-FETCH_UP ){
       //NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
       //NOPRE prefetch_V(&(src[i+FETCH_UP]));
       //NOPRE prefetch_4MVVVV( 
		       //NOPRE fat4,
		       //NOPRE &(temp[0][i+FETCH_UP]),
		       //NOPRE &(temp[1][i+FETCH_UP]),
		       //NOPRE &(temp[2][i+FETCH_UP]),
		       //NOPRE &(temp[3][i+FETCH_UP]) );
#ifndef NO_LONG_LINKS
       //NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]);
       //NOPRE prefetch_4MVVVV( 
		       //NOPRE long4,
		       //NOPRE &(temp[4][i+FETCH_UP]),
		       //NOPRE &(temp[5][i+FETCH_UP]),
		       //NOPRE &(temp[6][i+FETCH_UP]),
		       //NOPRE &(temp[7][i+FETCH_UP]) );
#endif
    //NOPRE }

    fat4 = &(t_fatlink[4*i]);
    mult_adj_su3_mat_4vec( fat4, &(src[i]), &(temp[0][i]),
			   &(temp[1][i]), &(temp[2][i]), &(temp[3][i]) );
#ifndef NO_LONG_LINKS
    /* multiply by 3-link matrices too */
    long4 = &(t_longlink[4*i]);
    mult_adj_su3_mat_4vec( long4, &(src[i]),&(temp[4][i]),
			   &(temp[5][i]), &(temp[6][i]), &(temp[7][i]) );
#endif
  } END_LOOP_OMP
      
  /* Start gathers from negative directions */
  for( dir=XUP; dir <= TUP; dir++){
    if (start==1) tag[OPP_DIR(dir)] = start_gather_field( temp[dir],
	  sizeof(su3_vector), OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)] );
    else restart_gather_field( temp[dir], sizeof(su3_vector), 
	   OPP_DIR( dir), parity, gen_pt[OPP_DIR(dir)], tag[OPP_DIR(dir)] );
  }

  /* Start 3-neighbour gathers from negative directions */
  for( dir=X3UP; dir <= T3UP; dir++){
    if (start==1) tag[OPP_3_DIR(dir)]=start_gather_field(
		 temp[INDEX_3RD(dir)+4], sizeof(su3_vector), 
		 OPP_3_DIR( dir), parity, gen_pt[OPP_3_DIR(dir)] );
    else restart_gather_field(temp[INDEX_3RD(dir)+4], 
	      sizeof(su3_vector), OPP_3_DIR( dir),parity, 
	      gen_pt[OPP_3_DIR(dir)], tag[OPP_3_DIR(dir)] );
  }

  /* Wait gathers from positive directions, multiply by matrix and
     accumulate */
  /* wait for the 3-neighbours from positive directions, multiply */
  for(dir=XUP; dir<=TUP; dir++){
    wait_gather(tag[dir]);
#ifndef NO_LONG_LINKS
    wait_gather(tag[DIR3(dir)]);
#endif
  }
  
  FORSOMEPARITYDOMAIN_OMP(i,s,parity, private(fat4,long4) ){
    //NOPRE if( i < loopend-FETCH_UP ){
      //NOPRE fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
      //NOPRE prefetch_4MVVVV( 
		      //NOPRE fat4,
		      //NOPRE (su3_vector *)gen_pt[XUP][i+FETCH_UP],
		      //NOPRE (su3_vector *)gen_pt[YUP][i+FETCH_UP],
		      //NOPRE (su3_vector *)gen_pt[ZUP][i+FETCH_UP],
		      //NOPRE (su3_vector *)gen_pt[TUP][i+FETCH_UP] );
      //NOPRE prefetch_VVVV( 
		    //NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
		    //NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
		    //NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
		    //NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
#ifndef NO_LONG_LINKS
      //NOPRE long4 = &(t_longlink[4*(i+FETCH_UP)]);
      //NOPRE prefetch_4MVVVV( 
		      //NOPRE long4,
		      //NOPRE (su3_vector *)gen_pt[X3UP][i+FETCH_UP],
		      //NOPRE (su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
		      //NOPRE (su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
		      //NOPRE (su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
      //NOPRE prefetch_VVVV( 
		    //NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
		    //NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
		    //NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
		    //NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
#endif
    //NOPRE }
    
    fat4 = &(t_fatlink[4*i]);
    mult_su3_mat_vec_sum_4dir( fat4,
	       (su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
	       (su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
	       &(dest[i]) );
    
#ifndef NO_LONG_LINKS
    long4 = &(t_longlink[4*i]);
    mult_su3_mat_vec_sum_4dir( long4,
	    (su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
	    (su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
	    &(temp[8][i]));
#endif
  } END_LOOP_OMP
   
  /* Wait gathers from negative directions, accumulate (negative) */
  /* and the same for the negative 3-rd neighbours */
  for(dir=XUP; dir<=TUP; dir++){
    wait_gather(tag[OPP_DIR(dir)]);
  }
  for(dir=X3UP; dir<=T3UP; dir++){
    wait_gather(tag[OPP_3_DIR(dir)]);
  }
  
  FORSOMEPARITYDOMAIN_OMP(i,s,parity, ){
    //NOPRE if( i < loopend-FETCH_UP ){
      //NOPRE prefetch_VVVVV( 
		     //NOPRE &(dest[i+FETCH_UP]),
		     //NOPRE (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
		     //NOPRE (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
		     //NOPRE (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
		     //NOPRE (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
      //NOPRE prefetch_VVVVV( 
		     //NOPRE &(temp[8][i+FETCH_UP]), 
		     //NOPRE (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
		     //NOPRE (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
		     //NOPRE (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
		     //NOPRE (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
    //NOPRE }
    
    sub_four_su3_vecs( &(dest[i]),
		       (su3_vector *)(gen_pt[XDOWN][i]),
		       (su3_vector *)(gen_pt[YDOWN][i]),
		       (su3_vector *)(gen_pt[ZDOWN][i]),
		       (su3_vector *)(gen_pt[TDOWN][i]) );
    sub_four_su3_vecs( &(temp[8][i]), 
		       (su3_vector *)(gen_pt[X3DOWN][i]),
		       (su3_vector *)(gen_pt[Y3DOWN][i]),
		       (su3_vector *)(gen_pt[Z3DOWN][i]),
		       (su3_vector *)(gen_pt[T3DOWN][i]) );
    /* Now need to add these things together */
    add_su3_vector(&(dest[i]), &(temp[8][i]),&(dest[i]));
  } END_LOOP_OMP
      
}
Beispiel #2
0
/* Special dslash_site for use by congrad.  Uses restart_gather_site() when
  possible. Third to last argument is an array of message tags, to be set
  if this is the first use, otherwise reused. If start=1,use
  start_gather_site, otherwise use restart_gather_site. 
  The calling program must clean up the gathers! */
void dslash_fn_site_special( field_offset src, field_offset dest,
			     int parity, msg_tag **tag, int start,
			     fn_links_t *fn){
  register int i;
  register site *s;
  register int dir,otherparity=0;
  register su3_matrix *fat4;
  su3_matrix *t_fatlink;
#ifndef NO_LONG_LINKS
  register su3_matrix *long4;
  su3_matrix *t_longlink;
  su3_vector *templongvec, *templongv1;
#endif
  su3_vector *tempvec;
  char myname[] = "dslash_fn_site_special";
  
  if(fn == NULL){
    printf("dslash_fn_site_special: invalid fn links!\n");
    terminate(1);
  }
#ifndef NO_LONG_LINKS
  t_longlink = get_lnglinks(fn);
#endif
  t_fatlink = get_fatlinks(fn);

  tempvec = (su3_vector *) malloc(sizeof(su3_vector)*4*sites_on_node);
  if(tempvec == NULL){
    printf("%s(%d)No room for temporary\n",myname, this_node);
    terminate(1);
  }
  
#ifndef NO_LONG_LINKS
  templongvec = (su3_vector *) malloc(sizeof(su3_vector)*4*sites_on_node);
  if(templongvec == NULL){
    printf("%s(%d)No room for temporary\n",myname, this_node);
    terminate(1);
  }
  
  templongv1 = create_v_field();
#endif
  
  switch(parity){
  case EVEN:	otherparity=ODD; break;
  case ODD:	otherparity=EVEN; break;
  case EVENANDODD:	otherparity=EVENANDODD; break;
  }
  
  /* Start gathers from positive directions */
  for(dir=XUP; dir<=TUP; dir++){
    if(start==1) tag[dir] = start_gather_site( src, sizeof(su3_vector),
					       dir, parity, gen_pt[dir] );
    else restart_gather_site( src, sizeof(su3_vector),
			      dir, parity, gen_pt[dir] , tag[dir] );
  }
  
  /* and start the 3rd neighbor gather */
  for(dir=X3UP; dir<=T3UP; dir++){
    if(start==1) tag[dir] = start_gather_site( src, sizeof(su3_vector),
					       dir, parity, gen_pt[dir] );
    else restart_gather_site( src, sizeof(su3_vector),
			      dir, parity, gen_pt[dir] , tag[dir] ); 
  }
  
  /* Multiply by adjoint matrix at other sites */
  FORSOMEPARITYDOMAIN_OMP(i,s,otherparity,private(fat4,long4)){
    if( i < loopend-FETCH_UP ){
      fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
      prefetch_4MV4V( 
		     fat4,
		     (su3_vector *)F_PT(s+FETCH_UP,src),
		     tempvec+4*i+FETCH_UP );
#ifndef NO_LONG_LINKS
      long4 = &(t_longlink[4*(i+FETCH_UP)]);
      prefetch_4MV4V(
		     long4,
		     (su3_vector *)F_PT(s+FETCH_UP,src),
		     templongvec+4*i+FETCH_UP );
#endif
    }
    
    fat4 = &(t_fatlink[4*i]);
#ifndef NO_LONG_LINKS
    long4 = &(t_longlink[4*i]);
#endif
    mult_adj_su3_mat_vec_4dir( fat4,
			       (su3_vector *)F_PT(s,src), (tempvec+4*i) );
    /* multiply by 3-link matrices too */
#ifndef NO_LONG_LINKS
    mult_adj_su3_mat_vec_4dir( long4,
			       (su3_vector *)F_PT(s,src), (templongvec+4*i) );
#endif
  } END_LOOP_OMP
      
  /* Start gathers from negative directions */
  for( dir=XUP; dir <= TUP; dir++){
    if (start==1){
      /* We need the strided gather so we can pick off one of a
	 group of four vectors in tempvec */
      tag[OPP_DIR(dir)] = 
	declare_strided_gather( (char *)(tempvec+dir), 4*sizeof(su3_vector), 
				sizeof(su3_vector), OPP_DIR( dir), parity, 
				gen_pt[OPP_DIR(dir)] );
      prepare_gather(tag[OPP_DIR(dir)]);
      do_gather(tag[OPP_DIR(dir)]);
    }	else {
      do_gather(tag[OPP_DIR(dir)]);
    }
  }
  
#ifndef NO_LONG_LINKS
  /* and 3rd neighbours */
  for( dir=X3UP; dir <= T3UP; dir++){
    /**printf("dslash_fn_site_special: down gathers, start=%d\n",start);**/
    if (start==1){
      tag[OPP_3_DIR(dir)] = 
	declare_strided_gather( (char *)(templongvec+INDEX_3RD(dir)), 
				4*sizeof(su3_vector), 
				sizeof(su3_vector), OPP_3_DIR(dir), 
				parity, gen_pt[OPP_3_DIR(dir)] );
      prepare_gather(tag[OPP_3_DIR(dir)]);
      do_gather(tag[OPP_3_DIR(dir)]);
    }	else {
      do_gather(tag[OPP_3_DIR(dir)]);
    }
  }
#endif
  
  /* Wait gathers from positive directions, multiply by matrix and
     accumulate */
  for(dir=XUP; dir<=TUP; dir++){
    wait_gather(tag[dir]);
  }
  
  /* wait for the 3-neighbours from positive directions, multiply */
  for(dir=X3UP; dir<=T3UP; dir++){
    wait_gather(tag[dir]);
  }
  FORSOMEPARITYDOMAIN_OMP(i,s,parity, private(fat4,long4) ){
    if( i < loopend-FETCH_UP ){
      fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
      prefetch_4MVVVV( 
		      fat4,
		      (su3_vector *)gen_pt[XUP][i+FETCH_UP],
		      (su3_vector *)gen_pt[YUP][i+FETCH_UP],
		      (su3_vector *)gen_pt[ZUP][i+FETCH_UP],
		      (su3_vector *)gen_pt[TUP][i+FETCH_UP] );
#ifndef NO_LONG_LINKS
      long4 = &(t_longlink[4*(i+FETCH_UP)]);
      prefetch_VV(
		  (su3_vector *)F_PT(s+FETCH_UP,dest),
		  templongv1+i+FETCH_UP);
      prefetch_4MVVVV( 
		      long4,
		      (su3_vector *)gen_pt[X3UP][i+FETCH_UP],
		      (su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
		      (su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
		      (su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
#endif
    }
    fat4 = &(t_fatlink[4*i]);
    mult_su3_mat_vec_sum_4dir( fat4,
	       (su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
	       (su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
	       (su3_vector *)F_PT(s,dest));
#ifndef NO_LONG_LINKS
    long4 = &(t_longlink[4*i]);
    mult_su3_mat_vec_sum_4dir( long4,
	       (su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
	       (su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
	       templongv1+i);
#endif
  } END_LOOP_OMP
      
  /* Wait gathers from negative directions, accumulate (negative) */
  for(dir=XUP; dir<=TUP; dir++){
    wait_gather(tag[OPP_DIR(dir)]);
  } 
  
  /* and the same for the negative 3-rd neighbours */
  
  for(dir=X3UP; dir<=T3UP; dir++){
    wait_gather(tag[OPP_3_DIR(dir)]);
  }
  
  FORSOMEPARITYDOMAIN_OMP(i,s,parity, ){
    if( i < loopend-FETCH_UP ){
#ifndef NO_LONG_LINKS
      prefetch_VV(
		  (su3_vector *)F_PT(s+FETCH_UP,dest),
		  templongv1+i+FETCH_UP);
#endif
      prefetch_VVVV( 
		    (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
		    (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
		    (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
		    (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
      prefetch_VVVV( 
		    (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
		    (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
		    (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
		    (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
    }
    sub_four_su3_vecs( (su3_vector *)F_PT(s,dest),
		       (su3_vector *)(gen_pt[XDOWN][i]),
		       (su3_vector *)(gen_pt[YDOWN][i]),
		       (su3_vector *)(gen_pt[ZDOWN][i]),
		       (su3_vector *)(gen_pt[TDOWN][i]) );
#ifndef NO_LONG_LINKS
    sub_four_su3_vecs( templongv1+i,
		       (su3_vector *)(gen_pt[X3DOWN][i]),
		       (su3_vector *)(gen_pt[Y3DOWN][i]),
		       (su3_vector *)(gen_pt[Z3DOWN][i]),
		       (su3_vector *)(gen_pt[T3DOWN][i]) );
    /*** Now need to add these things together ***/
    add_su3_vector((su3_vector *)F_PT(s,dest), templongv1+i,
		   (su3_vector *)F_PT(s,dest));
#endif
  } END_LOOP_OMP
      
}
Beispiel #3
0
/* D_slash routine - sets dest. on each site equal to sum of
   sources parallel transported to site, with minus sign for transport
   from negative directions.  Use "fatlinks" for one link transport,
   "longlinks" for three link transport. */
void dslash_fn( field_offset src, field_offset dest, int parity ) {
   register int i;
   register site *s;
   register int dir,otherparity;
   register su3_matrix *fat4, *long4;
   msg_tag *tag[16];

    if(!valid_longlinks)load_longlinks();
    if(!valid_fatlinks)load_fatlinks();
    switch(parity){
	case EVEN:	otherparity=ODD; break;
	case ODD:	otherparity=EVEN; break;
	case EVENANDODD:	otherparity=EVENANDODD; break;
    }

    /* Start gathers from positive directions */
    /* And start the 3-step gather too */
    for( dir=XUP; dir<=TUP; dir++ ){
	tag[dir] = start_gather( src, sizeof(su3_vector), dir, parity,
	    gen_pt[dir] );
	tag[DIR3(dir)] = start_gather( src, sizeof(su3_vector), DIR3(dir),
	    parity, gen_pt[DIR3(dir)] );
    }

    /* Multiply by adjoint matrix at other sites */
    /* Use fat link for single link transport */
    FORSOMEPARITY( i, s, otherparity ){
      if( i < loopend-FETCH_UP ){
#ifdef DSLASH_TMP_LINKS
	fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
	long4 = &(t_longlink[4*(i+FETCH_UP)]);
#else
	fat4 = (s+FETCH_UP)->fatlink;
	long4 = (s+FETCH_UP)->longlink;
#endif
	prefetch_4MV4V( 
		       fat4,
		       (su3_vector *)F_PT(s+FETCH_UP,src),
		       (s+FETCH_UP)->tempvec );
	prefetch_4MV4V(
		       long4,
		       (su3_vector *)F_PT(s+FETCH_UP,src),
		       (s+FETCH_UP)->templongvec );
      }

#ifdef DSLASH_TMP_LINKS
      fat4 = &(t_fatlink[4*i]);
      long4 = &(t_longlink[4*i]);
#else
      fat4 = s->fatlink;
      long4 = s->longlink;
#endif
	mult_adj_su3_mat_vec_4dir( fat4,
	    (su3_vector *)F_PT(s,src), s->tempvec );
	/* multiply by 3-link matrices too */
	mult_adj_su3_mat_vec_4dir( long4,
	    (su3_vector *)F_PT(s,src), s->templongvec );
    } END_LOOP

    /* Start gathers from negative directions */
    for( dir=XUP; dir <= TUP; dir++){
	tag[OPP_DIR(dir)] = start_gather( F_OFFSET(tempvec[dir]),
	    sizeof(su3_vector), OPP_DIR( dir), parity,
	    gen_pt[OPP_DIR(dir)] );
    }

    /* Start 3-neighbour gathers from negative directions */
    for( dir=X3UP; dir <= T3UP; dir++){
	tag[OPP_3_DIR(dir)] 
           = start_gather( F_OFFSET(templongvec[INDEX_3RD(dir)]),
			   sizeof(su3_vector), OPP_3_DIR( dir), parity,
			   gen_pt[OPP_3_DIR(dir)] );
    }

    /* Wait gathers from positive directions, multiply by matrix and
	accumulate */
    /* wait for the 3-neighbours from positive directions, multiply */
    for(dir=XUP; dir<=TUP; dir++){
	wait_gather(tag[dir]);
	wait_gather(tag[DIR3(dir)]);
    }
    /* Wait gathers from negative directions, accumulate (negative) */
    /* and the same for the negative 3-rd neighbours */
    for(dir=XUP; dir<=TUP; dir++){
	wait_gather(tag[OPP_DIR(dir)]);
    }
    for(dir=X3UP; dir<=T3UP; dir++){
	wait_gather(tag[OPP_3_DIR(dir)]);
    }


    FORSOMEPARITY(i,s,parity){
#ifdef DSLASH_TMP_LINKS
      fat4 = &(t_fatlink[4*i]);
      long4 = &(t_longlink[4*i]);
#else
      fat4 = s->fatlink;
      long4 = s->longlink;
#endif
      mult_su3_mat_vec_sum_4dir( fat4,
	    (su3_vector *)gen_pt[XUP][i], (su3_vector *)gen_pt[YUP][i],
	    (su3_vector *)gen_pt[ZUP][i], (su3_vector *)gen_pt[TUP][i],
	    (su3_vector *)F_PT(s,dest));

      mult_su3_mat_vec_sum_4dir( long4,
	    (su3_vector *)gen_pt[X3UP][i], (su3_vector *)gen_pt[Y3UP][i],
	    (su3_vector *)gen_pt[Z3UP][i], (su3_vector *)gen_pt[T3UP][i],
	    (su3_vector *) &(s->templongv1));

      if( i < loopend-FETCH_UP ){
#ifdef DSLASH_TMP_LINKS
	fat4 = &(t_fatlink[4*(i+FETCH_UP)]);
	long4 = &(t_longlink[4*(i+FETCH_UP)]);
#else
	fat4 = (s+FETCH_UP)->fatlink;
	long4 = (s+FETCH_UP)->longlink;
#endif
	prefetch_4MVVVV( 
              fat4,
	      (su3_vector *)gen_pt[XUP][i+FETCH_UP],
              (su3_vector *)gen_pt[YUP][i+FETCH_UP],
              (su3_vector *)gen_pt[ZUP][i+FETCH_UP],
              (su3_vector *)gen_pt[TUP][i+FETCH_UP] );
	prefetch_4MVVVV( 
              long4,
              (su3_vector *)gen_pt[X3UP][i+FETCH_UP],
              (su3_vector *)gen_pt[Y3UP][i+FETCH_UP],
              (su3_vector *)gen_pt[Z3UP][i+FETCH_UP],
              (su3_vector *)gen_pt[T3UP][i+FETCH_UP] );
	prefetch_VVVV( 
              (su3_vector *)gen_pt[XDOWN][i+FETCH_UP],
              (su3_vector *)gen_pt[YDOWN][i+FETCH_UP],
              (su3_vector *)gen_pt[ZDOWN][i+FETCH_UP],
              (su3_vector *)gen_pt[TDOWN][i+FETCH_UP] );
	prefetch_VVVV( 
              (su3_vector *)gen_pt[X3DOWN][i+FETCH_UP],
              (su3_vector *)gen_pt[Y3DOWN][i+FETCH_UP],
              (su3_vector *)gen_pt[Z3DOWN][i+FETCH_UP],
              (su3_vector *)gen_pt[T3DOWN][i+FETCH_UP] );
        }

        sub_four_su3_vecs( (su3_vector *)F_PT(s,dest),
	    (su3_vector *)(gen_pt[XDOWN][i]),
	    (su3_vector *)(gen_pt[YDOWN][i]),
	    (su3_vector *)(gen_pt[ZDOWN][i]),
	    (su3_vector *)(gen_pt[TDOWN][i]) );
        sub_four_su3_vecs( &(s->templongv1), 
	    (su3_vector *)(gen_pt[X3DOWN][i]),
	    (su3_vector *)(gen_pt[Y3DOWN][i]),
	    (su3_vector *)(gen_pt[Z3DOWN][i]),
	    (su3_vector *)(gen_pt[T3DOWN][i]) );
        /* Now need to add these things together */
        add_su3_vector((su3_vector *)F_PT(s,dest), & (s->templongv1),
			           (su3_vector *)F_PT(s,dest));
    } END_LOOP