Example #1
0
int main(int argc,char *argv[])
{
  int meascount;
  int prompt;
  Real avm_iters,avs_iters;
  
  double starttime,endtime,dclock();
  double dtime;
  
  int MinCG,MaxCG;
  Real RsdCG;
  
  register int i;
  register site *s;
  
  int spinindex,spin,color,j,k,t,t_off;
  int kh,kl;
  int nr_fb;
  char nr_fb_label[3][2] = { "0", "F", "B" };
  int flag;
  int kprop;
  int num_prop;
  Real space_vol;

  int status;

  propagator hdibar_prop[MAX_KAP][MAX_KAP][HDIPROPS];
  propagator nrbar_prop[MAX_KAP][MAX_KAP][NRPROPS];
  
  char scratch_file[MAX_KAP][MAXFILENAME];
  
  Real norm_fac[10];
  static char *mes_kind[10] = {"PION","PS505","PS055","PS0505",
			       "RHO33","RHO0303","SCALAR","SCALA0","PV35","B12"};

  complex *pmes_prop[MAX_KAP][MAX_KAP][10];
  int pmes_prop_done[MAX_KAP][MAX_KAP];

  w_prop_file *fp_in_w[MAX_KAP];  /* For reading binary propagator files */
  w_prop_file *fp_out_w[MAX_KAP]; /* For writing binary propagator files */
  w_prop_file *fp_scr[MAX_KAP];
  
  initialize_machine(&argc,&argv);
#ifdef HAVE_QDP
  QDP_initialize(&argc, &argv);
#endif
  /* Remap standard I/O */
  if(remap_stdio_from_args(argc, argv) == 1)terminate(1);
  
  g_sync();
  /* set up */
  prompt = setup_H_cl();
  

  /* loop over input sets */
  
  while( readin(prompt) == 0)
    {

      MaxCG = niter;
      starttime=dclock();

      avm_iters=0.0;
      meascount=0;
      
      /* Allocate space for relativistic meson propagator */
      for(num_prop=0;num_prop<10;num_prop++)
	for(i=0;i<num_kap;i++)for(j=0;j<=i;j++){
	  pmes_prop[i][j][num_prop] = (complex *)malloc(nt*sizeof(complex));
	  for(t=0;t<nt;t++){
	    pmes_prop[i][j][num_prop][t] = cmplx(0.0,0.0); 
	  }
	  pmes_prop_done[i][j] = 0;
	}

      /* Allocate space for non relativistic baryon propagators */
      for(kprop=0;kprop<NRPROPS;kprop++)
	for(i=0;i<num_kap;i++)for(j=0;j<num_kap;j++){
	  nrbar_prop[i][j][kprop].c
	    = (complex *)malloc(nt*sizeof(complex));
	  if(nrbar_prop[i][j][kprop].c == NULL)
	    {
	      printf("control_H_cl: Can't malloc nrbar prop %d %d %d\n",
		     i,j,kprop);
	      terminate(1);
	    }
	  for(t=0;t<nt;t++)nrbar_prop[i][j][kprop].c[t] 
	    = cmplx(0.0,0.0); 
	  nrbar_prop[i][j][kprop].label
	    = (char *)malloc(10*sizeof(char));
	  if(nrbar_prop[i][j][kprop].c == NULL)
	    {
	      printf("control_H_cl: Can't malloc nrbar prop label %d %d %d\n",
		     i,j,kprop);
	      terminate(1);
	    }
	}
      
      /* Allocate space for H-dibaryon channel propagators */
      for(kprop=0;kprop<HDIPROPS;kprop++)
	for(kh=0;kh<num_kap_heavy;kh++)for(kl=0;kl<num_kap_light;kl++){
	  /* kappa indexing scheme is consistent with baryon propagator
	     even though we compute only the propagators with
	     one heavy (s) quark and two light (u,d) quarks */
	  i = kh; j = kl + num_kap_heavy;
	  hdibar_prop[i][j][kprop].c
	    = (complex *)malloc(nt*sizeof(complex));
	  if(hdibar_prop[i][j][kprop].c == NULL)
	    {
	      printf("control_H_cl: Can't malloc baryon prop %d %d %d\n",
		     i,j,kprop);
	      terminate(1);
	    }
	  for(t=0;t<nt;t++)hdibar_prop[i][j][kprop].c[t] 
	    = cmplx(0.0,0.0); 
	  hdibar_prop[i][j][kprop].label
	    = (char *)malloc(10*sizeof(char));
	  if(hdibar_prop[i][j][kprop].label == NULL)
	    {
	      printf("control_H_cl: Can't malloc baryon prop label %d %d %d\n",
		     i,j,kprop);
	      terminate(1);
	    }
	}
      
      if( fixflag == COULOMB_GAUGE_FIX)
	{
	  if(this_node == 0) 
	    printf("Fixing to Coulomb gauge\n");
	  STARTIOTIME;
	  gaugefix(TUP,(Real)1.5,500,GAUGE_FIX_TOL);
	  STOPIOTIME("gauge fix");
	  invalidate_this_clov(gen_clov);
	}
      else
	if(this_node == 0)printf("COULOMB GAUGE FIXING SKIPPED.\n");
      
      /* save lattice if requested */
      if( saveflag != FORGET ){
	/* Note: beta, kappa are kept only for save_old_binary */
	STARTIOTIME;
	savelat_p = save_lattice( saveflag, savefile, stringLFN );
	STOPIOTIME("save lattice");
      }

      if(this_node==0)printf("END OF HEADER\n");
      
      /* Loop over all kappas to compute and store quark propagator */
      for(k=0;k<num_kap;k++){
	
	kappa = kap[k];
	source_r0=wqs[k].r0;
	RsdCG=resid[k];
	if(this_node==0)printf("Kappa=%e r0=%e residue=%e\n",
			       (double)kappa,(double)source_r0,(double)RsdCG);
	
	/* open file for kth wilson propagator */
	
	fp_in_w[k]  = r_open_wprop(startflag_w[k], startfile_w[k]);
	fp_out_w[k] = w_open_wprop(saveflag_w[k],  savefile_w[k],
				   wqs[k].type);
	
	/* Open scratch file and write header */
	sprintf(scratch_file[k],"%s_%02d",scratchstem_w,k);
	if(scratchflag == SAVE_CHECKPOINT)
	  {
	    fp_scr[k] = w_checkpoint_w_i(scratch_file[k]);
	    /* Close, temporarily */
	    w_checkpoint_w_c(fp_scr[k]);
	  }
	else
	  /* If serial, write header and leave it open */
	  fp_scr[k] = w_serial_w_i(scratch_file[k]);
	
	/* Loop over source colors */
	for(color=0;color<3;color++){
	  
	  for(spinindex=0;spinindex<n_spins;spinindex++){
	    spin = spins[spinindex];
	    
	    meascount ++;
	    if(this_node==0)printf("color=%d spin=%d\n",color,spin);

	    if(startflag_w[k] == CONTINUE)
	      {
		if(k == 0)
		  {
		    node0_printf("Can not continue propagator here! Zeroing it instead\n");
		    startflag_w[k] = FRESH;
		  }
		else
		  {
		    FORALLSITES(i,s)
		      copy_wvec(&(s->quark_propagator.c[color].d[spin]),
				&(s->psi));
		  }
	      }

	    /* Saves one multiplication by zero in cgilu */
	    if(startflag_w[k] == FRESH)flag = 0;
	    else 
	      flag = 1;      
	    
	    /* load psi if requested */
#ifdef IOTIME
	    status = reload_wprop_sc_to_site( startflag_w[k], fp_in_w[k], 
			       spin, color, F_OFFSET(psi),1);
#else
	    status = reload_wprop_sc_to_site( startflag_w[k], fp_in_w[k], 
			       spin, color, F_OFFSET(psi),0);
#endif	    
	    if(status != 0)
	      {
		node0_printf("control_H_cl: Recovering from error by resetting initial guess to zero\n");
		reload_wprop_sc_to_site( FRESH, fp_in_w[k], 
			       spin, color, F_OFFSET(psi),0);
		flag = 0;
	      }

	    
	    /* Invert to find propagator */

	    /* Complete the source structure */
	    wqs[k].color = color;
	    wqs[k].spin = spin;

	    /* For clover_info */
	    wqstmp = wqs[k];

	   /* If we are starting afresh, we set a minimum number
	      of iterations */
	   if(startflag_w[k] == FRESH || status != 0)MinCG = nt; 
	   else MinCG = 0;

	    /* Load inversion control structure */
	    qic.prec = PRECISION;
	    qic.min = MinCG;
	    qic.max = MaxCG;
	    qic.nrestart = nrestart;
	    qic.resid = RsdCG;
	    qic.start_flag = flag;
	    
	    /* Load Dirac matrix parameters */
	    dcp.Kappa = kappa;
	    dcp.Clov_c = clov_c;
	    dcp.U0 = u0;

#ifdef BI
	    /* compute the propagator.  Result in psi. */
	    avs_iters 
	      = (Real)wilson_invert_site_wqs(F_OFFSET(chi),F_OFFSET(psi),
					  w_source,&wqs[k],
					  bicgilu_cl_site,&qic,(void *)&dcp);
#else
	    /* compute the propagator.  Result in psi. */
	    avs_iters = 
	      (Real)wilson_invert_site_wqs(F_OFFSET(chi),F_OFFSET(psi),
					w_source,&wqs[k],
					cgilu_cl_site,&qic,(void *)&dcp);
#endif
	    avm_iters += avs_iters;
	    
	    FORALLSITES(i,s)
	      copy_wvec(&(s->psi),
			&(s->quark_propagator.c[color].d[spin]));
	    
	    STARTIOTIME;
	    /* Write psi to scratch disk */
	    if(scratchflag == SAVE_CHECKPOINT)
	      {
		w_checkpoint_w_o(fp_scr[k]);
		w_checkpoint_w(fp_scr[k],spin,color,F_OFFSET(psi));
		w_checkpoint_w_c(fp_scr[k]);
	      }
	    else
	      w_serial_w(fp_scr[k],spin,color,F_OFFSET(psi));
	    STOPIOTIME("do fast quark dump");
	    /* save psi if requested */
#ifdef IOTIME
	    save_wprop_sc_from_site( saveflag_w[k],fp_out_w[k],
			     spin,color,F_OFFSET(psi),1);
#else
	    save_wprop_sc_from_site( saveflag_w[k],fp_out_w[k],
			     spin,color,F_OFFSET(psi),0);
#endif
	  } /* source spins */
	} /* source colors */
	
	/* Close and release scratch file */
	if(scratchflag == SAVE_CHECKPOINT)
	  w_checkpoint_w_f(fp_scr[k]);
	else
	  w_serial_w_f(fp_scr[k]);

	if(this_node==0)printf("Saved binary wilson_vector in file  %s\n",
			       scratch_file[k]);
	
	/* close files for wilson propagators */
	r_close_wprop(startflag_w[k],fp_in_w[k]);
	w_close_wprop(saveflag_w[k],fp_out_w[k]);
	
      } /* kappas */
      
      
      /* Loop over choice forward - backward for NR source and sink */

      for(nr_fb = 1; nr_fb <= 2; nr_fb++)if(nr_fb & nr_forw_back)
	{
	  
	  /* Reset completion flags */
	    for(i=0;i<num_kap;i++)for(j=0;j<num_kap;j++){
	      for(kprop=0;kprop<NRPROPS;kprop++)
		nrbar_prop[i][j][kprop].done = 0;
	      for(kprop=0;kprop<HDIPROPS;kprop++)
		hdibar_prop[i][j][kprop].done = 0;
	    }

	  /* Loop over heavy kappas for the point sink spectrum */
	  for(k=0;k<num_kap_heavy;k++){
	    
	    /* Read the kth heavy kappa propagator from the scratch file */
	    kappa = kappa_heavy = kap[k];
	    if(scratchflag == SAVE_CHECKPOINT)
	      fp_scr[k] = r_parallel_w_i(scratch_file[k]);
	    else
	      fp_scr[k] = r_serial_w_i(scratch_file[k]);
	    
	    STARTIOTIME;
	    for(color=0;color<3;color++) for(spin=0;spin<4;spin++){
	      if(scratchflag == SAVE_CHECKPOINT)
		r_parallel_w(fp_scr[k], spin, color,
			     F_OFFSET(quark_propagator.c[color].d[spin])); 
	      else
		r_serial_w(fp_scr[k], spin, color,
			   F_OFFSET(quark_propagator.c[color].d[spin])); 
	    }
	    STOPIOTIME("to read 12 spin-color combinations");

	    if(scratchflag == SAVE_CHECKPOINT)
	      r_parallel_w_f(fp_scr[k]); 
	    else
	      r_serial_w_f(fp_scr[k]); 
	    
	    /* Convert to NR propagator */
	    
	    STARTPRTIME;
	    nr_propagator(F_OFFSET(quark_propagator),
			  F_OFFSET(nr_prop1), nr_fb);
	    diquarkprop(F_OFFSET(nr_prop1),
			F_OFFSET(diquark_prop1));
	    STOPPRTIME("make nr and diquark");
	    
	    /* Diagonal spectroscopy - not needed */
	    
/**	    w_nrbaryon(F_OFFSET(nr_prop1), F_OFFSET(nr_prop1),
		       F_OFFSET(diquark_prop1), nrbar_prop[k][k]); **/
	    
/**	    w_hdibaryon(F_OFFSET(diquark_prop1),
			F_OFFSET(diquark_prop1), hdibar_prop[k][k]); **/
	    
	    /* Heavy-light spectroscopy */
	    /* Loop over light kappas for the point sink spectrum */
	    for(j=num_kap_heavy;j<num_kap;j++){

	      /* Read the propagator from the scratch file */
	      kappa = kappa_light = kap[j];
	      if(scratchflag == SAVE_CHECKPOINT)
		fp_scr[j] = r_parallel_w_i(scratch_file[j]);
	      else
		fp_scr[j] = r_serial_w_i(scratch_file[j]);
	      
	      STARTIOTIME;
	      for(color=0;color<3;color++) for(spin=0;spin<4;spin++){
		if(scratchflag == SAVE_CHECKPOINT)
		  r_parallel_w(fp_scr[j], spin, color,
			       F_OFFSET(quark_prop2.c[color].d[spin])); 
		else
		  r_serial_w(fp_scr[j], spin, color,
			     F_OFFSET(quark_prop2.c[color].d[spin])); 
	      }
	      STOPIOTIME("do fast quark read");
	      if(scratchflag == SAVE_CHECKPOINT)
		r_parallel_w_f(fp_scr[j]);
	      else
		r_serial_w_f(fp_scr[j]);
	      
	      /* Convert to NR propagator */
	      
	      STARTPRTIME;
	      nr_propagator(F_OFFSET(quark_prop2),
			    F_OFFSET(nr_prop2),nr_fb);
	      diquarkprop(F_OFFSET(nr_prop2),
			  F_OFFSET(diquark_prop2));
	      STOPPRTIME("make nr and diquark propagators");
	      
	      /* Diagonal spectroscopy - baryons only - done if
		 any of them was not previously done */
	    
	      for(kprop=0;kprop<NRPROPS;kprop++)
		{
		  if(nrbar_prop[j][j][kprop].done == 0)
		    {
		      STARTPRTIME;
		      w_nrbaryon(F_OFFSET(nr_prop2), F_OFFSET(nr_prop2),
				 F_OFFSET(diquark_prop2), nrbar_prop[j][j]);
		      STOPPRTIME("do diagonal baryons");
		      break;
		    }
		}
	    
	      /* Heavy-light spectroscopy - baryons and H */

	      /* We don't do baryon heavy-light if the kappa values
		 are the same, since the result is the same as the
		 diagonal light propagator */
	      
	      if(kappa_heavy != kappa_light)
		{
		  /* Relativistic meson propagator: Do only once */		  
		  if(pmes_prop_done[j][k] == 0) {
		    STARTPRTIME;
		    for(color=0;color<3;color++){
		      w_meson_site(F_OFFSET(quark_propagator.c[color]),
			      F_OFFSET(quark_prop2.c[color]), pmes_prop[j][k]);
		    }
		    pmes_prop_done[j][k] = 1;
		    STOPPRTIME("do off-diagonal relativistic meson");
		  }

		  STARTPRTIME;
		  w_nrbaryon(F_OFFSET(nr_prop2),
			     F_OFFSET(nr_prop1),F_OFFSET(diquark_prop1),  
			     nrbar_prop[j][k]);
		  
		  w_nrbaryon(F_OFFSET(nr_prop1),
			     F_OFFSET(nr_prop2),F_OFFSET(diquark_prop2),  
			     nrbar_prop[k][j]);
		  STOPPRTIME("do two sets of hl baryons");
		}
	      
	      /* For H we do only the case prop2 = u (light) index j
		 and prop1 = s (heavy) index k */

	      STARTPRTIME;
	      w_hdibaryon(F_OFFSET(diquark_prop2),
			  F_OFFSET(diquark_prop1), hdibar_prop[k][j]);
	      STOPPRTIME("do one set of hl H dibaryons");
	      
	    } /* light kappas */
	  } /* heavy kappas */

	  /* Stick with same convention as clover_invert/control_cl_hl.c */
	  space_vol = (Real)(nx*ny*nz);
	  for(num_prop=0;num_prop<10;num_prop++) norm_fac[num_prop] = space_vol;
	  norm_fac[4] *= 3.0;
	  norm_fac[5] *= 3.0;
	  norm_fac[8] *= 3.0;
	  norm_fac[9] *= 3.0;

	  /* print relativistic meson propagators */
	  for(num_prop=0;num_prop<10;num_prop++)
	    for(i=0;i<num_kap;i++)
	      for(j=0;j<=i;j++)
		if(pmes_prop_done[i][j] == 1){
		  for(t = 0; t < nt; t++){
		    t_off = (t + source_time)%nt;
		    g_floatsum( &pmes_prop[i][j][num_prop][t_off].real );
		    pmes_prop[i][j][num_prop][t_off].real  /= norm_fac[num_prop];
		    g_floatsum( &pmes_prop[i][j][num_prop][t_off].imag );
		    pmes_prop[i][j][num_prop][t_off].imag  /= norm_fac[num_prop];
		    if(this_node == 0)
		      printf("POINT%s %d %d %d  %e %e\n",
			     mes_kind[num_prop],i,j,t,
			     (double)pmes_prop[i][j][num_prop][t_off].real,
			     (double)pmes_prop[i][j][num_prop][t_off].imag);
		  }
		}

	  /* Once printed, this propagator should be neither
             calculated nor printed again */
	  for(i=0;i<num_kap;i++)
	    for(j=0;j<=i;j++)
	      if(pmes_prop_done[i][j] == 1)
		pmes_prop_done[i][j] = 2;
	  

	  /* print non-relativistic baryon propagators */
	  if(this_node == 0)
	    for(kprop=0;kprop<NRPROPS;kprop++)
	      for(i=0;i<num_kap;i++){
		for(j=0;j<i;j++)
		  if(nrbar_prop[i][j][kprop].done==1){
		    for(t = 0; t < nt; t++){
		      t_off = (t + source_time)%nt;
		      /* Periodic boundary conditions - no wraparound sign */
		      printf("%s_NR%s %d %d %d %d  %e %e\n",
			     nr_fb_label[nr_fb],
			     nrbar_prop[i][j][kprop].label,i,j,j,t,
			     (double)nrbar_prop[i][j][kprop].c[t_off].real,
			     (double)nrbar_prop[i][j][kprop].c[t_off].imag);
		    }
		  }
		
		if(nrbar_prop[i][i][kprop].done==1)
		  for(t = 0; t < nt; t++){
		    t_off = (t + source_time)%nt;
		    printf("%s_NR%s %d %d %d %d  %e %e\n",
			   nr_fb_label[nr_fb],
			   nrbar_prop[i][j][kprop].label,i,i,i,t,
			   (double)nrbar_prop[i][i][kprop].c[t_off].real,
			   (double)nrbar_prop[i][i][kprop].c[t_off].imag);
		  }
		
		for(j=i+1;j<num_kap;j++)
		  if(nrbar_prop[i][j][kprop].done==1)
		    for(t = 0; t < nt; t++){
		      t_off = (t + source_time)%nt;
		      printf("%s_NR%s %d %d %d %d  %e %e\n",
			     nr_fb_label[nr_fb],
			     nrbar_prop[i][j][kprop].label,j,j,i,t,
			     (double)nrbar_prop[i][j][kprop].c[t_off].real,
			     (double)nrbar_prop[i][j][kprop].c[t_off].imag);
		    }
	      }

	  
	  /* print H-dibaryon mixed channel propagators */
	  if(this_node == 0)
	    for(kprop=0;kprop<HDIPROPS;kprop++)
	      for(i=0;i<num_kap;i++){
		for(j=0;j<num_kap;j++)if(hdibar_prop[i][j][kprop].done==1){
		  for(t = 0; t < nt; t++){
		    t_off = (t + source_time)%nt;
		    printf("%s_%s %d %d %d %d %e %e\n",
			   nr_fb_label[nr_fb],
			   hdibar_prop[i][j][kprop].label,i,j,j,t,
			   (double)hdibar_prop[i][j][kprop].c[t_off].real,
			   (double)hdibar_prop[i][j][kprop].c[t_off].imag);
		  }
		}
	      }
	} /* Loop over nr forward - backward */

      /* Cleanup */
      for(kprop=0;kprop<NRPROPS;kprop++)
	for(i=0;i<num_kap;i++)for(j=0;j<num_kap;j++){
	  free(nrbar_prop[i][j][kprop].c);
	  free(nrbar_prop[i][j][kprop].label);
	}
      
      for(kprop=0;kprop<HDIPROPS;kprop++)
	for(kh=0;kh<num_kap_heavy;kh++)for(kl=0;kl<num_kap_light;kl++){
	  i = kh; j = kl + num_kap_heavy;
	  free(hdibar_prop[i][j][kprop].c);
	  free(hdibar_prop[i][j][kprop].label);
	}
      
      if(this_node==0)printf("RUNNING COMPLETED\n");
      if(meascount>0){
	if(this_node==0)printf("total cg iters for measurement= %e\n",
			       (double)avm_iters);
	if(this_node==0)printf("cg iters for measurement= %e\n",
			       (double)avm_iters/(double)meascount);
      }
      
      endtime=dclock();
      if(this_node==0){
	printf("Time = %e seconds\n",(double)(endtime-starttime));
	printf("total_iters = %d\n",total_iters);
      }
      fflush(stdout);
      
    }
    return 0;
} /* control_H_cl */
Example #2
0
int main(int argc, char **argv)
{
  int meascount[MAX_NKAP];
  int prompt, count1, count2;
  Real avm_iters[MAX_NKAP];
  double starttime, endtime;

  int MaxMR, restart_flag;
  Real RsdMR;		/******/

  int spin, color, nk;		/******/
  int max_prop;

  int cl_cg = CL_CG;
  double ssplaq, stplaq;


  FILE *fp_m_out = NULL;  /*** meson IO stuff **/
  int fb_m_out = 0;	   /*** meson IO stuff **/

  w_prop_file *fp_in_w[MAX_NKAP];        /* For propagator files */
  w_prop_file *fp_out_w[MAX_NKAP];       /* For propagator files */
  
  double g_time ; 

  int i ;
  int MinMR;
  
/*** variables required for the static variational code ***/
  int nodata = 0 ;
  complex *meson = NULL;

/****** start of the execution of the code ************/


  initialize_machine(&argc, &argv);

  /* Remap standard I/O */
  if(remap_stdio_from_args(argc, argv) == 1)terminate(1);

  g_sync();

  /* set up */
  prompt = setup_h();

  /**DEBUG***/  
#ifdef DEBUGDEF
  light_quark_pion(0) ;
#endif

    /* loop over input sets */
  while( readin(prompt) == 0)
  {

    if( fixflag == COULOMB_GAUGE_FIX)
    {
      if(this_node == 0) 
	printf("Fixing to Coulomb gauge\n");
      g_time = -dclock();

      gaugefix(TUP,(Real)1.5,500,GAUGE_FIX_TOL);

      g_time += dclock();
      if(this_node==0)printf("Time to gauge fix = %e\n",g_time);
      invalidate_this_clov(gen_clov);
      
    }
    else
      if(this_node == 0)printf("COULOMB GAUGE FIXING SKIPPED.\n");

    /* save lattice if requested */
    if( saveflag != FORGET )
    {
      save_lattice( saveflag, savefile, stringLFN );
    }


    /* call plaquette measuring process */
    d_plaquette(&ssplaq, &stplaq);
    if (this_node == 0)
      printf("START %e %e\n",(double) ssplaq, (double) stplaq);


    /******* set up code for the static variational calculation *****/
    if( nkap == 1 )
    {
      nodata = nt*nosmear*144 ;
      /** reserve memory for the smeared meson correlators on each node ****/
      if( ( meson = (complex *) calloc( (size_t) nodata, sizeof(complex) )  ) == NULL )
      {
	printf("ERROR: could not reserve buffer space for the meson smearing functions\n");
	terminate(1);
      }
      
      /** call a number of set up routines for the static variational code ***/
      setup_vary(meson, nodata);
      
    }  /*** end of set up section for the static-variational calculation ***/

    /***DEBUG check_calc_matrix() ;   ****/  


    starttime=dclock();



    MaxMR = niter;
    RsdMR = (Real) sqrt((double) rsqprop);

    if (this_node == 0)
      printf("Residue=%e\n",(double) RsdMR);
	     

    for (nk = 0; nk < nkap; nk++)
    {
      avm_iters[nk] = 0.0;
      meascount[nk] = 0;
    }
    max_prop = 12;
    count1 = 0;
    count2 = 0;


    for (spin = start_spin; spin < 4; spin++)
    {

      for (color = 0; color < 3; color++)
      {

	count1++;
	if (count1 == 1)
	  color += start_color;

	for (nk = 0; nk < nkap; nk++)
	{

	  count2++;
	  if (count2 == 1)
	    nk += start_kap;


	  kappa = cappa[nk];

	  meascount[nk]++;

	  /* open file for wilson propagators */
	  fp_in_w[nk]  = r_open_wprop(startflag_w[nk], startfile_w[nk]);

	  if ((spin + color) == 0)
	  {
	    /*** first pass of the code  **/
	    fp_out_w[nk] = w_open_wprop(saveflag_w[nk],  savefile_w[nk],
					wqs.type);

	    /* open file for meson output and write the header */
	    if (saveflag_m == SAVE_MESON_ASCII)
	    {
	      fp_m_out = w_ascii_m_i(savefile_m[nk], max_prop);
	      fb_m_out = -1;	/* i.e. file is NOT binary */
	    }
	    else if (saveflag_m == SAVE_MESON_BINARY)
	    {
	      fb_m_out = w_binary_m_i(savefile_m[nk], max_prop);
	      fp_m_out = NULL;	/* i.e. file is NOT ascii */
	    }
	    else
	    {
	      if( this_node == 0 ) 
		printf("ERROR in main saveflag_m = %d is out of range in initial opening\n",saveflag_m)  ;
	      terminate(1); 
	    }


	  } /*** end of spin =0 && color == 0 **/
	  else
	  {
	    fp_out_w[nk] = w_open_wprop(saveflag_w[nk],  savefile_w[nk],
					wqs.type);

	    /* open file for meson output for appending output*/
	    if (saveflag_m == SAVE_MESON_ASCII)
	    {
	      fp_m_out = a_ascii_m_i(savefile_m[nk], max_prop);
	      fb_m_out = -1;	/* i.e. file is NOT binary */
	    }
	    if (saveflag_m == SAVE_MESON_BINARY)
	    {
	      fb_m_out = a_binary_m_i(savefile_m[nk], max_prop);
	      fp_m_out = NULL;	/* i.e. file is NOT ascii */
	    }
	    else
	    {
	      if( this_node == 0 ) 
		printf("ERROR in main saveflag_m = %d is out of range in appending opening\n",saveflag_m)  ;
	      terminate(1); 
	    }



	  }  /*** end of spin && color not equal to zero ***/


	  if (this_node == 0)
	    printf("color=%d spin=%d kappa=%f nk=%d\n", color, spin, (double) kappa, nk);

	  /* load psi if requested */
	  init_qs(&wqstmp2);
	  reload_wprop_sc_to_site(startflag_w[nk], fp_in_w[nk],&wqstmp2,
			    spin, color, F_OFFSET(psi),1);

	  if (nk == 0 || count2 == 1)
	    restart_flag = flag;
	  else
	    restart_flag = 1;

	  
	  /* Conjugate gradient inversion uses site structure
	     temporary"chi" */
	  
	  
	  /* Complete the source structure */
	  wqs.color = color;
	  wqs.spin = spin;
	  wqs.parity = EVENANDODD;
	  
	  /* For wilson_info */
	  wqstmp = wqs;
	  
	  /* If we are starting fresh, we want to set a mininum number of
	     iterations */
	  if(startflag_w[nk] == FRESH)MinMR = nt/2; else MinMR = 0;

	  /* Load inversion control structure */
	  qic.prec = PRECISION;
	  qic.min = MinMR;
	  qic.max = MaxMR;
	  qic.nrestart = nrestart;
	  qic.parity = EVENANDODD;
	  qic.start_flag = restart_flag;
	  qic.nsrc = 1;
	  qic.resid = RsdMR;
	  qic.relresid = 0;
	    
	  /* Load Dirac matrix parameters */
	  dwp.Kappa = kappa;
	  
	  switch (cl_cg) {
	  case CG:
	    /* Load temporaries specific to inverter */
	    
	    /* compute the propagator.  Result in psi. */
	    avm_iters[nk] += 
	      (Real)wilson_invert_site_wqs(F_OFFSET(chi),F_OFFSET(psi),
					   w_source_h,&wqs,
					   cgilu_w_site,&qic,(void *)&dwp);
	    break;
	  case MR:
	    /* Load temporaries specific to inverter */
	    
	    /* compute the propagator.  Result in psi. */
	    avm_iters[nk] += 
	      (Real)wilson_invert_site_wqs(F_OFFSET(chi),F_OFFSET(psi),
					   w_source_h,&wqs,
					   mrilu_w_site,&qic,(void *)&dwp);
		break;
	      default:
		node0_printf("main(%d): Inverter choice %d not supported\n",
			     this_node,cl_cg);
	  }
	  
	  /* save psi if requested */
	  save_wprop_sc_from_site( saveflag_w[nk],fp_out_w[nk], &wqstmp2,
			  spin,color,F_OFFSET(psi),1);


	  light_meson(F_OFFSET(psi), color, spin, wqs.type, fp_m_out, fb_m_out);

	  if (this_node == 0)
	    printf("Light mesons found\n");

	/*** calculate the correlators required for the static variational code **/
	if( nkap == 1 )
	{

	  /** calculate the smeared meson correlators required for Bparam **/
	  calc_smeared_meson(meson, F_OFFSET(psi) ,  F_OFFSET(mp), color, spin);
			 
	  /** calculate the object required for the 2-pt variational calculation ***/
	  buildup_strip(F_OFFSET(psi)   ,  color,  spin); 

	} /** end of the partial calculations for the variationl project ***/



	  /*
	   * find source again since mrilu overwrites it; for hopping
	   * expansion 
	   */
	  /* source must be of definite parity */

	  wqs.parity = source_parity;
	  w_source_h(F_OFFSET(chi), &wqs);

	  hopping(F_OFFSET(chi), F_OFFSET(mp), F_OFFSET(psi), nhop,
		  kappa_c, wqs.parity, color, spin, wqs.type,
		  fp_m_out, fb_m_out);

	  /* close files */

	  r_close_wprop(startflag_w[nk], fp_in_w[nk]);
	  w_close_wprop(saveflag_w[nk],fp_out_w[nk]);

	  if (saveflag_m == SAVE_MESON_ASCII)
	    w_ascii_m_f(fp_m_out, savefile_m[nk]);
	  else if (saveflag_m == SAVE_MESON_BINARY)
	    w_binary_m_f(fb_m_out, savefile_m[nk]);


	  if (spin == end_spin && color == end_color && nk == end_kap)
	    goto end_of_loops;


	}
      }
    }				/* end of loop over spin, color, kappa */

end_of_loops:


    if (this_node == 0)
      printf("RUNNING COMPLETED\n");

    /**DEBUG***/  
#ifdef DEBUGDEF
    light_quark_pion(2) ;
#endif

    for (nk = 0; nk < nkap; nk++)
    {
      if (meascount[nk] > 0)
      {
	if (this_node == 0)
	  printf("total mr iters for measurement= %e\n",
		 (double) avm_iters[nk]);
	if (this_node == 0)
	  printf("average mr iters per spin-color= %e\n",
		 (double) avm_iters[nk] / (double) meascount[nk]);
      }
    }


    endtime=dclock();
    node0_printf("Time = %e seconds\n", (double) (endtime - starttime));

    fflush(stdout);

    /*** calculation section for the variational code *****/
    if( nkap == 1 )
    {
      
      /** sum up the smeared meson correlators over all the nodes ***/
      for(i=0 ; i < nodata ;++i)
      {
	g_complexsum(meson + i) ;
      }
	  

      /* write the smeared correlators to a single disk file ***/
      IF_MASTER
	write_smear_mesonx(meson);
      
      free(meson);  /*** free up the memory for the b-parameter correlators ***/
      
      calc_vary_matrix() ;  /** calculate the static variational matrix **/
      node0_printf(">> The end of the static variational code <<<<\n");
      
    }/** end of the final static variational code *****/


    node0_printf("Time = %e seconds\n",(double)(endtime-starttime));


    fflush(stdout);
    
  } /* end of while(prompt) */

  return 0;

}  /* end of main() */
void calc_heavy_light_form()
{
  int color , spin ; 
  int k_sequential ; 
  int p_insert ;
  int k_spectator , k_zonked_light , k_zonked_heavy  ;
  int HH3_corr_dim , HH3_corr_stride ;
  int HL3_corr_dim  , HL3_corr_stride ;

  complex *HH3_corr ;      /*** The heavy-heavy 3pt correlators *****/
  complex *HL3_corr ;      /*** The heavy-light 3pt correlators *****/

  complex *HL2_GE_corr ;      /*** The heavy-light 2pt correlators *****/
  complex *LL2_GG_corr ;      /*** The light-light 2pt correlators *****/

  complex *HL2_GG_corr ;      /*** The sequential correlators *****/
  complex *HL2_GL_corr ; /*** The local-sink correlators required for fB *****/

  int HL2_GE_corr_dim ;
  int LL2_GG_corr_dim ; 
  int HL2_GG_corr_dim ; 
  int HL2_GL_corr_dim , HL2_GL_corr_stride ; 

  w_prop_file  *zonked_light_ssink_fp[MAX_KAPPA] ; /*** Quark propagator IO stuff **/
  w_prop_file  *zonked_heavy_fp[MAX_KAPPA] ; /*** Quark propagator IO stuff **/
  w_prop_file  *zonked_heavy_ssink_fp[MAX_KAPPA] ; /*** Quark propagator IO stuff **/

  int t_source = wqs_zonked_heavy[0].t0   ; /** assume all the kappa start from the same point **/
  field_offset null_argument = 0 ; 
  int do_heavy_heavy;
  int change_mom_smear;
  int exists_zonked_light_ssink[MAX_KAPPA];
  int exists_zonked_heavy_ssink[MAX_KAPPA];
  int exists_zonked_heavy[MAX_KAPPA];

  /**********----------------------------------------**********/

  setup_w_meson_store();
  setup_HL3_corr( &HL3_corr, &HL3_corr_dim, &HL3_corr_stride );
  setup_HH3_corr( &HH3_corr, &HH3_corr_dim, &HH3_corr_stride); 

  setup_LL2_corr(&LL2_GG_corr , &LL2_GG_corr_dim );

  setup_HL2_corr(&HL2_GE_corr, &HL2_GE_corr_dim);
  setup_HL2_corr(&HL2_GG_corr, &HL2_GG_corr_dim) ; 
  setup_HL2_corr_with_rotations(&HL2_GL_corr , &HL2_GL_corr_dim , &HL2_GL_corr_stride) ; 

  set_zonked_save_intermediate();


  /*** Generate heavy quark propagators
       Gaussian smear the propagators at the sink
       Calculate the light-light and heavy-light smeared sink
       2 pt function 
       Calculate heavy-heavy local sink 2 pt function 
  *****/

  /* Figure out what needs to be done and open any needed and available
     sink-smeared LIGHT quark propagator output files */

  for(k_zonked_light=0 ; k_zonked_light < no_zonked_light ; 
      ++k_zonked_light)
    {
      exists_zonked_light_ssink[k_zonked_light] = 
	file_exists_broadcast(qfile_zonked_light_ssink[k_zonked_light]);

      kappa = kappa_zonked_light[k_zonked_light];
      /* If it doesn't exist and we are computing two pt functions
         that use it, we will create it, so open the file for writing */
	    
      wqstmp = wqs_zonked_light[k_zonked_light];   /* For clover_info */
      strcat(wqstmp.descrp,"; SMEARED SINK.");
      if(!exists_zonked_light_ssink[k_zonked_light] && 
	 ((saveflag_HL2_GG != FORGET) || (saveflag_LL2_GG != FORGET) ))
	zonked_light_ssink_fp[k_zonked_light]
	  = w_open_wprop(saveflag_zonked_light_ssink, 
			 qfile_zonked_light_ssink[k_zonked_light],
			 wqs_zonked_light[k_zonked_light].type); 
    }

  /* See if the heavy-heavy correlators need to be computed */
  do_heavy_heavy = !file_exists_broadcast(filename_HH2_GL);

  /* Figure out what needs to be done and open any needed and available
     sink-smeared HEAVY quark propagator output files */

  for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
      ++k_zonked_heavy)
    {
      exists_zonked_heavy[k_zonked_heavy] =
	file_exists_broadcast(qfile_zonked_heavy[k_zonked_heavy]);

      exists_zonked_heavy_ssink[k_zonked_heavy] =
	file_exists_broadcast(qfile_zonked_heavy_ssink[k_zonked_heavy]);

      /* We (re)compute the heavy_heavy 2 pt function if any of the
	 heavy local propagator files is missing */
      if(!exists_zonked_heavy[k_zonked_heavy])do_heavy_heavy = 1;

      kappa = kappa_zonked_heavy[k_zonked_heavy];
      /* If it doesn't exist, we will write, so open the file for writing */
      wqstmp = wqs_zonked_heavy[k_zonked_heavy];   /* For clover_info */
      if(!exists_zonked_heavy[k_zonked_heavy])
	zonked_heavy_fp[k_zonked_heavy] 
	  = w_open_wprop(saveflag_zonked_heavy[k_zonked_heavy], 
			 qfile_zonked_heavy[k_zonked_heavy],
			 wqs_zonked_heavy[k_zonked_heavy].type); 

      /* If it doesn't exist and we need it for the HL2_GG correlator,
         we will write, so open the file for writing */
      wqstmp = wqs_zonked_heavy[k_zonked_heavy];   /* For clover_info */
      strcat(wqstmp.descrp,"; SMEARED SINK.");
      if(!exists_zonked_heavy_ssink[k_zonked_heavy] && 
	 (saveflag_HL2_GG != FORGET))
	zonked_heavy_ssink_fp[k_zonked_heavy]
	  = w_open_wprop(saveflag_zonked_heavy_ssink, 
			 qfile_zonked_heavy_ssink[k_zonked_heavy],
			 wqs_zonked_heavy[k_zonked_heavy].type); 
    }

  /* We will compute the heavy_heavy spectrum if any one of the heavy
     local-sink propagator files is missing or if the correlator file
     doesn't exist.  Call to initialize the calculation */
  if(do_heavy_heavy)
    meson_spectrum(null_argument , t_source, 0 , no_zonked_heavy,  
		   SETUP_CORR,filename_HH2_GL) ; 

  /* Calculate the local and smeared two-point functions HL2_GL HL2_GG
     and LL2_GG */
  for(color=0 ; color < 3 ; ++color)
    {
      node0_printf("\nSTARTING SMEARED COLOR %d\n",color);
      fflush(stdout);
      /*** generate smeared light quark propagators ***/
      /* We don't load the light propagators and don't calculate the
	 smeared sink two-point functions unless a flag is set (see
	 setup_form) */
      if((saveflag_HL2_GG != FORGET) || (saveflag_LL2_GG != FORGET) ){
	
	for(k_zonked_light=0 ; k_zonked_light < no_zonked_light ; 
	    ++k_zonked_light)
	  {
	    /* If the smeared light quark prop file exists, use it */
	    if(exists_zonked_light_ssink[k_zonked_light])
	      {
		/* Load the smeared sink propagator */
		node0_printf("REUSING previous zonked light tmp file %s\n",
			     qfile_zonked_light_ssink[k_zonked_light]); 
		fflush(stdout);
		for(spin=0; spin < 4 ; ++spin ) 
		  load_in_zonked_light_ssink(color,spin,k_zonked_light,
			F_OFFSET(quark_zonked_light[k_zonked_light].d[spin]));
	      } /* end if exists */
	    else
	      {
		/* Load the local sink propagator and smear at sink */
		for(spin=0; spin < 4 ; ++spin ) 
		  load_in_zonked_light2(color,spin,k_zonked_light,
	              F_OFFSET(quark_zonked_light[k_zonked_light].d[spin]));
		
		/*** smear the light zonked quarks at the sink ****/
		/* NOTE: WE ARE ASSUMING THE SHELL SMEARING FUNCTIONS
		   ARE THE SAME FOR ZONKED AND SPECTATOR QUARKS */
		M_SINK_SMEAR(quark_zonked_light[k_zonked_light],  
			     heavy_smear_func_mom[0] ) ; 
		
		/** write the light ssink quark props to disk ***/
		for(spin=0; spin < 4 ; ++spin ) 
		  save_wprop_sc_from_site(saveflag_zonked_light_ssink, 
                        zonked_light_ssink_fp[k_zonked_light],
			spin, color, 
			F_OFFSET(quark_zonked_light[k_zonked_light].d[spin]),
			1) ;
	      } /* end if file doesn't exist */
	  } /* k_zonked_light */
      } /* end if (saveflag_HL2_GG != FORGET) || (saveflag_LL2_GG != FORGET) */

      /*** generate the heavy zonked local sink propagators if they
	don't already exist. We'll need them for the 3 pt functions even
	if we aren't calculating the smeared two pt functions. ***/
      
      for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
	  ++k_zonked_heavy)
	{
	  if(!exists_zonked_heavy[k_zonked_heavy])
	    {
	      for(spin = 0 ; spin < 4 ; ++spin)
		generate_heavy_zonked(color,spin,
		   kappa_zonked_heavy[k_zonked_heavy],
		   inverter_type_zonked_heavy[k_zonked_heavy],
		   F_OFFSET(quark_zonked_heavy[k_zonked_heavy].d[spin])) ; 

	      /*** store the heavy quark propagator to disk ****/
	      for(spin=0; spin < 4 ; ++spin ) 
		save_wprop_sc_from_site(saveflag_zonked_heavy[k_zonked_heavy], 
                        zonked_heavy_fp[k_zonked_heavy],
			spin, color, 
			F_OFFSET(quark_zonked_heavy[k_zonked_heavy].d[spin]), 
			1) ;
	    } /* end if doesn't exist */
	  else if(do_heavy_heavy)
	    {
	      /* Load heavy zonked local from existing tmp now if we
                 need it for the heavy_heavy prop */
	      for(spin=0; spin < 4 ; ++spin ) 
		load_in_zonked_heavy_local(color,spin,k_zonked_heavy,
		     F_OFFSET(quark_zonked_heavy[k_zonked_heavy].d[spin]));
	    } /* end else if exists but do_heavy_heavy */
	} /* k_zonked_heavy */

      /** Calculate the heavy degenerate spectrum (HH2_GL) **/
      if(do_heavy_heavy)
	{
	  for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
	      ++k_zonked_heavy)
	    meson_spectrum(F_OFFSET(quark_zonked_heavy[k_zonked_heavy]),
			   t_source ,k_zonked_heavy ,
			   no_zonked_heavy, CALCULATE_SPECTRUM,
			   filename_HH2_GL) ; 
	}

      /*** generate the smeared zonked local sink propagators if they
	don't already exist and if we need them for two point functions ***/

      if(saveflag_HL2_GG != FORGET){
	for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
	    ++k_zonked_heavy)
	  {
	    /* If the sink smeared heavy quark prop file exists, use it */
	    if(exists_zonked_heavy_ssink[k_zonked_heavy])
	      {
		node0_printf("REUSING previous zonked heavy tmp file %s\n",
			     qfile_zonked_heavy_ssink[k_zonked_heavy]); 
		fflush(stdout);
		for(spin=0; spin < 4 ; ++spin ) 
		  load_in_zonked_heavy_smear(color,spin,k_zonked_heavy,
			F_OFFSET(quark_zonked_heavy[k_zonked_heavy].d[spin]));
	      } /* end if exists */
	    else
	      {
		/* Load local heavy quark prop unless we just computed
                   or loaded it */
		if(exists_zonked_heavy[k_zonked_heavy] && 
		   !do_heavy_heavy)
		  {
		    node0_printf("REUSING previous zonked heavy file %s\n",
			       qfile_zonked_heavy[k_zonked_heavy]); 
		    fflush(stdout);
		    for(spin=0; spin < 4 ; ++spin ) 
		      load_in_zonked_heavy_local(color,spin,k_zonked_heavy,
			F_OFFSET(quark_zonked_heavy[k_zonked_heavy].d[spin]));
		  }

		/* smear heavy zonked quark at sink */

		M_SINK_SMEAR(quark_zonked_heavy[k_zonked_heavy], 
			     heavy_smear_func_mom[0]); 
		
		/*** store the smeared--smeared quark propagator to disk ***/
		for(spin=0; spin < 4 ; ++spin ) 
		  save_wprop_sc_from_site(saveflag_zonked_heavy_ssink, 
                     zonked_heavy_ssink_fp[k_zonked_heavy],
		     spin, color, 
		     F_OFFSET(quark_zonked_heavy[k_zonked_heavy].d[spin]), 1) ;
	      } /* end if doesn't exist */
	    
	  } /* k_zonked_heavy */

      } /* if(saveflag_HL2_GG != FORGET) */
	
      /** Calculate the source- and sink-smeared 2 pt functions **/
      
      if((saveflag_HL2_GG != FORGET) || (saveflag_LL2_GG != FORGET)) {
	for(k_spectator = 0 ; k_spectator < no_spectator ; ++k_spectator)
	  {
	    
	    /** generate or copy sink smeared spectator quark propagator **/
	    
	    /* If the spectator quark is the same as one of the zonked light
	       quarks, copy the preloaded smeared zonked light quark instead 
	       NOTE: WE ARE ASSUMING THE SMEARING FUNCTIONS ARE THE SAME
	       FOR ZONKED AND SPECTATOR QUARKS */
	    
	    restore_smeared_spectator(color,k_spectator);
	    
	    /****-----  
	      light-light two-pt (LL2_GG) 
	      (symmetric source and sink smearing)
	      -----*****/
	    
	    /* Skip this calculation if flag is set (see setup_form) */
	    if(saveflag_LL2_GG != FORGET){
	      
	      node0_printf("Computing LL2_GG correlator\n");
	      fflush(stdout);
	      for(k_zonked_light=0 ; k_zonked_light < no_zonked_light ; 
		  ++k_zonked_light)
		{
		  
		  /*** calculate the light-light two point functions 
		    (LL2_GG) (shell smearing functions) *****/
		  contract_LL2(LL2_GG_corr,  
		       F_OFFSET(quark_zonked_light[k_zonked_light]) , 
		       F_OFFSET(quark_spectator ) ,
		       k_zonked_light, k_spectator)  ;
		}  /*** end of the loop over the zonked light quark reads  ***/
	      
	    } /* end of if(saveflag_LL2_GG != FORGET) */
	    
	    /****-----  
	      heavy-light two pt (HL2_GG) (shell smeared source and sink)
	      -----*****/
	    
	    /* Skip this calculation if flag is set (see setup_form) */
	    if(saveflag_HL2_GG != FORGET){
	      node0_printf("Computing HL2_GG correlator\n");
	      fflush(stdout);
	      
	      for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
		  ++k_zonked_heavy)
		{
		  
		  /*** contract the heavy-light two point function 
		    (HL2_GG) (BAG SMEARING AT THE SINK) ******/
		  contract_HL2(HL2_GG_corr ,
		      F_OFFSET(quark_zonked_heavy[k_zonked_heavy]) , 
		      F_OFFSET(quark_spectator),
		      k_zonked_heavy, k_spectator)  ; 
		  
		}  /*** end of the loop over the zonked heavy quarks  ***/
	      
	    } /* End of if(saveflag_HL2_GG != FORGET) */

	  } /* k_spectator */
	    
      } /* End of if((saveflag_HL2_GG != FORGET) || (saveflag_LL2_GG != FORGET) ) */

    } /* color */
  
  /* Write heavy-heavy spectrum */
  if(do_heavy_heavy)
    meson_spectrum(null_argument ,t_source ,0 ,no_zonked_heavy, 
		   WRITE_RESULTS,filename_HH2_GL) ; 
  
  /*** write the sink-smeared 2 pt correlators to disk ****/
  
  finish_LL2_GG_corr(LL2_GG_corr, LL2_GG_corr_dim ) ; 
  finish_HL2_GG_corr(HL2_GG_corr, HL2_GG_corr_dim ) ; 
  
  free(LL2_GG_corr); 
  free(HL2_GG_corr) ; 
  
  
  /* Close the temporary propagator output files if we opened them before */

  for(k_zonked_light=0 ; k_zonked_light < no_zonked_light ; 
      ++k_zonked_light)
    {
      if(!exists_zonked_light_ssink[k_zonked_light] && 
	 ((saveflag_HL2_GG != FORGET) || (saveflag_LL2_GG != FORGET) ))
	w_close_wprop(saveflag_zonked_light_ssink, 
		     zonked_light_ssink_fp[k_zonked_light]); 
    }

  for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
      ++k_zonked_heavy)
    {
      if(!exists_zonked_heavy[k_zonked_heavy])
	w_close_wprop(saveflag_zonked_heavy[k_zonked_heavy],
		     zonked_heavy_fp[k_zonked_heavy]); 
      if(!exists_zonked_heavy_ssink[k_zonked_heavy] && 
	 (saveflag_HL2_GG != FORGET))
	w_close_wprop(saveflag_zonked_heavy_ssink,
		     zonked_heavy_ssink_fp[k_zonked_heavy]); 
    }


  /*** Calculate the heavy to heavy and heavy to light 3 pt functions
       Calculate the heavy-light 2 pt functions requiring local-sink quark
       propagators ***/

  for(color=0 ; color < 3 ; ++color)
  {
    node0_printf("\nSTARTING LOCAL COLOR %d\n",color);
    fflush(stdout);

    if((saveflag_HL2_GL != FORGET) ||
       (saveflag_HL2_GE != FORGET) ||
       (saveflag_HH3 != FORGET) ||
       (saveflag_HL3 != FORGET))
      {
	/*** reload all the light zonked quark propagators for this color, 
	  smeared at the source only **/
	for(k_zonked_light=0 ; k_zonked_light < no_zonked_light ; 
	    ++k_zonked_light)
	  for(spin=0; spin < 4 ; ++spin ) 
	    load_in_zonked_light2(color,spin,k_zonked_light,
		    F_OFFSET(quark_zonked_light[k_zonked_light].d[spin]));
	
	/*** reload all the heavy zonked quark propagators for this color, 
	  smeared at the source only **/
	for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
	    ++k_zonked_heavy)
	  for(spin=0; spin < 4 ; ++spin ) 
	    load_in_zonked_heavy_local(color,spin,k_zonked_heavy,
		     F_OFFSET(quark_zonked_heavy[k_zonked_heavy].d[spin]));
      }
	
    for(k_spectator = 0 ; k_spectator < no_spectator ; ++k_spectator)
    {
      
      /** restore spectator quark propagator to "quark_spectator" **/
      restore_local_spectator(color, k_spectator);

      /****-----  
	   heavy-light two-pt (HL2_GL) (shell smeared sources, local sink)
	   -----*****/

      /* Skip this calculation if flag is set (see setup_form) */
      if(saveflag_HL2_GL != FORGET){
	for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
	  ++k_zonked_heavy)
	{
	  
	  node0_printf("Computing HL2_GL correlator\n");
	  fflush(stdout);

	  /*** contract the heavy-light two point function 
	       (HL2_GL) (smeared-source local-sink) */
	  contract_HL2_with_rotations(HL2_GL_corr, 
	       F_OFFSET(quark_zonked_heavy[k_zonked_heavy]) , 
	       F_OFFSET(quark_spectator),
	       F_OFFSET(quark_rot ),
	       k_zonked_heavy, k_spectator)  ;
	  
	  
	}  /*** end of the loop over the zonked heavy inversions  ***/

      } /* End of if(saveflag_HL2_GL != FORGET) */


      for(p_insert = 0 ; p_insert < no_p_values ; ++p_insert)
	{
	  node0_printf("\nStarting momentum insertion %d\n",p_insert);
	  fflush(stdout);

	  /**** sink smear the spectator quark 
		with the sequential smearing function ****/
	  /* If the smearing function did not change, retain the 
	     previous result */
	  if(p_insert == 0 || 
	     strcmp(seq_smear_file[p_insert],
		    seq_smear_file[p_insert-1]) != 0)
	    {
	      node0_printf("Sequential smearing for momentum insertion %d\n",p_insert);
	      fflush(stdout);

	      restore_local_spectator(color, k_spectator);
	      M_SINK_SMEAR(quark_spectator,seq_smear_func[ p_insert ] ) ;
	      change_mom_smear = 1;
	    }
	  else
	    {
	      node0_printf("REUSING sequentially smeared spectator from previous momentum insertion\n");
	      fflush(stdout);
	      change_mom_smear = 0;
	    }


	  /****-----  
	       heavy-light two pt (HL2_GE) (shell source, relative sink)
	       -----*****/
	  
	  /* Skip this calculation if flag is set (see setup_form) */
	  /* Note: We may, in the future, want to distiguish
	     smearing functions for different B meson momenta.
	     However, at present we have not made any provision
	     for such a distinction in the relative-smeared
	     two-point function.  So we do the calculation only
	     for the first momentum in the list! CD */

	  if((saveflag_HL2_GE != FORGET) && p_insert > 0 && change_mom_smear)
	    node0_printf("WARNING: HL2_GE correlator is computed only for the first smearing function in the list\n");
	  fflush(stdout);
	  
	  if((saveflag_HL2_GE != FORGET) && p_insert == 0){
	    
	    node0_printf("Computing HL2_GE correlator\n");
	    fflush(stdout);
	    for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
		++k_zonked_heavy)
	      {
		
		/*** contract the heavy-light two point functiion 
		  (HL2_GE) (RELATIVE SMEARING AT THE SINK) */
		contract_HL2(
		      HL2_GE_corr, 
		      F_OFFSET(quark_zonked_heavy[k_zonked_heavy]) , 
		      F_OFFSET(quark_spectator),
		      k_zonked_heavy, k_spectator)  ;
		
	      }  /*** end of the loop over the zonked heavy inversions  ***/
	  }


	  if( (saveflag_HH3 != FORGET) || (saveflag_HL3 != FORGET))
	    
	    for(k_sequential = 0 ; k_sequential < no_sequential   ; ++k_sequential  )
	      {

		copy_site_spin_wilson_vector(F_OFFSET(quark_spectator) ,
					     F_OFFSET(quark_sequential) );
		
		if( this_node == 0 ) 
		  printf("Computing form factors for k_sequential = %g k_spectator = %g p = %d,%d,%d \n",
			 kappa_sequential[k_sequential], kappa_spectator[k_spectator] , 
			 p_momstore[p_insert ][0] , p_momstore[p_insert ][1] , p_momstore[p_insert ][2] ) ; 
		fflush(stdout);
		
		
		/*** generate the sequential source propagator ****/
		for(spin = 0 ; spin < 4 ; ++spin )
		  {
		    
		    /*** do the sequential source inversion ****/
		    kappa = kappa_sequential[k_sequential] ;
		    sequential_source(
			    F_OFFSET(quark_sequential.d[spin] ),
			    F_OFFSET(psi), 
			    p_momstore[p_insert ][0] ,
			    p_momstore[p_insert ][1] ,
			    p_momstore[p_insert ][2] ,
			    tf, color,spin,
			    kappa_sequential[k_sequential] ,  
			    inverter_type_sequential[k_sequential],
			    niter_zonked_heavy,  
			    nrestart_zonked_heavy, 
			    resid_zonked_heavy, 
			    p_insert ) ;  
		  } /*** end of the loop over spin ***/
		
		
		
		/****-----  
		  heavy to light form factor 
		  -----*****/
		
		if( (saveflag_HL3 != FORGET) ) {
		  for(k_zonked_light=0 ; k_zonked_light < no_zonked_light ; 
		      ++k_zonked_light)
		    {
		      
		      /*** tie the propagators together to calculate 
			the three point function *****/
		      contract_HL3(
			     HL3_corr, 
			     F_OFFSET(quark_zonked_light[k_zonked_light]),
			     F_OFFSET(quark_sequential) ,
			     F_OFFSET(quark_rot ),
			     p_insert, k_sequential, 
			     k_zonked_light, k_spectator )  ;
		      
		      
		    }  /*** end of the loop over the zonked light
		  	 quark reads ***/
		  
		} /* End of if(saveflag_HL3 != FORGET) */
		/****-----  
		  heavy to heavy form factor 
		  -----*****/
		
		if( (saveflag_HH3 != FORGET) ) {
		  for(k_zonked_heavy=0 ; k_zonked_heavy < no_zonked_heavy ; 
		      ++k_zonked_heavy)
		    {
		      
		      /*** tie the propagators together *****/
		      contract_HH3(
			     HH3_corr, 
			     F_OFFSET(quark_zonked_heavy[k_zonked_heavy]),
			     F_OFFSET(quark_sequential  ),
			     F_OFFSET(quark_rot ),
			     p_insert, k_sequential, 
			     k_zonked_heavy, k_spectator )  ;
		      
		    }  /*** end of the loop over the zonked heavy quarks ***/
		} /* End of if(saveflag_HH3 != FORGET) */
		
	      } /*** end of the loop over  p_insert  ****/
	}   /*** end of the loop over  k_sequential   ****/
      
    } /*** end of the loop over   k_spectator  ****/
    
  }  /*** end the loop over the source colour *****/

  /*** write the correlators to disk ****/

  finish_HL3_corr(HL3_corr, HL3_corr_dim , HL3_corr_stride) ; 
  finish_HH3_corr(HH3_corr, HH3_corr_dim , HH3_corr_stride) ;

  finish_HL2_GE_corr(HL2_GE_corr, HL2_GE_corr_dim ) ; 
  finish_HL2_GL_corr(HL2_GL_corr,
      HL2_GL_corr_dim, HL2_GL_corr_stride );
  
  /****  free up the memory used in this code  *****/
  free(HH3_corr) ; 
  free(HL3_corr) ; 
  free(HL2_GE_corr); 
  free(HL2_GL_corr);


}  /***** end of calc_heavy_light_form ****/
Example #4
0
int congrad_xxx(
    field_offset src,   /* type wilson_vector (where source is to be created)*/
    Real cgmass, /* unused here*/
    int source_chirality /* chirality sector for inversion (NOT USED)  */
    )
{
register int i;
register site *s;
int j,k, avs_iters, avm_iters,status,flag;
int MaxCG;
int ksource, spin,color,my_chirality,chb,che,chbo,cheo,ii,jj;
Real *RsdCG;
Real size_r,one_minus_m,r02inv;

wilson_vector **psim;

void setup_multi();

w_prop_file *fp_out_w[MAX_MASSES];       /* For propagator files */
w_prop_file *fp_in_w[MAX_MASSES];        /* For propagator files */
w_prop_file *h0_out_w[MAX_MASSES];       /* For intermediate propagator files */


#ifdef EIGO
wilson_vector wproj;
complex ctmp,cd,*cproj;

int l;
int icount, ivec;
int *chiral_check;
Real cdp, cdm;
Real *ca, *cb;
Real eps, mu, denom;
#endif

double source_norm;

RsdCG=resid;
MaxCG=niter;
avs_iters=0;
r02inv= -0.5/R0;

#ifdef MINN
  do_minn=1;
#endif

    setup_multi();

#ifdef EIGO
  if(Nvecs_hov != 0)cproj = (complex *)malloc(Nvecs_hov*sizeof(complex));
  /* check chirality of your modes (to identify zero modes) */
  if(Nvecs_hov != 0)chiral_check= (int *)malloc(Nvecs_hov*sizeof(int));
  for(j=0;j<Nvecs_hov;j++){
    cdp=0.0;
    cdm=0.0;
    FORALLSITES(i,s){
      for(l=0;l<2;l++)for(k=0;k<3;k++){
        cdp += cabs_sq(&(eigVec[j][i].d[l].c[k]));
      }
      for(l=2;l<4;l++)for(k=0;k<3;k++){
        cdm += cabs_sq(&(eigVec[j][i].d[l].c[k]));
      }
    }
    g_floatsum(&cdp);
    g_floatsum(&cdm);

    if(cdm< 1.e-6 && cdp >1.e-6)
      chiral_check[j] =1;
    else if (cdm >1.e-6 && cdp < 1.e-6)
      chiral_check[j] = -1;
    else if (cdm >1.e-6 && cdp > 1.e-6)
      chiral_check[j] =0;
    else{
      node0_printf("eigVec0[%d] is a null vector!\n",j);
      exit(1);
    }
  }
    /* the  mode  propagator matrix */
  /* I am stupid--how to do this in a 2-d array?? */
  if(Nvecs_hov != 0){
    ca= (Real *)malloc(num_masses*Nvecs_hov*sizeof(Real));
    cb= (Real *)malloc(num_masses*Nvecs_hov*sizeof(Real));
  }

  /* initialize the coefficients of the propagator matrix for modes */

  for(k=0;k<num_masses;k++)for(ivec=0;ivec<Nvecs_hov;ivec++){
    icount=Nvecs_hov*k + ivec;

    if(chiral_check[ivec]==0){
      mu=mass[k]/(2.0*R0);
      eps= sqrt(eigVal[ivec])/(2.0*R0);
      denom= (mu*mu+eps*eps*(1.0-mu*mu))*2.0*R0;
      ca[icount]= mu*(1.0-eps*eps)/denom;
      cb[icount]= eps*sqrt(1.0-eps*eps)/denom;
    }
    else{
      ca[icount]= 1.0/mass[k];
      cb[icount]= 0.0;
    }
    node0_printf("mass %e mode %d %d %e %e\n",mass[k],ivec,
                 chiral_check[ivec],ca[icount],cb[icount]);
  }
#endif


    /* open the prop files */

    for(k=0;k<num_masses;k++){
      fp_in_w[k]  = r_open_wprop(startflag_w[k], startfile_w[k]);
      fp_out_w[k] = w_open_wprop(saveflag_w[k],  savefile_w[k], wqs.type);
#ifdef H0INV
      h0_out_w[k] = w_open_wprop(saveflag_w3[k],  savefile_w3[k], wqs.type);
#endif
    }

  for(ksource = 0; ksource < wqs.nsource; ksource++){
    spin = convert_ksource_to_spin(ksource);
    color = convert_ksource_to_color(ksource);

//                /* Loop over source spins */
//    for(spin=0;spin<4;spin++){
//            /* Loop over source colors */
//    for(color=0;color<3;color++){

node0_printf("Propagator color %d spin %d\n",color,spin);
if(startflag_w[0] == FRESH){flag=0;}
else{
      /* check if there's a propagator already there--Do for all masses */
      flag=1;
      for(k=0;k<num_masses && flag==1 ;k++){
#ifdef IOTIME
      status = reload_wprop_sc_to_site( startflag_w[k], fp_in_w[k],
                                   &wqs, spin, color, F_OFFSET(psi),1);
#else
      status = reload_wprop_sc_to_site( startflag_w[k], fp_in_w[k],
                               &wqs, spin, color, F_OFFSET(psi),0);
#endif
      if(status != 0){
	node0_printf("congrad_outer_p: computing prop\n");
	/*
	reload_wprop_sc_to_site( FRESH, fp_in_w[k],
                               &wqs, spin, color, F_OFFSET(psi),0);
			       */
	flag = 0;
      }
      else{ /* status = 1--put the propagator in the new output file
so all the elements are in one place. This will fail if 
the propagator generation did not write the same number of elements
for each mass value propagator */
#ifdef IOTIME
                    save_wprop_sc_from_site( saveflag_w[k],fp_out_w[k],
                                    &wqs, spin,color,F_OFFSET(psi),1);
#else
                    save_wprop_sc_from_site( saveflag_w[k],fp_out_w[k],
                                    &wqs, spin,color,F_OFFSET(psi),0);
#endif
      }
      } /* k loop */
} /*startflag_w != FRESH */

      if(flag==0){  /* proceed to inversion */
      if(spin<2){my_chirality=1;chb=0;che=2;chbo=2;cheo=4;}
      else {my_chirality= -1;chb=2,che=4;chbo=0;cheo=2;}
      chirality_flag=my_chirality;

      /* Make source */

             /* Complete the source structure */

      /* NEEDS FIXING!! */
//            wqs.color = color;
//            wqs.spin = spin;

            /* For wilson_info */
            wqstmp = wqs;
	    //	    status = w_source_site(src,&wqs);
	    status = wv_source_site(src,&wqs);

	    /* check original source size... */
	    source_norm=0.0;
	    FORALLSITES(i,s){
	      source_norm += (double)magsq_wvec(((wilson_vector *)F_PT(s,src))  );
	    }
	    g_doublesum( &source_norm );

  if(this_node==0){
    printf("Original: source_norm = %e\n",source_norm);
    fflush(stdout);
  } 



	  FORALLSITES(i,s) copy_wvec((wilson_vector *)F_PT(s,src),&(s->chi0));
#ifdef EIGO
      /* project out the eigenvectors from the source */
node0_printf("removing %d modes from source\n",Nvecs_hov);
	  for(j=0;j<Nvecs_hov;j++){
	    cd=cmplx(0.0,0.0);
            FORALLSITES(i,s){
	      /* wproj will hold the chiral projections--
	       recall we have ``packed'' two chiralities into eigVec */
	      clear_wvec(&wproj);
	      for(ii=chb;ii<che;ii++)for(jj=0;jj<3;jj++){
		wproj.d[ii].c[jj]=eigVec[j][i].d[ii].c[jj];
	      }
	      ctmp =  wvec_dot( &(wproj),(wilson_vector *)F_PT(s,src));
	      CSUM(cd,ctmp);
	    }
	    g_complexsum(&cd);
	    cproj[j]=cd;
node0_printf("projector %d %e %e\n",j,cproj[j].real,cproj[j].imag);

	    CMULREAL(cd,-1.0,cd);

	    FORALLSITES(i,s){
	      clear_wvec(&wproj);
	      for(ii=chb;ii<che;ii++)for(jj=0;jj<3;jj++){
		wproj.d[ii].c[jj]=eigVec[j][i].d[ii].c[jj];
	      }
	      c_scalar_mult_add_wvec(&(s->chi0), &(wproj),
                             &cd, &(s->chi0) );
	    }
	  }