Ejemplo n.º 1
0
int main( int argc, char **argv ){
  register site *s;
  int i,si;
  int prompt;
  double dtime;
  su3_vector **eigVec ;
  su3_vector *tmp ;
  double *eigVal ;
  int total_R_iters ;
  double chirality ;

  initialize_machine(&argc,&argv);
#ifdef HAVE_QDP
  QDP_initialize(&argc, &argv);
#ifndef QDP_PROFILE
  QDP_profcontrol(0);
#endif
#endif
  /* Remap standard I/O */
  if(remap_stdio_from_args(argc, argv) == 1)terminate(1);

  g_sync();
  /* set up */
  prompt = setup();

  /* loop over input sets */
  while( readin(prompt) == 0){
    
    dtime = -dclock();
    invalidate_all_ferm_links(&fn_links);
    make_path_table(&ks_act_paths, &ks_act_paths_dmdu0);
    /* Load fat and long links for fermion measurements if needed */
    load_ferm_links(&fn_links, &ks_act_paths);
    /* call fermion_variable measuring routines */
    /* results are printed in output file */
    f_meas_imp( F_OFFSET(phi), F_OFFSET(xxx), mass,
		&fn_links, &fn_links_dmdu0);
    eigVal = (double *)malloc(Nvecs*sizeof(double));
    eigVec = (su3_vector **)malloc(Nvecs*sizeof(su3_vector*));
    for(i=0;i<Nvecs;i++)
      eigVec[i]=
	(su3_vector*)malloc(sites_on_node*sizeof(su3_vector));
    
    total_R_iters=Kalkreuter(eigVec, eigVal, eigenval_tol, 
			     error_decr, Nvecs, MaxIter, Restart, 
			     Kiters, EVEN, &fn_links) ;
    tmp = (su3_vector*)malloc(sites_on_node*sizeof(su3_vector));
    for(i=0;i<Nvecs;i++)
      { 
	/* Construct to odd part of the vector.                 *
	 * Note that the true odd part of the eigenvector is    *
	 *  i/sqrt(eigVal) Dslash Psi. But since I only compute *
	 * the chirality the i factor is irrelevant (-i)*i=1!!  */
	dslash_fn_field(eigVec[i], tmp, ODD, &fn_links) ;
	FORSOMEPARITY(si,s,ODD){ 
	  scalar_mult_su3_vector( &(tmp[si]),
				  1.0/sqrt(eigVal[i]), 
				  &(eigVec[i][si]) ) ;
	}
	
	measure_chirality(eigVec[i], &chirality, EVENANDODD);
	/* Here I divide by 2 since the EVEN vector is normalized to
	 * 1. The EVENANDODD vector is normalized to 2. I could have
	 * normalized the EVENANDODD vector to 1 and then not devide
	 * by to.  The measure_chirality routine assumes vectors
	 * normalized to 1.  */
	node0_printf("Chirality(%i): %g\n",i,chirality/2) ;
      }
    free(tmp);
    /**
       for(i=0;i<Nvecs;i++)
       {
       sprintf(label,"DENSITY(%i)",i) ;
       print_densities(eigVec[i], label, ny/2,nz/2,nt/2, EVEN) ;
       }
    **/
    for(i=0;i<Nvecs;i++)
      free(eigVec[i]) ;
    free(eigVec) ;
    free(eigVal) ;
#ifdef FN
    invalidate_all_ferm_links(&fn_links);
#endif
    fflush(stdout);
    
    node0_printf("RUNNING COMPLETED\n"); fflush(stdout);
    dtime += dclock();
    if(this_node==0){
      printf("Time = %e seconds\n",dtime);
      printf("total_iters = %d\n",total_iters);
      printf("total Rayleigh iters = %d\n",total_R_iters);
    }
    fflush(stdout);
  }
Ejemplo n.º 2
0
int
main( int argc, char **argv )
{
  int meascount,traj_done;
  int prompt;
  int s_iters, avs_iters, avspect_iters, avbcorr_iters;
  double dtime, dclock();
  
  initialize_machine(&argc,&argv);
#ifdef HAVE_QDP
  QDP_initialize(&argc, &argv);
#endif
  /* Remap standard I/O */
  if(remap_stdio_from_args(argc, argv) == 1)terminate(1);
  
  g_sync();
  /* set up */
  prompt = setup();

//  restore_random_state_scidac_to_site("randsave", F_OFFSET(site_prn));
//  restore_color_vector_scidac_to_site("xxx1save", F_OFFSET(xxx1),1);
//  restore_color_vector_scidac_to_site("xxx2save", F_OFFSET(xxx2),1);

  /* loop over input sets */
  while( readin(prompt) == 0) {
    
    /* perform warmup trajectories */
    dtime = -dclock();
    for( traj_done=0; traj_done < warms; traj_done++ ){
      update();
    }
    node0_printf("WARMUPS COMPLETED\n"); fflush(stdout);
    
    /* perform measuring trajectories, reunitarizing and measuring 	*/
    meascount=0;		/* number of measurements 		*/
    avspect_iters = avs_iters = avbcorr_iters = 0;
    for( traj_done=0; traj_done < trajecs; traj_done++ ){ 
      
      /* do the trajectories */
      s_iters=update();
      
      /* measure every "propinterval" trajectories */
      if( (traj_done%propinterval)==(propinterval-1) ){
	
	/* call gauge_variable fermion_variable measuring routines */
	/* results are printed in output file */
	rephase(OFF);
	g_measure( );
	rephase(ON);

	/* Load fat and long links for fermion measurements */
	load_ferm_links(&fn_links, &ks_act_paths);
#ifdef DM_DU0
	load_ferm_links(&fn_links_dmdu0, &ks_act_paths_dmdu0);
#endif

	/* Measure pbp, etc */
#ifdef ONEMASS
	f_meas_imp(F_OFFSET(phi),F_OFFSET(xxx),mass, &fn_links, 
		   &fn_links_dmdu0);
#else
	f_meas_imp( F_OFFSET(phi1), F_OFFSET(xxx1), mass1, 
		    &fn_links, &fn_links_dmdu0);
	f_meas_imp( F_OFFSET(phi2), F_OFFSET(xxx2), mass2,
		    &fn_links, &fn_links_dmdu0);
#endif

	/* Measure derivatives wrto chemical potential */
#ifdef D_CHEM_POT
#ifdef ONEMASS
	Deriv_O6( F_OFFSET(phi1), F_OFFSET(xxx1), F_OFFSET(xxx2), mass,
		  &fn_links, &fn_links_dmdu0);
#else
	Deriv_O6( F_OFFSET(phi1), F_OFFSET(xxx1), F_OFFSET(xxx2), mass1,
		  &fn_links, &fn_links_dmdu0);
	Deriv_O6( F_OFFSET(phi1), F_OFFSET(xxx1), F_OFFSET(xxx2), mass2,
		  &fn_links, &fn_links_dmdu0);
#endif
#endif

#ifdef SPECTRUM 
	/* Fix TUP Coulomb gauge - gauge links only*/
	rephase( OFF );
	gaugefix(TUP,(Real)1.8,500,(Real)GAUGE_FIX_TOL);
	rephase( ON );
#ifdef FN
	invalidate_all_ferm_links(&fn_links);
#ifdef DM_DU0
	invalidate_all_ferm_links(&fn_links_dmdu0);
#endif
#endif
	/* Load fat and long links for fermion measurements */
	load_ferm_links(&fn_links, &ks_act_paths);
#ifdef DM_DU0
	load_ferm_links(&fn_links_dmdu0, &ks_act_paths_dmdu0);
#endif	
	if(strstr(spectrum_request,",spectrum,") != NULL){
#ifdef ONEMASS
	  avspect_iters += spectrum2(mass,F_OFFSET(phi),F_OFFSET(xxx),
				     &fn_links);
#else
	  avspect_iters += spectrum2( mass1, F_OFFSET(phi1),
				      F_OFFSET(xxx1), &fn_links);
	  avspect_iters += spectrum2( mass2, F_OFFSET(phi1),
				      F_OFFSET(xxx1), &fn_links);
#endif
	}
	
	if(strstr(spectrum_request,",spectrum_point,") != NULL){
#ifdef ONEMASS
	  avspect_iters += spectrum_fzw(mass,F_OFFSET(phi),F_OFFSET(xxx),
					&fn_links);
#else
	  avspect_iters += spectrum_fzw( mass1, F_OFFSET(phi1),
					 F_OFFSET(xxx1), &fn_links);
	  avspect_iters += spectrum_fzw( mass2, F_OFFSET(phi1),
					 F_OFFSET(xxx1), &fn_links);
#endif
	}
	
	if(strstr(spectrum_request,",nl_spectrum,") != NULL){
#ifdef ONEMASS
	  avspect_iters += nl_spectrum(mass,F_OFFSET(phi),F_OFFSET(xxx),
				       F_OFFSET(tempmat1),F_OFFSET(staple),
				       &fn_links);
#else
	  avspect_iters += nl_spectrum( mass1, F_OFFSET(phi1), 
		F_OFFSET(xxx1), F_OFFSET(tempmat1),F_OFFSET(staple),
					&fn_links);
#endif
	}
	
	if(strstr(spectrum_request,",spectrum_mom,") != NULL){
#ifdef ONEMASS
	  avspect_iters += spectrum_mom(mass,mass,F_OFFSET(phi),5e-3,
					&fn_links);
#else
	  avspect_iters += spectrum_mom( mass1, mass1, 
					 F_OFFSET(phi1), 1e-1,
					 &fn_links);
#endif
	}
	
	if(strstr(spectrum_request,",spectrum_multimom,") != NULL){
#ifdef ONEMASS
	  avspect_iters += spectrum_multimom(mass,
					     spectrum_multimom_low_mass,
					     spectrum_multimom_mass_step,
					     spectrum_multimom_nmasses,
					     5e-3, &fn_links);
#else
	  avspect_iters += spectrum_multimom(mass1,
					     spectrum_multimom_low_mass,
					     spectrum_multimom_mass_step,
					     spectrum_multimom_nmasses,
					     5e-3, &fn_links);

#endif
	}
	
#ifndef ONEMASS
	if(strstr(spectrum_request,",spectrum_nd,") != NULL){
	  avspect_iters += spectrum_nd( mass1, mass2, 1e-1,
					&fn_links);
	}
#endif
	if(strstr(spectrum_request,",spectrum_nlpi2,") != NULL){
#ifdef ONEMASS
	  avspect_iters += spectrum_nlpi2(mass,mass,F_OFFSET(phi),5e-3,
					  &fn_links );
#else
	  avspect_iters += spectrum_nlpi2( mass1, mass1, 
					   F_OFFSET(phi1),1e-1,
					   &fn_links );
	  avspect_iters += spectrum_nlpi2( mass2, mass2, 
					   F_OFFSET(phi1),1e-1,
					   &fn_links );
#endif
	}
	
	if(strstr(spectrum_request,",spectrum_singlets,") != NULL){
#ifdef ONEMASS
	  avspect_iters += spectrum_singlets(mass, 5e-3, F_OFFSET(phi),
					     &fn_links);
#else
	  avspect_iters += spectrum_singlets(mass1, 5e-3, F_OFFSET(phi1),
					     &fn_links );
	  avspect_iters += spectrum_singlets(mass2, 5e-3, F_OFFSET(phi1),
					     &fn_links );
#endif
	}

	if(strstr(spectrum_request,",fpi,") != NULL)
	  {
	    avspect_iters += fpi_2( fpi_mass, fpi_nmasses, 2e-3,
				    &fn_links );
	  }
	
#ifdef HYBRIDS
	if(strstr(spectrum_request,",spectrum_hybrids,") != NULL){
#ifdef ONEMASS
	  avspect_iters += spectrum_hybrids( mass,F_OFFSET(phi),1e-1,
					     &fn_links);
#else
	  avspect_iters += spectrum_hybrids( mass1, F_OFFSET(phi1), 5e-3,
					     &fn_links);
	  avspect_iters += spectrum_hybrids( mass2, F_OFFSET(phi1), 2e-3,
					     &fn_links);
#endif
	}
#endif
	if(strstr(spectrum_request,",hvy_pot,") != NULL){
	  rephase( OFF );
	  hvy_pot( F_OFFSET(link[XUP]) );
	  rephase( ON );
	}
#endif /* SPECTRUM */
	avs_iters += s_iters;
	++meascount;
	fflush(stdout);
      }
    }	/* end loop over trajectories */
    
    node0_printf("RUNNING COMPLETED\n"); fflush(stdout);
    if(meascount>0)  {
      node0_printf("average cg iters for step= %e\n",
		   (double)avs_iters/meascount);
#ifdef SPECTRUM
      node0_printf("average cg iters for spectrum = %e\n",
		   (double)avspect_iters/meascount);
#endif
    }
    
    dtime += dclock();
    if(this_node==0){
      printf("Time = %e seconds\n",dtime);
      printf("total_iters = %d\n",total_iters);
    }
    fflush(stdout);
    
    /* save lattice if requested */
    if( saveflag != FORGET ){
      rephase( OFF );
      save_lattice( saveflag, savefile, stringLFN );
      rephase( ON );
#ifdef HAVE_QIO
//       save_random_state_scidac_from_site("randsave", "Dummy file XML",
//        "Random number state", QIO_SINGLEFILE, F_OFFSET(site_prn));
//       save_color_vector_scidac_from_site("xxx1save", "Dummy file XML",
//        "xxx vector", QIO_SINGLEFILE, F_OFFSET(xxx1),1);
//       save_color_vector_scidac_from_site("xxx2save", "Dummy file XML",
//        "xxx vector", QIO_SINGLEFILE, F_OFFSET(xxx2),1);
#endif
    }
  }
#ifdef HAVE_QDP
  QDP_finalize();
#endif  
  normal_exit(0);
  return 0;
}
Ejemplo n.º 3
0
int main( int argc, char **argv ){
  register site *s;
  int i,si;
  int prompt;
  double dtime;
  su3_vector **eigVec ;
  su3_vector *tmp ;
  double *eigVal ;
  int total_R_iters ;
  double chirality, chir_ev, chir_od ;
  imp_ferm_links_t **fn;

  initialize_machine(&argc,&argv);
  /* Remap standard I/O */
  if(remap_stdio_from_args(argc, argv) == 1)terminate(1);

  g_sync();
  /* set up */
  prompt = setup();

  /* loop over input sets */
  while( readin(prompt) == 0){
    dtime = -dclock();
#ifdef HISQ_SVD_COUNTER
    hisq_svd_counter = 0;
#endif
    restore_fermion_links_from_site(fn_links, PRECISION);
    /* call fermion_variable measuring routines */
    /* results are printed in output file */
    f_meas_imp( 1, PRECISION, F_OFFSET(phi), F_OFFSET(xxx), mass,
		0, fn_links);

    eigVal = (double *)malloc(Nvecs*sizeof(double));
    eigVec = (su3_vector **)malloc(Nvecs*sizeof(su3_vector*));
    for(i=0;i<Nvecs;i++)
      eigVec[i]=
	(su3_vector*)malloc(sites_on_node*sizeof(su3_vector));
    
    fn = get_fm_links(fn_links);
    active_parity = EVEN;
    total_R_iters=Kalkreuter(eigVec, eigVal, eigenval_tol, 
			     error_decr, Nvecs, MaxIter, Restart, 
			     Kiters);

    node0_printf("The above where eigenvalues of -Dslash^2 in MILC normalization\n");
    node0_printf("Here we also list eigenvalues of iDslash in continuum normalization\n");
    for(i=0;i<Nvecs;i++)
      { 
	if ( eigVal[i] > 0.0 ){ chirality = sqrt(eigVal[i]) / 2.0; }
	else { chirality = 0.0; }
	node0_printf("eigenval(%i): %10g\n",i,chirality) ;
      } 

    tmp = (su3_vector*)malloc(sites_on_node*sizeof(su3_vector));
    for(i=0;i<Nvecs;i++)
      { 
//	if ( eigVal[i] > 10.0*eigenval_tol )
//	  {
	    /* Construct to odd part of the vector.                 *
	     * Note that the true odd part of the eigenvector is    *
	     *  i/sqrt(eigVal) Dslash Psi. But since I only compute *
	     * the chirality the i factor is irrelevant (-i)*i=1!!  */
	    dslash_field(eigVec[i], tmp, ODD, fn[0]) ;
	    FORSOMEPARITY(si,s,ODD){ 
	      scalar_mult_su3_vector( &(tmp[si]),
				      1.0/sqrt(eigVal[i]), 
				      &(eigVec[i][si]) ) ;
	    }
	
//	    measure_chirality(eigVec[i], &chirality, EVENANDODD);
	    /* Here I divide by 2 since the EVEN vector is normalized to
	     * 1. The EVENANDODD vector is normalized to 2. I could have
	     * normalized the EVENANDODD vector to 1 and then not devide
	     * by to.  The measure_chirality routine assumes vectors
	     * normalized to 1.  */
//	    node0_printf("Chirality(%i): %g\n",i,chirality/2) ;
	    measure_chirality(eigVec[i], &chir_ev, EVEN);
	    measure_chirality(eigVec[i], &chir_od, ODD);
	    chirality = (chir_ev + chir_od) / 2.0;
	    node0_printf("Chirality(%i) -- even, odd, total: %10g, %10g, %10g\n",
			 i,chir_ev,chir_od,chirality) ;
//	  }
//	else
//	  {
	    /* This is considered a "zero mode", and treated as such */
//	    measure_chirality(eigVec[i], &chirality, EVEN);
//	    node0_printf("Chirality(%i): %g\n",i,chirality) ;
//	  }
      }
#ifdef EO
    cleanup_dslash_temps();
#endif
    free(tmp);
    /**
       for(i=0;i<Nvecs;i++)
       {
       sprintf(label,"DENSITY(%i)",i) ;
       print_densities(eigVec[i], label, ny/2,nz/2,nt/2, EVEN) ;
       }
    **/
    for(i=0;i<Nvecs;i++)
      free(eigVec[i]) ;
    free(eigVec) ;
    free(eigVal) ;
    invalidate_fermion_links(fn_links);
    fflush(stdout);
    
    node0_printf("RUNNING COMPLETED\n"); fflush(stdout);
    dtime += dclock();
    if(this_node==0){
      printf("Time = %e seconds\n",dtime);
      printf("total_iters = %d\n",total_iters);
      printf("total Rayleigh iters = %d\n",total_R_iters);
#ifdef HISQ_SVD_COUNTER
      printf("hisq_svd_counter = %d\n",hisq_svd_counter);
#endif
    }
    fflush(stdout);
  }
Ejemplo n.º 4
0
int
main( int argc, char **argv )
{
  int meascount,traj_done,i;
  int prompt;
  int s_iters, avs_iters, avspect_iters, avbcorr_iters;
  double dtime, dclock();
  
  initialize_machine(&argc,&argv);
#ifdef HAVE_QDP
  QDP_initialize(&argc, &argv);
#ifndef QDP_PROFILE
  QDP_profcontrol(0);
#endif
#endif
  /* Remap standard I/O */
  if(remap_stdio_from_args(argc, argv) == 1)terminate(1);
  
  g_sync();
  /* set up */
  prompt = setup();

  /* loop over input sets */
  while( readin(prompt) == 0) {
    
    /* perform warmup trajectories */
#ifdef MILC_GLOBAL_DEBUG
    global_current_time_step = 0;
#endif /* MILC_GLOBAL_DEBUG */

    dtime = -dclock();
    for( traj_done=0; traj_done < warms; traj_done++ ){
      update();
    }
    node0_printf("WARMUPS COMPLETED\n"); fflush(stdout);
    
    /* perform measuring trajectories, reunitarizing and measuring 	*/
    meascount=0;		/* number of measurements 		*/
    avspect_iters = avs_iters = avbcorr_iters = 0;

    for( traj_done=0; traj_done < trajecs; traj_done++ ){ 
#ifdef MILC_GLOBAL_DEBUG
#ifdef HISQ_REUNITARIZATION_DEBUG
  {
  int isite, idir;
  site *s;
  FORALLSITES(isite,s) {
    for( idir=XUP;idir<=TUP;idir++ ) {
      lattice[isite].on_step_Y[idir] = 0;
      lattice[isite].on_step_W[idir] = 0;
      lattice[isite].on_step_V[idir] = 0;
    }
  }
  }
#endif /* HISQ_REUNITARIZATION_DEBUG */
#endif /* MILC_GLOBAL_DEBUG */
      /* do the trajectories */
      s_iters=update();

      /* measure every "propinterval" trajectories */
      if( (traj_done%propinterval)==(propinterval-1) ){
	
	/* call gauge_variable fermion_variable measuring routines */
	/* results are printed in output file */
	rephase(OFF);
	g_measure( );
	rephase(ON);
#ifdef MILC_GLOBAL_DEBUG
#ifdef HISQ
        g_measure_plaq( );
#endif
#ifdef MEASURE_AND_TUNE_HISQ
        g_measure_tune( );
#endif /* MEASURE_AND_TUNE_HISQ */
#endif /* MILC_GLOBAL_DEBUG */

	/************************************************************/
	/* WARNING: The spectrum code below is under revision       */
	/* It works only in special cases                           */
	/* For the asqtad spectrum, please create the lattice first */
	/* and then run the appropriate executable in ks_imp_dyn.   */
	/************************************************************/
	/* Do some fermion measurements */
#ifdef SPECTRUM 
	/* Fix TUP Coulomb gauge - gauge links only*/
	rephase( OFF );
	gaugefix(TUP,(Real)1.8,500,(Real)GAUGE_FIX_TOL);
	rephase( ON );

	invalidate_all_ferm_links(&fn_links);
#ifdef DM_DU0
	invalidate_all_ferm_links(&fn_links_dmdu0);
#endif
#endif


	for(i=0;i<n_dyn_masses;i++){
	  // Remake the path table if the fermion coeffs change for this mass
// DT IT CAN"T BE RIGHT TO CALL IT WITH dyn_mass
	  //if(make_path_table(&ks_act_paths, &ks_act_paths_dmdu0,dyn_mass[i]))
//AB: NOT SURE IF WE ARE DOING THIS RIGHT HERE
//    HAVE TO THINK THROUGH HOW LINKS ARE LOADED FOR MEASUREMENTS
//    AND WHERE NAIK CORRECTION CAN EVER POSSIBLY ENTER
//	  if(make_path_table(&ks_act_paths, &ks_act_paths_dmdu0,   0.0/*TEMP*/   ))
	    {
	      // If they change, invalidate only fat and long links
	      //node0_printf("INVALIDATE\n");
	      invalidate_all_ferm_links(&fn_links);
#ifdef DM_DU0
	      invalidate_all_ferm_links(&fn_links_dmdu0);
#endif
	    }
	    /* Load fat and long links for fermion measurements if needed */
#ifdef HISQ
//AB: QUICK FIX TO USE NAIK EPSILON FOR SPECTRUM MEASUREMENTS,
//    WORKS ONLY IF IN THE RATIONAL FUNCTION FILE naik_term_epsilon IS NON-ZERO
//    FOR LAST PSEUDO-FIELD
//    IT IS ASSUMED THAT THIS CORRECTION CORRESPONDS TO LAST DYNAMICAL MASS
//AB: OLD WAY OF INITIALIZING THE LINKS:  fn_links.hl.current_X_set = 0;
//    INSTEAD WE DO:
////	    if(n_dyn_masses-1==i) { // last dynamical mass, assumed to be c-quark
////	      fn_links.hl.current_X_set = n_naiks-1;
//DT CHARM QUARK NEEDS SMALLER RESIDUAL
////	      node0_printf("TEMP: reset rsqprop from %e to %e\n",rsqprop,1e-8*rsqprop);
////	      rsqprop *= 1e-8;
////	    }
////	    else { // light quarks
	      fn_links.hl.current_X_set = 0;
////	    }
#endif
	    load_ferm_links(&fn_links, &ks_act_paths);
#ifdef DM_DU0
#ifdef HISQ
	    fn_links_dmdu0.hl.current_X_set = 0;
#endif
	    load_ferm_links(&fn_links_dmdu0, &ks_act_paths_dmdu0);
#endif
	    
	    f_meas_imp( F_OFFSET(phi1), F_OFFSET(xxx1), dyn_mass[i],
			&fn_links, &fn_links_dmdu0);
	    /* Measure derivatives wrto chemical potential */
#ifdef D_CHEM_POT
	    Deriv_O6( F_OFFSET(phi1), F_OFFSET(xxx1), F_OFFSET(xxx2), 
		      dyn_mass[i], &fn_links, &fn_links_dmdu0);
#endif
	    
#ifdef SPECTRUM 

	    // DT: At the moment spectrum_nd does only the first two masses
	    // this only makes sense to get the kaon, and only works if
	    // eps_naik is the same for both the first two quarks
            if( strstr(spectrum_request,",spectrum_nd,") != NULL && i==0 )
              avspect_iters += spectrum_nd( dyn_mass[0], dyn_mass[1],  1e-2, &fn_links);

	    // AB: spectrum() is used only for the charm quark,
	    // i.e., last dynamical mass
	    if(strstr(spectrum_request,",spectrum,") != NULL && n_dyn_masses-1==i)
	      avspect_iters += spectrum2( dyn_mass[i], F_OFFSET(phi1),
					  F_OFFSET(xxx1), &fn_links);
	    
	    if(strstr(spectrum_request,",spectrum_point,") != NULL)
	      avspect_iters += spectrum_fzw( dyn_mass[i], F_OFFSET(phi1),
					     F_OFFSET(xxx1), &fn_links);
	    
	    // AB: nl_spectrum is used only for strange,
	    // i.e., second mass
	    if(strstr(spectrum_request,",nl_spectrum,") != NULL && 1==i)
	      avspect_iters += nl_spectrum( dyn_mass[i], F_OFFSET(phi1), 
					    F_OFFSET(xxx1), 
					    F_OFFSET(tempmat1),
					    F_OFFSET(staple),
					    &fn_links);
	    
	    // AB: spectrum_mom is used only for charm,
	    // i.e., last mass
	    if(strstr(spectrum_request,",spectrum_mom,") != NULL && n_dyn_masses-1==i)
	      avspect_iters += spectrum_mom( dyn_mass[i], dyn_mass[i], 
					     F_OFFSET(phi1), 1e-1,
					     &fn_links);
	    
	    // For now we can't do the off-diagonal spectrum if Dirac operators
            // depend on masses.  We need two propagators
	    // if(strstr(spectrum_request,",spectrum_multimom,") != NULL)
	    //     avspect_iters += spectrum_multimom(dyn_mass[i],
	    //				 spectrum_multimom_low_mass,
	    //				 spectrum_multimom_mass_step,
	    //				 spectrum_multimom_nmasses,
	    //				 5e-3, &fn_links);

	    // For now we can't do the off-diagonal spectrum if Dirac operators
            // depend on masses.  We need two propagators
	    //	    if(strstr(spectrum_request,",spectrum_nd,") != NULL){
	    //	      avspect_iters += spectrum_nd( mass1, mass2, 1e-1,
	    //					    &fn_links);

	    // AB: spectrum_nlpi2 is used only for up/down,
	    // i.e., first mass
	    if(strstr(spectrum_request,",spectrum_nlpi2,") != NULL && 0==i)
	      avspect_iters += spectrum_nlpi2( dyn_mass[i], dyn_mass[i],
					       F_OFFSET(phi1),1e-1,
					       &fn_links );
	
	    if(strstr(spectrum_request,",spectrum_singlets,") != NULL)
	      avspect_iters += spectrum_singlets(dyn_mass[i], 5e-3, 
						 F_OFFSET(phi1), &fn_links );

	    // For now we can't do the off-diagonal spectrum if Dirac operators
            // depend on masses.  We need two propagators
	    // if(strstr(spectrum_request,",fpi,") != NULL)
	    // avspect_iters += fpi_2( fpi_mass, fpi_nmasses, 2e-3,
	    //			    &fn_links );
	
#ifdef HYBRIDS
	  if(strstr(spectrum_request,",spectrum_hybrids,") != NULL)
	    avspect_iters += spectrum_hybrids( dyn_mass[i], F_OFFSET(phi1), 
					       5e-3, &fn_links);
#endif
	  if(strstr(spectrum_request,",hvy_pot,") != NULL){
	    rephase( OFF );
	    hvy_pot( F_OFFSET(link[XUP]) );
	    rephase( ON );
	  }
#endif
//	    if(n_dyn_masses-1==i) { // last dynamical mass, assumed to be c-quark
//DT CHARM QUARK NEEDS SMALLER RESIDUAL
//AB NEED TO RETURN RESIDUAL TO THE ORIGINAL VALUE
//	      node0_printf("TEMP: reset rsqprop from %e to %e\n",rsqprop,1e+8*rsqprop);
//	      rsqprop *= 1e+8;
//	    }
	}
	avs_iters += s_iters;
	++meascount;
	fflush(stdout);
      }
    }	/* end loop over trajectories */
    
    node0_printf("RUNNING COMPLETED\n"); fflush(stdout);
    if(meascount>0)  {
      node0_printf("average cg iters for step= %e\n",
		   (double)avs_iters/meascount);
    }
    
    dtime += dclock();
    if(this_node==0){
      printf("Time = %e seconds\n",dtime);
      printf("total_iters = %d\n",total_iters);
    }
    fflush(stdout);
    
    /* save lattice if requested */
    if( saveflag != FORGET ){
      rephase( OFF );
      save_lattice( saveflag, savefile, stringLFN );
      rephase( ON );
    }
  }
#ifdef HAVE_QDP
  QDP_finalize();
#endif  
  normal_exit(0);
  return 0;
}