Example #1
0
int main(int argc,char *argv[])
{

  char *cname="";
  char *fname="main()";
  Start(&argc,&argv);

  VRB.Result(cname,fname,"Starting next configuration.\n");

  //Initialize Timing
  const int dtime_size=200;
  Float dtime[dtime_size];
  dtime[0]=dclock();

  int chkpoint_no=0;

  CommandLine::is(argc,argv);

  DoArg do_arg;
  EvoArg evo_arg;
  QPropWArg qpropw_arg;


  /* get parameter from command line */

  /* 17 parameters: */
  /* x.x  
	  chkpoints (1 to turn checkpoints on, 0 to turn them off)
          DIRECTORY do_arg evo_arg mass_list
          lat_stand_in lat_stand_out
	  number (which random gauge to take)
	  t_src
	  label id seqNum
	  logdir
	  plaquette_gfix_info_dir
	  midprop_contractions_dir
	  configs_done_log
	  gauge_rotated_lats_dir
	  gauge_rotated_lats_saved_log
	  
 */

  int chkpoints(CommandLine::arg_as_int() );
 
  chdir(CommandLine::arg());
  
  
  if ( !do_arg.Decode(CommandLine::arg(),"do_arg") ) { printf("Bum do_arg\n"); exit(-1);}
  if ( !evo_arg.Decode(CommandLine::arg(),"evo_arg") ) { printf("Bum evo_arg\n"); exit(-1);}
  if ( !mass_list.Decode(CommandLine::arg(),"mass_list") ) { printf("Bum mass_lilst\n"); exit(-1);}
  num_masses = mass_list.Floats.Floats_len;
  masses = mass_list.Floats.Floats_val;

  do_arg.gfix_chkb=1;
  do_arg.cg_reprod_freq=10;

  GJP.Initialize(do_arg);


//  const int num_nodes=do_arg.x_nodes*do_arg.y_nodes*do_arg.z_nodes*do_arg.t_nodes*do_arg.s_nodes;
  const int num_nodes=GJP.Xnodes()*GJP.Ynodes()*GJP.Znodes()*GJP.Tnodes()*GJP.Snodes();

  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  char lat_stand_in[200], lat_stand_out[200];
  char label[200], id[200];
  char logdir[200], plaquette_gfix_info_dir[200], midprop_contractions_dir[200], configs_done_log[200];
  char gauge_rotated_lats_dir[200], gauge_rotated_lats_saved_log[200];

  sprintf(lat_stand_in, CommandLine::arg() );
  sprintf(lat_stand_out, CommandLine::arg() );
  
  int gauge_ran(CommandLine::arg_as_int() );
  int t_src(CommandLine::arg_as_int() );
  if ( t_src<0 || t_src>=GJP.Sites(3) ) {
    printf("t_src must be in the range 0 <= t_src < %d.\n",GJP.Sites(3));
    printf("Abort.\n");
    return 1;
  }
  if ( t_src % GJP.TnodeSites() ) {
    printf("t_src must be at the beginning of a node since we are shifting\n");
    printf("the lattice rather than putting a source at time t_src.\n");
    printf("Abort.\n");
    return 2;
  }

  sprintf(label, CommandLine::arg() );
  sprintf(id, CommandLine::arg() );
  int seqNum(CommandLine::arg_as_int() );

  sprintf(logdir, CommandLine::arg() );
  sprintf(plaquette_gfix_info_dir, CommandLine::arg() );
  sprintf(midprop_contractions_dir, CommandLine::arg() );
  sprintf(configs_done_log, CommandLine::arg() );
  sprintf(gauge_rotated_lats_dir, CommandLine::arg() );
  sprintf(gauge_rotated_lats_saved_log, CommandLine::arg() );
  
  VRB.Result(cname,fname,"This is configuration number %d.\n",seqNum);

  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  const FP_FORMAT outformat ( 	FP_IEEE64BIG);

			     /* one of
				FP_UNKNOWN
				FP_AUTOMATIC
				FP_TIDSP32
				FP_IEEE32
				FP_IEEE32BIG
				FP_IEEE32LITTLE
				FP_IEEE64
				FP_IEEE64BIG
				FP_IEEE64LITTLE
			     */

  
  GwilsonFdwf lattice;


  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  //If lat_stand_in is "none" then generate a random lattice, otherwise load
  //the file lat_stand_in.
  
  if (strcmp(lat_stand_in,"none")==0){
    VRB.Result(cname,fname,"ncreate random gauge - field (taking # %i) \n", gauge_ran);
    
    for(int ii(0); ii < gauge_ran; ++ii)
      lattice.SetGfieldDisOrd();
    
    VRB.Result(cname,fname,"Created random gauge field.\n");
  } 
  else {
    ReadLatticeParallel readLat;
    
    VRB.Result(cname,fname,"  reading: %s (NERSC-format)\n",lat_stand_in);

    readLat.read(lattice,lat_stand_in);

    VRB.Result(cname,fname,"Lattice read.\n");
    
  }

  //If lat_stand_out is specified then save the non-gauge rotated lattice
  //to this file.

  if (strcmp(lat_stand_out,"none")!=0) {
      WriteLatticeParallel writeLat;

      VRB.Result(cname,fname,"  writing: %s (NERSC-format)\n",lat_stand_out);

      writeLat.write(lattice, lat_stand_out, outformat);
  
      VRB.Result(cname,fname,"Lattice written.\n");
  }

  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  //Shift lattice so that time slice t_src moves to t=0
  int t_shift=-t_src/GJP.TnodeSites();
  VRB.Result(cname,fname,"Shifting lattice by %d, command GDS.Set(0,0,0,%d).\n",-t_src,t_shift);
  GDS.Set(0,0,0,t_shift);
  lattice.Shift();
  VRB.Result(cname,fname,"Lattice shifted.\n");
  
  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  FixGaugeArg fix_arg;
  fix_arg.fix_gauge_kind = FIX_GAUGE_COULOMB_T;
  fix_arg.hyperplane_start = 0;
  fix_arg.hyperplane_step = 1;
  fix_arg.hyperplane_num = GJP.Tnodes()*GJP.TnodeSites();
  fix_arg.stop_cond = 1e-8;
  fix_arg.max_iter_num = 10000;

  FILE *fp;

  CommonArg common_arg_gfix;
  char gfix_fname[200];
  sprintf(gfix_fname,"%s/gfix.%d.dat",plaquette_gfix_info_dir,seqNum);
  common_arg_gfix.results=gfix_fname;
  if( (fp = Fopen((char *)(gfix_fname),"w")) == NULL ){
    ERR.FileA("main","main", (char *)(gfix_fname) );
  }
  Fclose(fp);

  CommonArg common_arg_plaq;
  NoArg plaq_arg;
  char plaq_fname[200];
  sprintf(plaq_fname,"%s/plaq.%d.dat",plaquette_gfix_info_dir,seqNum);
  common_arg_plaq.results=plaq_fname;
  if( (fp = Fopen((char *)(plaq_fname),"w")) == NULL ){
    ERR.FileA("main","main", (char *)(plaq_fname) );
  }
  Fclose(fp);

  VRB.Result(cname,fname,"Calculating plaquette.\n");
  AlgPlaq plaq(lattice,&common_arg_plaq,&plaq_arg);
  plaq.run();
  VRB.Result(cname,fname,"Plaquette calculated.\n");

  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  VRB.Result(cname,fname,"Calculating gauge fixing matrices.\n");
  AlgFixGauge fix_gauge(lattice,&common_arg_gfix,&fix_arg);
  fix_gauge.run();
  VRB.Result(cname,fname,"Gauge fixing matrices calculated.\n");

  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  qpropw_arg.x=0;
  qpropw_arg.y=0;
  qpropw_arg.z=0;
  qpropw_arg.t=0; //Shift lattice instead of putting source at t!=0 (see below)
  qpropw_arg.gauge_fix_src=1;
  qpropw_arg.gauge_fix_snk=1;
  qpropw_arg.store_midprop=0;
  qpropw_arg.save_prop=1;
  qpropw_arg.do_half_fermion=0;	
  
  qpropw_arg.cg.max_num_iter=99999;
  qpropw_arg.cg.stop_rsd=1e-10;
  qpropw_arg.cg.true_rsd=1e-10;
  qpropw_arg.cg.RitzMatOper=MAT_HERM;
  qpropw_arg.cg.Inverter=CG;
  qpropw_arg.cg.bicgstab_n=0;

  //See which masses we need to calculate props for
  float mass_read_tmp;
  Float mass_read;
  int bc;
  int mass_done[num_masses][2];
  char logfile[200];
  sprintf(logfile,"%s/%d",logdir,seqNum);
  if( (fp = fopen((char *)(logfile),"r")) != NULL ){
//    ERR.FileA("main","main", (char *)(logfile) );
    for (int i=0; i<num_masses; i++)
      for (int j=0; j<2; j++)
        mass_done[i][j]=0;
    VRB.Result(cname,fname,"Finished masses and boundary conditions as read from log (config no. %d):\n",seqNum);
    while (fscanf(fp,"%f",&mass_read_tmp)>=0){
      mass_read=(Float) mass_read_tmp;
      fscanf(fp,"%d",&bc);
      VRB.Result(cname,fname,"f %d\n",mass_read,bc);
      for (int i=0; i<num_masses; i++) {
        if (fabs(masses[i]-mass_read)<=1e-6)
  	mass_done[i][bc]=1;
      }
    }
    fclose(fp);
  }
  VRB.Result(cname,fname,"Masses and boundary conditions to do (config. no. %d):\n",seqNum);
  for (int i=0; i<num_masses; i++)
    for (bc=0; bc<2; bc++)
      if (!mass_done[i][bc])
	VRB.Result(cname,fname,"%f %d",masses[i],bc);
  
  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);

  //Loop over masses and calculate props
  CommonArg common_arg;
  char prop_name[200], midprop_contractions_fname[200], qio_out[200];
  VRB.Result(cname,fname,"Begin mass loop for configuration number %d.\n",seqNum);
  for (int i=0; i<num_masses; i++) {
    if (mass_done[i][0] && mass_done[i][1]){
      VRB.Result(cname,fname,"Nothing to do mass %f\n",masses[i]);
      continue;
    }
    qpropw_arg.cg.mass=masses[i];
    for (bc=0; bc<2; bc++) {
      if (!mass_done[i][bc]) {
	char bc_type[30], bc_label[30];
	if (bc) {
	  sprintf(bc_type,"antiperiodic");
	  sprintf(bc_label,"APRD");
	} else {
	  sprintf(bc_type,"periodic");
	  sprintf(bc_label,"PRD");
	}
	VRB.Result(cname,fname,"Inverting mass %f, %s boundary conditions, configuration number %d.\n",masses[i],bc_type,seqNum);
	//Checkpoint
	if (chkpoints)
	  chkpt(num_nodes,chkpoint_no,dtime,dtime_size);
	sprintf(prop_name,"32cube_b2.25_Iw_msea0.004_traj%d_m%0.3f_tshift%d_%s",seqNum,masses[i],-t_src,bc_label); //Note that t_src is w.r.t. the unshifted lattice in the file name, but the source time in the qio header will always say 0 because it is equal to qpropw_arg.t
	sprintf(midprop_contractions_fname,"%s/midprop_contractions.%s",midprop_contractions_dir,prop_name);
	if( (fp = Fopen((char *)(midprop_contractions_fname),"w")) == NULL ){
	  ERR.FileA("main","main", (char *)(midprop_contractions_fname) );
	}
	Fclose(fp);
#if TARGET==QCDOC
	sprintf(qio_out,"/pfs/qio_prop.%s",prop_name);
#else
	sprintf(qio_out,"%d",seqNum);
        mkdir(qio_out,0777);
	sprintf(qio_out,"%d/qio_prop.%s",seqNum,prop_name);
#endif
	common_arg.results=midprop_contractions_fname;
	qpropw_arg.file=qio_out;
	qpropw_arg.ensemble_id=id;
	qpropw_arg.ensemble_label=label;
	qpropw_arg.seqNum=seqNum;
	if (bc)
	  GJP.Tbc(BND_CND_APRD);
	else
	  GJP.Tbc(BND_CND_PRD);
	QPropWWallSrc propagator(lattice, &qpropw_arg, &common_arg);
	VRB.Result(cname,fname,"Finished inversion, mass %f, %s boundary conditions, configuration number %d.\n",masses[i],bc_type,seqNum);
	//Checkpoint
	if (chkpoints)
	  chkpt(num_nodes,chkpoint_no,dtime,dtime_size);
	if( (fp = Fopen((char *)(logfile),"a")) == NULL ){
	  ERR.FileA("main","main", (char *)(logfile) );
	}
	Fprintf(fp,"%0.3f %d\n",masses[i],bc);
	Fclose(fp);
      }
    }
  }
  VRB.Result(cname,fname,"Finished mass loop for configuration number %d.\n",seqNum);

  //Gauge rotate and save the lattice if this hasn't been done already
  //First check if the gauge rotated lattice has already been saved
  if( (fp = fopen((char *)(gauge_rotated_lats_saved_log),"r")) == NULL ){
    ERR.FileA("main","main", (char *)(gauge_rotated_lats_saved_log) );
  }
  int config_no_read;
  int config_saved=0;
  while (fscanf(fp,"%d",&config_no_read)>=0){
    if (config_no_read==seqNum){
      config_saved=1;
      break;
    }
  }
  fclose(fp);
  //If it hasn't then gauge rotate the lattice and save it
  if (!config_saved){
    VRB.Result(cname,fname,"Gauge rotating the lattice and saving it.\n");
    //Gauge rotate lattice
    VRB.Result(cname,fname,"Gauge rotating lattice.\n");
    rotate_gauge_explicit(lattice);
    VRB.Result(cname,fname,"Gauge rotated lattice.\n");
    //Shift the lattice back
    VRB.Result(cname,fname,"Shifting lattice back.  Shifting by %d, command GDS.Set(0,0,0,%d).\n",t_src,-t_shift);
    GDS.Set(0,0,0,-t_shift);
    lattice.Shift();
    VRB.Result(cname,fname,"Lattice shifted.\n");
    //Save it
    char rotated_lat_fname[200];
    sprintf(rotated_lat_fname,"%s/ckpoint_lat_gauge_rotated.%d",gauge_rotated_lats_dir,seqNum);
    VRB.Result(cname,fname,"Writing gauge rotated lattice.\n");
    WriteLatticeParallel writeLat;
    writeLat.write(lattice, rotated_lat_fname, outformat);
    VRB.Result(cname,fname,"Wrote gauge rotated lattice.\n");
    VRB.Result(cname,fname,"Finished gauge rotating and saving lattice.\n");
    //Checkpoint
    if (chkpoints)
      chkpt(num_nodes,chkpoint_no,dtime,dtime_size);
    //Record in log that this lattice has been saved
    if( (fp = Fopen((char *)(gauge_rotated_lats_saved_log),"a")) == NULL ){
    ERR.FileA("main","main", (char *)(gauge_rotated_lats_saved_log) );
    }
    Fprintf(fp,"%d\n",seqNum);
    Fclose(fp);
  }

  if( (fp = Fopen((char *)(configs_done_log),"a")) == NULL ){
    ERR.FileA("main","main", (char *)(configs_done_log ) );
  }
  Fprintf(fp,"%d\n",seqNum);
  Fclose(fp);


  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);
  

  fix_gauge.free();

  VRB.Result(cname,fname,"Finished configuration number %d.\n",seqNum);
  //Checkpoint
  if (chkpoints)
    chkpt(num_nodes,chkpoint_no,dtime,dtime_size);


  return 0;

}
Example #2
0
int main(int argc, char *argv[]){

  Start(&argc,&argv);

  char plaq_file[256];
  char top_file[256];
  char pbp_file[256];
  char hmc_file[256];

  char *cname=argv[0];
  char *fname="main()";
  Float dtime;

  CommonArg common_arg_plaq;
  CommonArg common_arg_pbp;
  CommonArg common_arg_hmc;
  CommonArg common_arg_top;

  WspectArg w_spect_arg;
  CgArg w_spect_cg_arg;
  WspectOutput w_spect_output;
  FixGaugeArg w_spect_fg_arg;

  if ( argc!=19 ) { 
	if(!UniqueID()){
    printf("Args:\t do_arg.vml hmc_arg.vml evo_arg.vml \n");
    printf("\t gauge_arg.vml bsn_arg.vml frm_arg.vml \n");
    printf("\t quo_tm_arg.vml quo_arg.vml rat_quo_arg.vml \n");
    printf("\t ab1_arg.vml ab2_arg.vml ab3_arg.vml ab4_arg.vml \n");
    printf("\t w_spect_arg.vml w_spect_cg_arg.vml w_spect_mass_list.vml \n");
    printf("\t pbp_arg.vml current_dir \n");
	}
    exit(-1);
  }

  chdir (argv[18]);
// these from Michael (?) and I added one "/"
//gauge_Unload
//gauge_unload



  if ( !do_arg.Decode(argv[1],"do_arg") ) { 
    do_arg.Encode("bum_arg","bum_arg"); printf("Bum do_arg\n"); exit(-1); }
  do_arg.Encode("do_arg.dat", "do_arg");
  if ( !hmc_arg.Decode(argv[2],"hmc_arg")){printf("Bum hmc_arg\n"); exit(-1);}
  hmc_arg.Encode("hmc_arg.dat","hmc_arg");
  if ( !evo_arg.Decode(argv[3],"evo_arg")){printf("Bum evo_arg\n"); exit(-1);}
  evo_arg.Encode("evo_arg.dat","evo_arg");

  if ( !gauge_arg.Decode(argv[4],"gauge_arg")){printf("Bum gauge_arg\n"); exit(-1);}
  gauge_arg.Encode("gauge_arg.dat","gauge_arg");
  if ( !bsn_arg.Decode(argv[5],"bsn_arg")){printf("Bum bsn_arg\n"); exit(-1);}
  bsn_arg.Encode("bsn_arg.dat","bsn_arg");
  if ( !frm_arg.Decode(argv[6],"frm_arg")){printf("Bum frm_arg\n"); exit(-1);}
  frm_arg.Encode("frm_arg.dat","frm_arg");

  if ( !quo_tm_arg.Decode(argv[7],"quo_tm_arg")){printf("Bum quo_tm_arg\n"); exit(-1);}
  quo_tm_arg.Encode("quo_tm_arg.dat","quo_tm_arg");
  if ( !quo_arg.Decode(argv[8],"quo_arg")){printf("Bum quo_arg\n"); exit(-1);}
  quo_arg.Encode("quo_arg.dat","quo_arg");
  if ( !rat_quo_arg.Decode(argv[9],"rat_quo_arg")){printf("Bum rat_quo_arg\n"); exit(-1);}
  rat_quo_arg.Encode("rat_quo_arg.dat","rat_quo_arg");

  if ( !ab1_arg.Decode(argv[10],"ab1_arg")){printf("Bum ab1_arg\n"); exit(-1);}
  ab1_arg.Encode("ab1_arg.dat","ab1_arg");
  if ( !ab2_arg.Decode(argv[11],"ab2_arg")){printf("Bum ab2_arg\n"); exit(-1);}
  ab2_arg.Encode("ab2_arg.dat","ab2_arg");
  if ( !ab3_arg.Decode(argv[12],"ab3_arg")){printf("Bum ab3_arg\n"); exit(-1);}
  ab3_arg.Encode("ab3_arg.dat","ab3_arg");
  if ( !ab4_arg.Decode(argv[13],"ab4_arg")){printf("Bum ab4_arg\n"); exit(-1);}
  ab4_arg.Encode("ab4_arg.dat","ab4_arg");

  if ( !pbp_arg.Decode(argv[14],"pbp_arg")){printf("Bum pbp_arg\n"); exit(-1);}
  pbp_arg.Encode("pbp_arg.dat","pbp_arg");
  if ( !w_spect_arg.Decode(argv[15],"w_spect_arg") )
    { printf("Bum w_spect_arg\n"); exit(-1);}   
  if ( !w_spect_cg_arg.Decode(argv[16],"w_spect_cg_arg") )
    { printf("Bum w_spect_cg_arg\n"); exit(-1);}
  if ( !w_spect_mass_list.Decode(argv[17],"w_spect_mass_list") )
    { printf("Bum w_spect_mass_list\n"); exit(-1);}

//  ApeSmearArg ape_arg("ape_arg.vml");
  ApeSmearArg ape_arg;
  if ( !ape_arg.Decode("ape_arg.vml","ape_arg") )
    { printf("Bum ape_arg\n"); exit(-1);}
  ape_arg.Encode("ape_arg.dat","ape_arg");

  chdir(evo_arg.work_directory);

  // do_arg.verbose_level=VERBOSE_RESULT_LEVEL;
  GJP.Initialize(do_arg);
  //VRB.Level(VERBOSE_FUNC_LEVEL);
  VRB.Level(3);
  LRG.Initialize();

  // Outer config loop
  int traj = evo_arg.traj_start;
  int w_int = evo_arg.measure_w_spect_interval;
  int g_int = evo_arg.gauge_unload_period;

  if( w_int>0 && g_int>0 && w_int%g_int !=0){
    ERR.General("", "main()", "w spec meas int(%d) not divisible by Gauge storing int(%d) \n",w_int,g_int);
  }

  if ( do_arg.start_conf_kind != START_CONF_FILE ) {
    checkpoint(traj);
  }

  hmc_arg_pass = hmc_arg;
  sum_arg.A_steps = 1;
  sum_arg.B_steps = 1;

  VRB.Result(cname,fname,"\n\n\n ******************* start create ACTIONS & INTEGRATORS\n\n\n");
  //!< Create fictitous Hamiltonian (mom + action)

  // must precede all AlgAction allocations
   AlgMomentum mom;
   AlgActionGauge gauge(mom, gauge_arg);
   AlgActionQuotient quotient(mom, quo_arg);
   AlgActionQuotient quotient_tm(mom, quo_tm_arg);
   AlgActionFermion Wfermion(mom, frm_arg);
   AlgActionRationalQuotient rat_quo(mom, rat_quo_arg);

   //!< Construct numerical integrators
   //  embedded integrators:  ab1_arg, ab2_arg, ab3_arg
   //  top-level integrator:  ab4_arg

    AlgIntAB &ab1 = AlgIntAB::Create(mom, gauge,    ab1_arg);
    AlgIntAB &ab2 = AlgIntAB::Create(ab1, quotient_tm,  ab2_arg);
    AlgIntAB &ab3 = AlgIntAB::Create(ab2, rat_quo, ab3_arg);
    AlgIntAB &ab4 = AlgIntAB::Create(ab3, quotient, ab4_arg);


  VRB.Result(cname,fname,"\n\n\n ******************* end create ACTIONS & INTEGRATORS\n\n\n");  

  for(int conf=0; conf< evo_arg.gauge_configurations; conf ++ ) {

    sprintf(plaq_file,"%s.%d",evo_arg.plaquette_stem,traj);
    FILE * truncate_it = Fopen(plaq_file,"w");
    Fclose(truncate_it);
    common_arg_plaq.set_filename(plaq_file);

    sprintf(top_file,"%s.%d","../results/alg_top/top",traj);
    truncate_it = Fopen(top_file,"w");
    Fclose(truncate_it);
    common_arg_top.set_filename(top_file);

    sprintf(pbp_file,"%s.%d",evo_arg.pbp_stem,traj);
	//    printf("pbp_file=%s\n",pbp_file);
    truncate_it = Fopen(pbp_file,"w");
    Fclose(truncate_it);
    common_arg_pbp.set_filename(pbp_file);
    pbp_arg.src_u_s = 0;
    pbp_arg.src_l_s = GJP.SnodeSites() * GJP.Snodes() - 1;
    pbp_arg.snk_u_s = GJP.SnodeSites() * GJP.Snodes() - 1;
    pbp_arg.snk_l_s = 0;

    sprintf(hmc_file,"%s.%d",evo_arg.evo_stem,traj);
    common_arg_hmc.set_filename(hmc_file);

    LRGState rng_state;
    // Inner trajectory loop
    for(int i=0;i<g_int;i++,traj++ ) {
    		
	// Wilson gauge action used for plaquette measurement
	VRB.Result(cname,fname,"\n\n\n ******************* start PLAQUETTE\n\n\n");
	Lattice &lat = LatticeFactory::Create(F_CLASS_NONE, G_CLASS_WILSON);
	AlgPlaq plaq(lat, &common_arg_plaq, &no_arg);
  	plaq.run();
	LatticeFactory::Destroy();
	VRB.Result(cname,fname,"\n\n\n ******************* end PLAQUETTE\n\n\n");

	// calculate the topological charge. Need to copy the lattice since
	// we need to smear it first. Use Chulwoo's "lattice container"
	{
	  Lattice &lat = LatticeFactory::Create(F_CLASS_NONE, G_CLASS_WILSON);
	  LatticeContainer lat_cont;
	  // copy lattice
	  lat_cont.Get(lat);
	  //----- mess up lattice -------------------------
	  //AlgPlaq plaq(lat, &common_arg_top, &no_arg);
	  //plaq.run();
	  AlgApeSmear ape(lat,&common_arg_top,&ape_arg);
	  AlgTcharge  tcharge(lat, &common_arg_top);
	  for (int i(0);i<20;i++)
	    {
	      VRB.Result(cname,fname,"%i\n",i);
	      VRB.Result(cname,fname,"   running tcharge\n"); tcharge.run(); 
	      VRB.Result(cname,fname,"   running ape\n"); ape.run();
	      VRB.Result(cname,fname,"   running ape\n"); ape.run();
	      VRB.Result(cname,fname,"   running ape\n"); ape.run();
	    }
	  tcharge.run();
	  // restore the lattice
	  lat_cont.Set(lat);
	  //plaq.run();
	  LatticeFactory::Destroy();
	}
	if ((traj%g_int == 0) && evo_arg.measure_pbp){
	    dtime = -dclock();
	    VRB.Result(cname,fname,"Running pbp\n");
	    VRB.Result(cname,fname,"\n\n\n ******************* start PBP\n\n\n");
	    rng_state.GetStates();
	    Lattice &lat = LatticeFactory::Create(F_CLASS_DWF, G_CLASS_WILSON);
	    AlgPbp pbp(lat,&common_arg_pbp,&pbp_arg);
	    for(int pbp_counter = 0; pbp_counter < evo_arg.measure_pbp; pbp_counter++) {
	    	pbp.run();
	    }
	    LatticeFactory::Destroy();
	    rng_state.SetStates();
	    VRB.Result(cname,fname,"\n\n\n ******************* end PBP\n\n\n");
	    dtime += dclock();
	    print_flops("AlgPbp","",0,dtime);	
	}
	
	if ( (evo_arg.reproduce_interval > 0) &&
	     (traj % evo_arg.reproduce_interval) == 0 ) { 
		VRB.Result(cname,fname,"Running traj %d with reproduction\n",traj);
		hmc_arg_pass.reproduce = REPRODUCE_YES;
	} else { 
		VRB.Result(cname,fname,"Running traj %d without reproduction\n",traj);
		hmc_arg_pass.reproduce = REPRODUCE_NO;	
	}

	//!< Run hybrid Monte Carlo
#if 0
        VRB.Result(cname,fname,"\n\n\n ******************* start HMC\n\n\n");
        if ( (traj > 19) && (hmc_arg_pass.metropolis == METROPOLIS_NO)) { 
           VRB.Result(cname,fname,"Switching to ACCECPT/REJECT at traj %d \n",traj);
           hmc_arg_pass.metropolis = METROPOLIS_YES;
        }
#endif
	VRB.Result(cname,fname,"\n\n\n ******************* start HMC\n\n\n");
       	AlgHmc hmc(ab4, common_arg_hmc, hmc_arg_pass);
	Float time = -dclock();
	hmc.run();
	VRB.Result(cname,fname,"\n\n\n ******************* end HMC\n\n\n");
	time += dclock();
	print_flops("AlgHmc","run()",0,time);
	    
    }//End of inter-cfg sweep

    //if (evo_arg.measure_w_spect_interval){
    if ((w_int>0) && (traj%w_int==0)){

      dtime = -dclock();
      VRB.Result(cname,fname,"Measuring hadron masses\n");
      char filenames[32][MAX_FILENAME];
      char suffix[MAX_FILENAME];
      CommonArg common_arg;
      CommonArg common_arg_fg;
      char *w_dir = evo_arg.w_spect_directory;
  
      Lattice &lat = LatticeFactory::Create(F_CLASS_DWF, G_CLASS_NONE);
      rng_state.GetStates();

      // some hard coding of VML-lized input parameters.
      w_spect_arg.extended_mesons_on=0;
      w_spect_output.fold = BARYON_RAW;
      w_spect_fg_arg.stop_cond = 1e-8;
      w_spect_fg_arg.max_iter_num = 20000;
      w_spect_fg_arg.hyperplane_start = 0;
      w_spect_fg_arg.hyperplane_step = 1;
      int dir = w_spect_arg.prop_dir;
      w_spect_fg_arg.hyperplane_num = GJP.NodeSites(dir) * GJP.Nodes(dir);

      switch (dir) {
        case 0  : w_spect_fg_arg.fix_gauge_kind = FIX_GAUGE_COULOMB_X ; break ;
        case 1  : w_spect_fg_arg.fix_gauge_kind = FIX_GAUGE_COULOMB_Y ; break ;
        case 2  : w_spect_fg_arg.fix_gauge_kind = FIX_GAUGE_COULOMB_Z ; break ;
        case 3  : w_spect_fg_arg.fix_gauge_kind = FIX_GAUGE_COULOMB_T ; break ;
        default : w_spect_fg_arg.fix_gauge_kind = FIX_GAUGE_NONE      ;
      }
  
  
      int n_mf = w_spect_mass_list.Floats.Floats_len;
      Float *mf = w_spect_mass_list.Floats.Floats_val;
  
      for (int mf_ind=0;mf_ind <n_mf; mf_ind++){
        
        sprintf(suffix,"_mf%0.3f",mf[mf_ind]);
        sprintf(filenames[0],"%s/scalar%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[1],"%s/vector_x%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[2],"%s/vector_y%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[3],"%s/tensor_xy%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[4], "%s/vector_z%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[5], "%s/tensor_xz%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[6], "%s/tensor_yz%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[7], "%s/axial_vector_t%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[8], "%s/vector_t%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[9], "%s/tensor_xt%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[10], "%s/tensor_yt%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[11], "%s/axial_vector_z%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[12], "%s/tensor_zt%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[13], "%s/axial_vector_y%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[14], "%s/axial_vector_x%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[15], "%s/pseudoscalar%s.dat.%d",w_dir,suffix,traj);
  
        w_spect_output.meson_name00            = &(filenames[0][0]) ;
        w_spect_output.meson_name01            = &(filenames[1][0]) ;
        w_spect_output.meson_name02            = &(filenames[2][0]) ;
        w_spect_output.meson_name03            = &(filenames[3][0]) ;
        w_spect_output.meson_name04            = &(filenames[4][0]) ;
        w_spect_output.meson_name05            = &(filenames[5][0]) ;
        w_spect_output.meson_name06            = &(filenames[6][0]) ;
        w_spect_output.meson_name07            = &(filenames[7][0]) ;
        w_spect_output.meson_name08            = &(filenames[8][0]) ;
        w_spect_output.meson_name09            = &(filenames[9][0]) ;
        w_spect_output.meson_name10            = &(filenames[10][0]) ;
        w_spect_output.meson_name11            = &(filenames[11][0]) ;
        w_spect_output.meson_name12            = &(filenames[12][0]) ;
        w_spect_output.meson_name13            = &(filenames[13][0]) ;
        w_spect_output.meson_name14            = &(filenames[14][0]) ;
        w_spect_output.meson_name15            = &(filenames[15][0]) ;
    
        int ind_end=16;
        sprintf(filenames[ind_end++],"%s/cg%s.dat.%d" ,w_dir,suffix,traj);
	//        sprintf(filenames[ind_end++],"%s/cg2%s.dat.%d" ,w_dir,suffix,traj);
	//        sprintf(filenames[ind_end++],"%s/pbp%s.dat.%d" ,w_dir,suffix,traj);
        sprintf(filenames[ind_end++],"%s/mid_point%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[ind_end++], "%s/a0%s.dat.%d",w_dir,suffix,traj);
	//        sprintf(filenames[ind_end++], "%s/a1%s.dat.%d",w_dir,suffix,traj);
	//        sprintf(filenames[ind_end++], "%s/b1%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[ind_end++], "%s/nucleon%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[ind_end++], "%s/nucleon_prime%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[ind_end++], "%s/delta_x%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[ind_end++], "%s/delta_y%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[ind_end++], "%s/delta_z%s.dat.%d",w_dir,suffix,traj);
        sprintf(filenames[ind_end++], "%s/delta_t%s.dat.%d",w_dir,suffix,traj);
	//        sprintf(filenames[ind_end++], "%s/rho%s.dat.%d",w_dir,suffix,traj);
	//        sprintf(filenames[ind_end++], "%s/rho_prime%s.dat.%d",w_dir,suffix,traj);
	
        ind_end=16;
        //-------------------------------------------------
        //cg,nucleon and other stuff
        //-------------------------------------------------
        w_spect_output.cg            = &(filenames[ind_end++][0]) ;
	//        w_spect_output.cg2           = &(filenames[ind_end++][0]) ;
	//        w_spect_output.pbp           = &(filenames[ind_end++][0]) ;
        w_spect_output.mid_point     = &(filenames[ind_end++][0]) ;
        w_spect_output.a0_p          = &(filenames[ind_end++][0]) ;
	//        w_spect_output.a1            = &(filenames[ind_end++][0]) ;
	//        w_spect_output.b1            = &(filenames[ind_end++][0]) ;
        w_spect_output.nucleon       = &(filenames[ind_end++][0]) ;
        w_spect_output.nucleon_prime = &(filenames[ind_end++][0]) ;
        w_spect_output.delta_x       = &(filenames[ind_end++][0]) ;
        w_spect_output.delta_y       = &(filenames[ind_end++][0]) ;
        w_spect_output.delta_z       = &(filenames[ind_end++][0]) ;
        w_spect_output.delta_t       = &(filenames[ind_end++][0]) ;
	//        w_spect_output.rho           = &(filenames[ind_end++][0]) ;
	//        w_spect_output.rho_prime     = &(filenames[ind_end++][0]) ;
	
        for(int ind=0;ind<ind_end;ind++){
          filenames[ind][MAX_FILENAME-1]='\0';
          truncate_it = Fopen(filenames[ind],"w");
          Fclose(truncate_it);
        }
	
        common_arg.results = (void *) &w_spect_output ;
        w_spect_cg_arg.mass = mf[mf_ind] ;
	
        if ( ( w_spect_arg.source_kind == BOX_W ) &&	  \
	     ( w_spect_arg.src_box_e[w_spect_arg.prop_dir] !=	\
	       w_spect_arg.src_box_b[w_spect_arg.prop_dir] ) )
          ERR.General(cname, fname, "Invalid box source.\n") ;
	
        if ( (w_spect_arg.source_kind == BOX_W)		\
	     || (w_spect_arg.source_kind == WALL_W) ) {
          AlgFixGauge fg(lat, &common_arg_fg, &w_spect_fg_arg) ;
          fg.run() ;
        }
        AlgWspect ws(lat,&common_arg,&w_spect_arg,&w_spect_cg_arg);
	
        ws.SetCounter(traj, 0) ;
        
        ws.run();
        
        //------------------------------------------------
        // free gauge fixing matrices, if needed.
        //------------------------------------------------
        
        if ( (w_spect_arg.source_kind == BOX_W)		\
	     || (w_spect_arg.source_kind == WALL_W) ) {
          AlgFixGauge fg(lat, &common_arg_fg, &w_spect_fg_arg) ;
          fg.free() ;
        }
      }
      dtime += dclock();
      print_flops("AlgWspect","",0,dtime);
      rng_state.SetStates();
      LatticeFactory::Destroy();
    }// end w spect meas

    checkpoint(traj);
    
  } //End config loop
  
  AlgIntAB::Destroy(ab4);
  AlgIntAB::Destroy(ab3);
  AlgIntAB::Destroy(ab2);
  AlgIntAB::Destroy(ab1);
  
  End();

  return(0);
}