int update_tm(double *plaquette_energy, double *rectangle_energy, char * filename, const int return_check, const int acctest, const int traj_counter) { su3 *v, *w; int accept, i=0, j=0, iostatus=0; double yy[1]; double dh, expmdh, ret_dh=0., ret_gauge_diff=0., tmp; double atime=0., etime=0.; double ks = 0., kc = 0., ds, tr, ts, tt; char tmp_filename[50]; /* Energy corresponding to the Gauge part */ double new_plaquette_energy=0., new_rectangle_energy = 0.; /* Energy corresponding to the Momenta part */ double enep=0., enepx=0., ret_enep = 0.; /* Energy corresponding to the pseudo fermion part(s) */ FILE * datafile=NULL, * ret_check_file=NULL; hamiltonian_field_t hf; paramsXlfInfo *xlfInfo; hf.gaugefield = g_gauge_field; hf.momenta = moment; hf.derivative = df0; hf.update_gauge_copy = g_update_gauge_copy; hf.traj_counter = traj_counter; integrator_set_fields(&hf); sprintf(tmp_filename, ".conf.t%05d.tmp",traj_counter); atime = gettime(); /* * here the momentum and spinor fields are initialized * and their respective actions are calculated */ /* * copy the gauge field to gauge_tmp */ #ifdef TM_USE_OMP #pragma omp parallel for private(w,v) #endif for(int ix=0;ix<VOLUME;ix++) { for(int mu=0;mu<4;mu++) { v=&hf.gaugefield[ix][mu]; w=&gauge_tmp[ix][mu]; _su3_assign(*w,*v); } } #ifdef DDalphaAMG MG_reset(); #endif /* heatbath for all monomials */ for(i = 0; i < Integrator.no_timescales; i++) { for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) { monomial_list[ Integrator.mnls_per_ts[i][j] ].hbfunction(Integrator.mnls_per_ts[i][j], &hf); } } if(Integrator.monitor_forces) monitor_forces(&hf); /* initialize the momenta */ enep = random_su3adj_field(reproduce_randomnumber_flag, hf.momenta); g_sloppy_precision = 1; /* run the trajectory */ if(Integrator.n_int[Integrator.no_timescales-1] > 0) { Integrator.integrate[Integrator.no_timescales-1](Integrator.tau, Integrator.no_timescales-1, 1); } g_sloppy_precision = 0; /* compute the final energy contributions for all monomials */ dh = 0.; for(i = 0; i < Integrator.no_timescales; i++) { for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) { dh += monomial_list[ Integrator.mnls_per_ts[i][j] ].accfunction(Integrator.mnls_per_ts[i][j], &hf); } } enepx = moment_energy(hf.momenta); if (!bc_flag) { /* if PBC */ new_plaquette_energy = measure_plaquette( (const su3**) hf.gaugefield); if(g_rgi_C1 > 0. || g_rgi_C1 < 0.) { new_rectangle_energy = measure_rectangles( (const su3**) hf.gaugefield); } } if(g_proc_id == 0 && g_debug_level > 3) printf("called moment_energy: dh = %1.10e\n", (enepx - enep)); /* Compute the energy difference */ dh = dh + (enepx - enep); if(g_proc_id == 0 && g_debug_level > 3) { printf("called momenta_acc dH = %e\n", (enepx - enep)); } expmdh = exp(-dh); /* the random number is only taken at node zero and then distributed to the other sites */ ranlxd(yy,1); #ifdef TM_USE_MPI MPI_Bcast(&yy[0], 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); #endif /* when acctest is 0 (i.e. do not perform acceptance test), the trajectory is accepted whatever the energy difference */ accept = (!acctest | (expmdh > yy[0])); if(g_proc_id == 0) { fprintf(stdout, "# Trajectory is %saccepted.\n", (accept ? "" : "not ")); } /* Here a reversibility test is performed */ /* The trajectory is integrated back */ if(return_check) { if(g_proc_id == 0) { fprintf(stdout, "# Performing reversibility check.\n"); } if(accept) { /* save gauge file to disk before performing reversibility check */ xlfInfo = construct_paramsXlfInfo((*plaquette_energy)/(6.*VOLUME*g_nproc), traj_counter); // Should write this to temporary file first, and then check if(g_proc_id == 0 && g_debug_level > 0) { fprintf(stdout, "# Writing gauge field to file %s.\n", tmp_filename); } if((iostatus = write_gauge_field( tmp_filename, 64, xlfInfo) != 0 )) { /* Writing failed directly */ fprintf(stderr, "Error %d while writing gauge field to %s\nAborting...\n", iostatus, tmp_filename); exit(-2); } /* There is double writing of the gauge field, also in hmc_tm.c in this case */ /* No reading back check needed here, as reading back is done further down */ if(g_proc_id == 0 && g_debug_level > 0) { fprintf(stdout, "# Writing done.\n"); } free(xlfInfo); } #ifdef DDalphaAMG MG_reset(); #endif g_sloppy_precision = 1; /* run the trajectory back */ Integrator.integrate[Integrator.no_timescales-1](-Integrator.tau, Integrator.no_timescales-1, 1); g_sloppy_precision = 0; /* compute the energy contributions from the pseudo-fermions */ ret_dh = 0.; for(i = 0; i < Integrator.no_timescales; i++) { for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) { ret_dh += monomial_list[ Integrator.mnls_per_ts[i][j] ].accfunction(Integrator.mnls_per_ts[i][j], &hf); } } ret_enep = moment_energy(hf.momenta); /* Compute the energy difference */ ret_dh += ret_enep - enep ; /* Compute Differences in the fields */ ks = 0.; kc = 0.; #ifdef TM_USE_OMP #pragma omp parallel private(w,v,tt,tr,ts,ds,ks,kc) { int thread_num = omp_get_thread_num(); #endif su3 ALIGN v0; #ifdef TM_USE_OMP #pragma omp for #endif for(int ix = 0; ix < VOLUME; ++ix) { for(int mu = 0; mu < 4; ++mu) { v=&hf.gaugefield[ix][mu]; w=&gauge_tmp[ix][mu]; _su3_minus_su3(v0, *v, *w); _su3_square_norm(ds, v0); tr = sqrt(ds) + kc; ts = tr + ks; tt = ts-ks; ks = ts; kc = tr-tt; } } kc=ks+kc; #ifdef TM_USE_OMP g_omp_acc_re[thread_num] = kc; } /* OpenMP parallel section closing brace */ /* sum up contributions from thread-local kahan summations */ for(int k = 0; k < omp_num_threads; ++k) ret_gauge_diff += g_omp_acc_re[k]; #else ret_gauge_diff = kc; #endif #ifdef TM_USE_MPI tmp = ret_gauge_diff; MPI_Reduce(&tmp, &ret_gauge_diff, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); #endif /* compute the total H */ tmp = enep; for(i = 0; i < Integrator.no_timescales; i++) { for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) { tmp += monomial_list[ Integrator.mnls_per_ts[i][j] ].energy0; } } /* Output */ if(g_proc_id == 0) { ret_check_file = fopen("return_check.data","a"); fprintf(ret_check_file,"%08d ddh = %1.4e ddh/dh = %1.4e ddh/H = %1.4e ddU= %1.4e\n", traj_counter, ret_dh, ret_dh/dh, ret_dh/tmp, ret_gauge_diff/4./((double)(VOLUME*g_nproc))/3.); fclose(ret_check_file); } if(accept) { /* Read back gauge field FIXME unlike in hmc_tm we abort immediately if there is a failure */ if(g_proc_id == 0 && g_debug_level > 0) { fprintf(stdout, "# Trying to read gauge field from file %s.\n", tmp_filename); } if((iostatus = read_gauge_field(tmp_filename,g_gauge_field) != 0)) { fprintf(stderr, "Error %d while reading gauge field from %s\nAborting...\n", iostatus, tmp_filename); exit(-2); } if(g_proc_id == 0 && g_debug_level > 0) { fprintf(stdout, "# Reading done.\n"); } } if(g_proc_id == 0) { fprintf(stdout, "# Reversibility check done.\n"); } } /* end of reversibility check */ if(accept) { *plaquette_energy = new_plaquette_energy; *rectangle_energy = new_rectangle_energy; /* put the links back to SU(3) group */ if (!bc_flag) { /* periodic boundary conditions */ #ifdef TM_USE_OMP #pragma omp parallel for private(v) #endif for(int ix=0;ix<VOLUME;ix++) { for(int mu=0;mu<4;mu++) { v=&hf.gaugefield[ix][mu]; restoresu3_in_place(v); } } } } else { /* reject: copy gauge_tmp to hf.gaugefield */ #ifdef TM_USE_OMP #pragma omp parallel for private(w) private(v) #endif for(int ix=0;ix<VOLUME;ix++) { for(int mu=0;mu<4;mu++){ v=&hf.gaugefield[ix][mu]; w=&gauge_tmp[ix][mu]; _su3_assign(*v,*w); } } } hf.update_gauge_copy = 1; g_update_gauge_copy = 1; g_update_gauge_copy_32 = 1; #ifdef TM_USE_MPI xchange_gauge(hf.gaugefield); #endif /*Convert to a 32 bit gauge field, after xchange*/ convert_32_gauge_field(g_gauge_field_32, hf.gaugefield, VOLUMEPLUSRAND + g_dbw2rand); etime=gettime(); /* printing data in the .data file */ if(g_proc_id==0) { datafile = fopen(filename, "a"); if (!bc_flag) { /* if Periodic Boundary Conditions */ fprintf(datafile, "%.8d %14.12f %14.12f %e ", traj_counter, (*plaquette_energy)/(6.*VOLUME*g_nproc), dh, expmdh); } for(i = 0; i < Integrator.no_timescales; i++) { for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) { if(monomial_list[ Integrator.mnls_per_ts[i][j] ].type != GAUGE && monomial_list[ Integrator.mnls_per_ts[i][j] ].type != SFGAUGE && monomial_list[ Integrator.mnls_per_ts[i][j] ].type != NDPOLY && monomial_list[ Integrator.mnls_per_ts[i][j] ].type != NDCLOVER && monomial_list[ Integrator.mnls_per_ts[i][j] ].type != CLOVERNDTRLOG && monomial_list[ Integrator.mnls_per_ts[i][j] ].type != CLOVERTRLOG ) { fprintf(datafile,"%d %d ", monomial_list[ Integrator.mnls_per_ts[i][j] ].iter0, monomial_list[ Integrator.mnls_per_ts[i][j] ].iter1); } } } fprintf(datafile, "%d %e", accept, etime-atime); if(g_rgi_C1 > 0. || g_rgi_C1 < 0) { fprintf(datafile, " %e", (*rectangle_energy)/(12*VOLUME*g_nproc)); } fprintf(datafile, "\n"); fflush(datafile); fclose(datafile); } return(accept); }
int main(int argc, char *argv[]) { FILE *parameterfile = NULL; int j, i, ix = 0, isample = 0, op_id = 0; char datafilename[206]; char parameterfilename[206]; char conf_filename[50]; char * input_filename = NULL; char * filename = NULL; double plaquette_energy; struct stout_parameters params_smear; #ifdef _KOJAK_INST #pragma pomp inst init #pragma pomp inst begin(main) #endif #if (defined SSE || defined SSE2 || SSE3) signal(SIGILL, &catch_ill_inst); #endif DUM_DERI = 8; DUM_MATRIX = DUM_DERI + 5; NO_OF_SPINORFIELDS = DUM_MATRIX + 4; //4 extra fields (corresponding to DUM_MATRIX+0..5) for deg. and ND matrix mult. NO_OF_SPINORFIELDS_32 = 6; verbose = 0; g_use_clover_flag = 0; process_args(argc,argv,&input_filename,&filename); set_default_filenames(&input_filename, &filename); init_parallel_and_read_input(argc, argv, input_filename); /* this DBW2 stuff is not needed for the inversion ! */ if (g_dflgcr_flag == 1) { even_odd_flag = 0; } g_rgi_C1 = 0; if (Nsave == 0) { Nsave = 1; } if (g_running_phmc) { NO_OF_SPINORFIELDS = DUM_MATRIX + 8; } tmlqcd_mpi_init(argc, argv); g_dbw2rand = 0; /* starts the single and double precision random number */ /* generator */ start_ranlux(rlxd_level, random_seed^nstore); /* we need to make sure that we don't have even_odd_flag = 1 */ /* if any of the operators doesn't use it */ /* in this way even/odd can still be used by other operators */ for(j = 0; j < no_operators; j++) if(!operator_list[j].even_odd_flag) even_odd_flag = 0; #ifndef TM_USE_MPI g_dbw2rand = 0; #endif #ifdef _GAUGE_COPY j = init_gauge_field(VOLUMEPLUSRAND, 1); j += init_gauge_field_32(VOLUMEPLUSRAND, 1); #else j = init_gauge_field(VOLUMEPLUSRAND, 0); j += init_gauge_field_32(VOLUMEPLUSRAND, 0); #endif if (j != 0) { fprintf(stderr, "Not enough memory for gauge_fields! Aborting...\n"); exit(-1); } j = init_geometry_indices(VOLUMEPLUSRAND); if (j != 0) { fprintf(stderr, "Not enough memory for geometry indices! Aborting...\n"); exit(-1); } if (no_monomials > 0) { if (even_odd_flag) { j = init_monomials(VOLUMEPLUSRAND / 2, even_odd_flag); } else { j = init_monomials(VOLUMEPLUSRAND, even_odd_flag); } if (j != 0) { fprintf(stderr, "Not enough memory for monomial pseudo fermion fields! Aborting...\n"); exit(-1); } } if (even_odd_flag) { j = init_spinor_field(VOLUMEPLUSRAND / 2, NO_OF_SPINORFIELDS); j += init_spinor_field_32(VOLUMEPLUSRAND / 2, NO_OF_SPINORFIELDS_32); } else { j = init_spinor_field(VOLUMEPLUSRAND, NO_OF_SPINORFIELDS); j += init_spinor_field_32(VOLUMEPLUSRAND, NO_OF_SPINORFIELDS_32); } if (j != 0) { fprintf(stderr, "Not enough memory for spinor fields! Aborting...\n"); exit(-1); } if (g_running_phmc) { j = init_chi_spinor_field(VOLUMEPLUSRAND / 2, 20); if (j != 0) { fprintf(stderr, "Not enough memory for PHMC Chi fields! Aborting...\n"); exit(-1); } } g_mu = g_mu1; if (g_cart_id == 0) { /*construct the filenames for the observables and the parameters*/ strncpy(datafilename, filename, 200); strcat(datafilename, ".data"); strncpy(parameterfilename, filename, 200); strcat(parameterfilename, ".para"); parameterfile = fopen(parameterfilename, "w"); write_first_messages(parameterfile, "invert", git_hash); fclose(parameterfile); } /* define the geometry */ geometry(); /* define the boundary conditions for the fermion fields */ boundary(g_kappa); phmc_invmaxev = 1.; init_operators(); /* list and initialize measurements*/ if(g_proc_id == 0) { printf("\n"); for(int j = 0; j < no_measurements; j++) { printf("# measurement id %d, type = %d\n", j, measurement_list[j].type); } } init_measurements(); /* this could be maybe moved to init_operators */ #ifdef _USE_HALFSPINOR j = init_dirac_halfspinor(); if (j != 0) { fprintf(stderr, "Not enough memory for halffield! Aborting...\n"); exit(-1); } /* for mixed precision solvers, the 32 bit halfspinor field must always be there */ j = init_dirac_halfspinor32(); if (j != 0) { fprintf(stderr, "Not enough memory for 32-bit halffield! Aborting...\n"); exit(-1); } # if (defined _PERSISTENT) if (even_odd_flag) init_xchange_halffield(); # endif #endif for (j = 0; j < Nmeas; j++) { sprintf(conf_filename, "%s.%.4d", gauge_input_filename, nstore); if (g_cart_id == 0) { printf("#\n# Trying to read gauge field from file %s in %s precision.\n", conf_filename, (gauge_precision_read_flag == 32 ? "single" : "double")); fflush(stdout); } if( (i = read_gauge_field(conf_filename,g_gauge_field)) !=0) { fprintf(stderr, "Error %d while reading gauge field from %s\n Aborting...\n", i, conf_filename); exit(-2); } if (g_cart_id == 0) { printf("# Finished reading gauge field.\n"); fflush(stdout); } #ifdef TM_USE_MPI xchange_gauge(g_gauge_field); #endif /*Convert to a 32 bit gauge field, after xchange*/ convert_32_gauge_field(g_gauge_field_32, g_gauge_field, VOLUMEPLUSRAND); /*compute the energy of the gauge field*/ plaquette_energy = measure_plaquette( (const su3**) g_gauge_field); if (g_cart_id == 0) { printf("# The computed plaquette value is %e.\n", plaquette_energy / (6.*VOLUME*g_nproc)); fflush(stdout); } if (use_stout_flag == 1){ params_smear.rho = stout_rho; params_smear.iterations = stout_no_iter; /* if (stout_smear((su3_tuple*)(g_gauge_field[0]), ¶ms_smear, (su3_tuple*)(g_gauge_field[0])) != 0) */ /* exit(1) ; */ g_update_gauge_copy = 1; plaquette_energy = measure_plaquette( (const su3**) g_gauge_field); if (g_cart_id == 0) { printf("# The plaquette value after stouting is %e\n", plaquette_energy / (6.*VOLUME*g_nproc)); fflush(stdout); } } /* if any measurements are defined in the input file, do them here */ measurement * meas; for(int imeas = 0; imeas < no_measurements; imeas++){ meas = &measurement_list[imeas]; if (g_proc_id == 0) { fprintf(stdout, "#\n# Beginning online measurement.\n"); } meas->measurefunc(nstore, imeas, even_odd_flag); } if (reweighting_flag == 1) { reweighting_factor(reweighting_samples, nstore); } /* Compute minimal eigenvalues, if wanted */ if (compute_evs != 0) { eigenvalues(&no_eigenvalues, 5000, eigenvalue_precision, 0, compute_evs, nstore, even_odd_flag); } if (phmc_compute_evs != 0) { #ifdef TM_USE_MPI MPI_Finalize(); #endif return(0); } /* Compute the mode number or topological susceptibility using spectral projectors, if wanted*/ if(compute_modenumber != 0 || compute_topsus !=0){ invert_compute_modenumber(); } // set up blocks if Deflation is used if (g_dflgcr_flag) init_blocks(nblocks_t, nblocks_x, nblocks_y, nblocks_z); if(SourceInfo.type == SRC_TYPE_VOL || SourceInfo.type == SRC_TYPE_PION_TS || SourceInfo.type == SRC_TYPE_GEN_PION_TS) { index_start = 0; index_end = 1; } g_precWS=NULL; if(use_preconditioning == 1){ /* todo load fftw wisdom */ #if (defined HAVE_FFTW ) && !( defined TM_USE_MPI) loadFFTWWisdom(g_spinor_field[0],g_spinor_field[1],T,LX); #else use_preconditioning=0; #endif } if (g_cart_id == 0) { fprintf(stdout, "#\n"); /*Indicate starting of the operator part*/ } for(op_id = 0; op_id < no_operators; op_id++) { boundary(operator_list[op_id].kappa); g_kappa = operator_list[op_id].kappa; g_mu = operator_list[op_id].mu; g_c_sw = operator_list[op_id].c_sw; // DFLGCR and DFLFGMRES if(operator_list[op_id].solver == DFLGCR || operator_list[op_id].solver == DFLFGMRES) { generate_dfl_subspace(g_N_s, VOLUME, reproduce_randomnumber_flag); } if(use_preconditioning==1 && PRECWSOPERATORSELECT[operator_list[op_id].solver]!=PRECWS_NO ){ printf("# Using preconditioning with treelevel preconditioning operator: %s \n", precWSOpToString(PRECWSOPERATORSELECT[operator_list[op_id].solver])); /* initial preconditioning workspace */ operator_list[op_id].precWS=(spinorPrecWS*)malloc(sizeof(spinorPrecWS)); spinorPrecWS_Init(operator_list[op_id].precWS, operator_list[op_id].kappa, operator_list[op_id].mu/2./operator_list[op_id].kappa, -(0.5/operator_list[op_id].kappa-4.), PRECWSOPERATORSELECT[operator_list[op_id].solver]); g_precWS = operator_list[op_id].precWS; if(PRECWSOPERATORSELECT[operator_list[op_id].solver] == PRECWS_D_DAGGER_D) { fitPrecParams(op_id); } } for(isample = 0; isample < no_samples; isample++) { for (ix = index_start; ix < index_end; ix++) { if (g_cart_id == 0) { fprintf(stdout, "#\n"); /*Indicate starting of new index*/ } /* we use g_spinor_field[0-7] for sources and props for the moment */ /* 0-3 in case of 1 flavour */ /* 0-7 in case of 2 flavours */ prepare_source(nstore, isample, ix, op_id, read_source_flag, source_location, random_seed); //randmize initial guess for eigcg if needed-----experimental if( (operator_list[op_id].solver == INCREIGCG) && (operator_list[op_id].solver_params.eigcg_rand_guess_opt) ){ //randomize the initial guess gaussian_volume_source( operator_list[op_id].prop0, operator_list[op_id].prop1,isample,ix,0); //need to check this } operator_list[op_id].inverter(op_id, index_start, 1); } } if(use_preconditioning==1 && operator_list[op_id].precWS!=NULL ){ /* free preconditioning workspace */ spinorPrecWS_Free(operator_list[op_id].precWS); free(operator_list[op_id].precWS); } if(operator_list[op_id].type == OVERLAP){ free_Dov_WS(); } } nstore += Nsave; } #ifdef TM_USE_OMP free_omp_accumulators(); #endif free_blocks(); free_dfl_subspace(); free_gauge_field(); free_gauge_field_32(); free_geometry_indices(); free_spinor_field(); free_spinor_field_32(); free_moment_field(); free_chi_spinor_field(); free(filename); free(input_filename); free(SourceInfo.basename); free(PropInfo.basename); #ifdef TM_USE_QUDA _endQuda(); #endif #ifdef TM_USE_MPI MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); #endif return(0); #ifdef _KOJAK_INST #pragma pomp inst end(main) #endif }