int update_tm(double *plaquette_energy, double *rectangle_energy,
char * filename, const int return_check, const int acctest,
const int traj_counter) {
su3 *v, *w;
int accept, i=0, j=0, iostatus=0;
double yy[1];
double dh, expmdh, ret_dh=0., ret_gauge_diff=0., tmp;
double atime=0., etime=0.;
double ks = 0., kc = 0., ds, tr, ts, tt;
char tmp_filename[50];
/* Energy corresponding to the Gauge part */
double new_plaquette_energy=0., new_rectangle_energy = 0.;
/* Energy corresponding to the Momenta part */
double enep=0., enepx=0., ret_enep = 0.;
/* Energy corresponding to the pseudo fermion part(s) */
FILE * datafile=NULL, * ret_check_file=NULL;
hamiltonian_field_t hf;
paramsXlfInfo *xlfInfo;
hf.gaugefield = g_gauge_field;
hf.momenta = moment;
hf.derivative = df0;
hf.update_gauge_copy = g_update_gauge_copy;
hf.traj_counter = traj_counter;
integrator_set_fields(&hf);
sprintf(tmp_filename, ".conf.t%05d.tmp",traj_counter);
atime = gettime();
/*
* here the momentum and spinor fields are initialized
* and their respective actions are calculated
*/
/*
* copy the gauge field to gauge_tmp
*/
#ifdef TM_USE_OMP
#pragma omp parallel for private(w,v)
#endif
for(int ix=0;ix<VOLUME;ix++) {
for(int mu=0;mu<4;mu++) {
v=&hf.gaugefield[ix][mu];
w=&gauge_tmp[ix][mu];
_su3_assign(*w,*v);
}
}
#ifdef DDalphaAMG
MG_reset();
#endif
/* heatbath for all monomials */
for(i = 0; i < Integrator.no_timescales; i++) {
for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) {
monomial_list[ Integrator.mnls_per_ts[i][j] ].hbfunction(Integrator.mnls_per_ts[i][j], &hf);
}
}
if(Integrator.monitor_forces) monitor_forces(&hf);
/* initialize the momenta */
enep = random_su3adj_field(reproduce_randomnumber_flag, hf.momenta);
g_sloppy_precision = 1;
/* run the trajectory */
if(Integrator.n_int[Integrator.no_timescales-1] > 0) {
Integrator.integrate[Integrator.no_timescales-1](Integrator.tau,
Integrator.no_timescales-1, 1);
}
g_sloppy_precision = 0;
/* compute the final energy contributions for all monomials */
dh = 0.;
for(i = 0; i < Integrator.no_timescales; i++) {
for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) {
dh += monomial_list[ Integrator.mnls_per_ts[i][j] ].accfunction(Integrator.mnls_per_ts[i][j], &hf);
}
}
enepx = moment_energy(hf.momenta);
if (!bc_flag) { /* if PBC */
new_plaquette_energy = measure_plaquette( (const su3**) hf.gaugefield);
if(g_rgi_C1 > 0. || g_rgi_C1 < 0.) {
new_rectangle_energy = measure_rectangles( (const su3**) hf.gaugefield);
}
}
if(g_proc_id == 0 && g_debug_level > 3) printf("called moment_energy: dh = %1.10e\n", (enepx - enep));
/* Compute the energy difference */
dh = dh + (enepx - enep);
if(g_proc_id == 0 && g_debug_level > 3) {
printf("called momenta_acc dH = %e\n", (enepx - enep));
}
expmdh = exp(-dh);
/* the random number is only taken at node zero and then distributed to
the other sites */
ranlxd(yy,1);
#ifdef TM_USE_MPI
MPI_Bcast(&yy[0], 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
#endif
/* when acctest is 0 (i.e. do not perform acceptance test), the trajectory is accepted whatever the energy difference */
accept = (!acctest | (expmdh > yy[0]));
if(g_proc_id == 0) {
fprintf(stdout, "# Trajectory is %saccepted.\n", (accept ? "" : "not "));
}
/* Here a reversibility test is performed */
/* The trajectory is integrated back */
if(return_check) {
if(g_proc_id == 0) {
fprintf(stdout, "# Performing reversibility check.\n");
}
if(accept) {
/* save gauge file to disk before performing reversibility check */
xlfInfo = construct_paramsXlfInfo((*plaquette_energy)/(6.*VOLUME*g_nproc), traj_counter);
// Should write this to temporary file first, and then check
if(g_proc_id == 0 && g_debug_level > 0) {
fprintf(stdout, "# Writing gauge field to file %s.\n", tmp_filename);
}
if((iostatus = write_gauge_field( tmp_filename, 64, xlfInfo) != 0 )) {
/* Writing failed directly */
fprintf(stderr, "Error %d while writing gauge field to %s\nAborting...\n", iostatus, tmp_filename);
exit(-2);
}
/* There is double writing of the gauge field, also in hmc_tm.c in this case */
/* No reading back check needed here, as reading back is done further down */
if(g_proc_id == 0 && g_debug_level > 0) {
fprintf(stdout, "# Writing done.\n");
}
free(xlfInfo);
}
#ifdef DDalphaAMG
MG_reset();
#endif
g_sloppy_precision = 1;
/* run the trajectory back */
Integrator.integrate[Integrator.no_timescales-1](-Integrator.tau,
Integrator.no_timescales-1, 1);
g_sloppy_precision = 0;
/* compute the energy contributions from the pseudo-fermions */
ret_dh = 0.;
for(i = 0; i < Integrator.no_timescales; i++) {
for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) {
ret_dh += monomial_list[ Integrator.mnls_per_ts[i][j] ].accfunction(Integrator.mnls_per_ts[i][j], &hf);
}
}
ret_enep = moment_energy(hf.momenta);
/* Compute the energy difference */
ret_dh += ret_enep - enep ;
/* Compute Differences in the fields */
ks = 0.;
kc = 0.;
#ifdef TM_USE_OMP
#pragma omp parallel private(w,v,tt,tr,ts,ds,ks,kc)
{
int thread_num = omp_get_thread_num();
#endif
su3 ALIGN v0;
#ifdef TM_USE_OMP
#pragma omp for
#endif
for(int ix = 0; ix < VOLUME; ++ix)
{
for(int mu = 0; mu < 4; ++mu)
{
v=&hf.gaugefield[ix][mu];
w=&gauge_tmp[ix][mu];
_su3_minus_su3(v0, *v, *w);
_su3_square_norm(ds, v0);
tr = sqrt(ds) + kc;
ts = tr + ks;
tt = ts-ks;
ks = ts;
kc = tr-tt;
}
}
kc=ks+kc;
#ifdef TM_USE_OMP
g_omp_acc_re[thread_num] = kc;
} /* OpenMP parallel section closing brace */
/* sum up contributions from thread-local kahan summations */
for(int k = 0; k < omp_num_threads; ++k)
ret_gauge_diff += g_omp_acc_re[k];
#else
ret_gauge_diff = kc;
#endif
#ifdef TM_USE_MPI
tmp = ret_gauge_diff;
MPI_Reduce(&tmp, &ret_gauge_diff, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
#endif
/* compute the total H */
tmp = enep;
for(i = 0; i < Integrator.no_timescales; i++) {
for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) {
tmp += monomial_list[ Integrator.mnls_per_ts[i][j] ].energy0;
}
}
/* Output */
if(g_proc_id == 0) {
ret_check_file = fopen("return_check.data","a");
fprintf(ret_check_file,"%08d ddh = %1.4e ddh/dh = %1.4e ddh/H = %1.4e ddU= %1.4e\n", traj_counter,
ret_dh, ret_dh/dh, ret_dh/tmp, ret_gauge_diff/4./((double)(VOLUME*g_nproc))/3.);
fclose(ret_check_file);
}
if(accept) {
/* Read back gauge field
FIXME unlike in hmc_tm we abort immediately if there is a failure */
if(g_proc_id == 0 && g_debug_level > 0) {
fprintf(stdout, "# Trying to read gauge field from file %s.\n", tmp_filename);
}
if((iostatus = read_gauge_field(tmp_filename,g_gauge_field) != 0)) {
fprintf(stderr, "Error %d while reading gauge field from %s\nAborting...\n", iostatus, tmp_filename);
exit(-2);
}
if(g_proc_id == 0 && g_debug_level > 0) {
fprintf(stdout, "# Reading done.\n");
}
}
if(g_proc_id == 0) {
fprintf(stdout, "# Reversibility check done.\n");
}
} /* end of reversibility check */
if(accept) {
*plaquette_energy = new_plaquette_energy;
*rectangle_energy = new_rectangle_energy;
/* put the links back to SU(3) group */
if (!bc_flag) { /* periodic boundary conditions */
#ifdef TM_USE_OMP
#pragma omp parallel for private(v)
#endif
for(int ix=0;ix<VOLUME;ix++) {
for(int mu=0;mu<4;mu++) {
v=&hf.gaugefield[ix][mu];
restoresu3_in_place(v);
}
}
}
}
else { /* reject: copy gauge_tmp to hf.gaugefield */
#ifdef TM_USE_OMP
#pragma omp parallel for private(w) private(v)
#endif
for(int ix=0;ix<VOLUME;ix++) {
for(int mu=0;mu<4;mu++){
v=&hf.gaugefield[ix][mu];
w=&gauge_tmp[ix][mu];
_su3_assign(*v,*w);
}
}
}
hf.update_gauge_copy = 1;
g_update_gauge_copy = 1;
g_update_gauge_copy_32 = 1;
#ifdef TM_USE_MPI
xchange_gauge(hf.gaugefield);
#endif
/*Convert to a 32 bit gauge field, after xchange*/
convert_32_gauge_field(g_gauge_field_32, hf.gaugefield, VOLUMEPLUSRAND + g_dbw2rand);
etime=gettime();
/* printing data in the .data file */
if(g_proc_id==0) {
datafile = fopen(filename, "a");
if (!bc_flag) { /* if Periodic Boundary Conditions */
fprintf(datafile, "%.8d %14.12f %14.12f %e ", traj_counter,
(*plaquette_energy)/(6.*VOLUME*g_nproc), dh, expmdh);
}
for(i = 0; i < Integrator.no_timescales; i++) {
for(j = 0; j < Integrator.no_mnls_per_ts[i]; j++) {
if(monomial_list[ Integrator.mnls_per_ts[i][j] ].type != GAUGE
&& monomial_list[ Integrator.mnls_per_ts[i][j] ].type != SFGAUGE
&& monomial_list[ Integrator.mnls_per_ts[i][j] ].type != NDPOLY
&& monomial_list[ Integrator.mnls_per_ts[i][j] ].type != NDCLOVER
&& monomial_list[ Integrator.mnls_per_ts[i][j] ].type != CLOVERNDTRLOG
&& monomial_list[ Integrator.mnls_per_ts[i][j] ].type != CLOVERTRLOG ) {
fprintf(datafile,"%d %d ", monomial_list[ Integrator.mnls_per_ts[i][j] ].iter0,
monomial_list[ Integrator.mnls_per_ts[i][j] ].iter1);
}
}
}
fprintf(datafile, "%d %e", accept, etime-atime);
if(g_rgi_C1 > 0. || g_rgi_C1 < 0) {
fprintf(datafile, " %e", (*rectangle_energy)/(12*VOLUME*g_nproc));
}
fprintf(datafile, "\n");
fflush(datafile);
fclose(datafile);
}
return(accept);
}
int main(int argc, char *argv[])
{
FILE *parameterfile = NULL;
int j, i, ix = 0, isample = 0, op_id = 0;
char datafilename[206];
char parameterfilename[206];
char conf_filename[50];
char * input_filename = NULL;
char * filename = NULL;
double plaquette_energy;
struct stout_parameters params_smear;
#ifdef _KOJAK_INST
#pragma pomp inst init
#pragma pomp inst begin(main)
#endif
#if (defined SSE || defined SSE2 || SSE3)
signal(SIGILL, &catch_ill_inst);
#endif
DUM_DERI = 8;
DUM_MATRIX = DUM_DERI + 5;
NO_OF_SPINORFIELDS = DUM_MATRIX + 4;
//4 extra fields (corresponding to DUM_MATRIX+0..5) for deg. and ND matrix mult.
NO_OF_SPINORFIELDS_32 = 6;
verbose = 0;
g_use_clover_flag = 0;
process_args(argc,argv,&input_filename,&filename);
set_default_filenames(&input_filename, &filename);
init_parallel_and_read_input(argc, argv, input_filename);
/* this DBW2 stuff is not needed for the inversion ! */
if (g_dflgcr_flag == 1) {
even_odd_flag = 0;
}
g_rgi_C1 = 0;
if (Nsave == 0) {
Nsave = 1;
}
if (g_running_phmc) {
NO_OF_SPINORFIELDS = DUM_MATRIX + 8;
}
tmlqcd_mpi_init(argc, argv);
g_dbw2rand = 0;
/* starts the single and double precision random number */
/* generator */
start_ranlux(rlxd_level, random_seed^nstore);
/* we need to make sure that we don't have even_odd_flag = 1 */
/* if any of the operators doesn't use it */
/* in this way even/odd can still be used by other operators */
for(j = 0; j < no_operators; j++) if(!operator_list[j].even_odd_flag) even_odd_flag = 0;
#ifndef TM_USE_MPI
g_dbw2rand = 0;
#endif
#ifdef _GAUGE_COPY
j = init_gauge_field(VOLUMEPLUSRAND, 1);
j += init_gauge_field_32(VOLUMEPLUSRAND, 1);
#else
j = init_gauge_field(VOLUMEPLUSRAND, 0);
j += init_gauge_field_32(VOLUMEPLUSRAND, 0);
#endif
if (j != 0) {
fprintf(stderr, "Not enough memory for gauge_fields! Aborting...\n");
exit(-1);
}
j = init_geometry_indices(VOLUMEPLUSRAND);
if (j != 0) {
fprintf(stderr, "Not enough memory for geometry indices! Aborting...\n");
exit(-1);
}
if (no_monomials > 0) {
if (even_odd_flag) {
j = init_monomials(VOLUMEPLUSRAND / 2, even_odd_flag);
}
else {
j = init_monomials(VOLUMEPLUSRAND, even_odd_flag);
}
if (j != 0) {
fprintf(stderr, "Not enough memory for monomial pseudo fermion fields! Aborting...\n");
exit(-1);
}
}
if (even_odd_flag) {
j = init_spinor_field(VOLUMEPLUSRAND / 2, NO_OF_SPINORFIELDS);
j += init_spinor_field_32(VOLUMEPLUSRAND / 2, NO_OF_SPINORFIELDS_32);
}
else {
j = init_spinor_field(VOLUMEPLUSRAND, NO_OF_SPINORFIELDS);
j += init_spinor_field_32(VOLUMEPLUSRAND, NO_OF_SPINORFIELDS_32);
}
if (j != 0) {
fprintf(stderr, "Not enough memory for spinor fields! Aborting...\n");
exit(-1);
}
if (g_running_phmc) {
j = init_chi_spinor_field(VOLUMEPLUSRAND / 2, 20);
if (j != 0) {
fprintf(stderr, "Not enough memory for PHMC Chi fields! Aborting...\n");
exit(-1);
}
}
g_mu = g_mu1;
if (g_cart_id == 0) {
/*construct the filenames for the observables and the parameters*/
strncpy(datafilename, filename, 200);
strcat(datafilename, ".data");
strncpy(parameterfilename, filename, 200);
strcat(parameterfilename, ".para");
parameterfile = fopen(parameterfilename, "w");
write_first_messages(parameterfile, "invert", git_hash);
fclose(parameterfile);
}
/* define the geometry */
geometry();
/* define the boundary conditions for the fermion fields */
boundary(g_kappa);
phmc_invmaxev = 1.;
init_operators();
/* list and initialize measurements*/
if(g_proc_id == 0) {
printf("\n");
for(int j = 0; j < no_measurements; j++) {
printf("# measurement id %d, type = %d\n", j, measurement_list[j].type);
}
}
init_measurements();
/* this could be maybe moved to init_operators */
#ifdef _USE_HALFSPINOR
j = init_dirac_halfspinor();
if (j != 0) {
fprintf(stderr, "Not enough memory for halffield! Aborting...\n");
exit(-1);
}
/* for mixed precision solvers, the 32 bit halfspinor field must always be there */
j = init_dirac_halfspinor32();
if (j != 0)
{
fprintf(stderr, "Not enough memory for 32-bit halffield! Aborting...\n");
exit(-1);
}
# if (defined _PERSISTENT)
if (even_odd_flag)
init_xchange_halffield();
# endif
#endif
for (j = 0; j < Nmeas; j++) {
sprintf(conf_filename, "%s.%.4d", gauge_input_filename, nstore);
if (g_cart_id == 0) {
printf("#\n# Trying to read gauge field from file %s in %s precision.\n",
conf_filename, (gauge_precision_read_flag == 32 ? "single" : "double"));
fflush(stdout);
}
if( (i = read_gauge_field(conf_filename,g_gauge_field)) !=0) {
fprintf(stderr, "Error %d while reading gauge field from %s\n Aborting...\n", i, conf_filename);
exit(-2);
}
if (g_cart_id == 0) {
printf("# Finished reading gauge field.\n");
fflush(stdout);
}
#ifdef TM_USE_MPI
xchange_gauge(g_gauge_field);
#endif
/*Convert to a 32 bit gauge field, after xchange*/
convert_32_gauge_field(g_gauge_field_32, g_gauge_field, VOLUMEPLUSRAND);
/*compute the energy of the gauge field*/
plaquette_energy = measure_plaquette( (const su3**) g_gauge_field);
if (g_cart_id == 0) {
printf("# The computed plaquette value is %e.\n", plaquette_energy / (6.*VOLUME*g_nproc));
fflush(stdout);
}
if (use_stout_flag == 1){
params_smear.rho = stout_rho;
params_smear.iterations = stout_no_iter;
/* if (stout_smear((su3_tuple*)(g_gauge_field[0]), ¶ms_smear, (su3_tuple*)(g_gauge_field[0])) != 0) */
/* exit(1) ; */
g_update_gauge_copy = 1;
plaquette_energy = measure_plaquette( (const su3**) g_gauge_field);
if (g_cart_id == 0) {
printf("# The plaquette value after stouting is %e\n", plaquette_energy / (6.*VOLUME*g_nproc));
fflush(stdout);
}
}
/* if any measurements are defined in the input file, do them here */
measurement * meas;
for(int imeas = 0; imeas < no_measurements; imeas++){
meas = &measurement_list[imeas];
if (g_proc_id == 0) {
fprintf(stdout, "#\n# Beginning online measurement.\n");
}
meas->measurefunc(nstore, imeas, even_odd_flag);
}
if (reweighting_flag == 1) {
reweighting_factor(reweighting_samples, nstore);
}
/* Compute minimal eigenvalues, if wanted */
if (compute_evs != 0) {
eigenvalues(&no_eigenvalues, 5000, eigenvalue_precision,
0, compute_evs, nstore, even_odd_flag);
}
if (phmc_compute_evs != 0) {
#ifdef TM_USE_MPI
MPI_Finalize();
#endif
return(0);
}
/* Compute the mode number or topological susceptibility using spectral projectors, if wanted*/
if(compute_modenumber != 0 || compute_topsus !=0){
invert_compute_modenumber();
}
// set up blocks if Deflation is used
if (g_dflgcr_flag)
init_blocks(nblocks_t, nblocks_x, nblocks_y, nblocks_z);
if(SourceInfo.type == SRC_TYPE_VOL || SourceInfo.type == SRC_TYPE_PION_TS || SourceInfo.type == SRC_TYPE_GEN_PION_TS) {
index_start = 0;
index_end = 1;
}
g_precWS=NULL;
if(use_preconditioning == 1){
/* todo load fftw wisdom */
#if (defined HAVE_FFTW ) && !( defined TM_USE_MPI)
loadFFTWWisdom(g_spinor_field[0],g_spinor_field[1],T,LX);
#else
use_preconditioning=0;
#endif
}
if (g_cart_id == 0) {
fprintf(stdout, "#\n"); /*Indicate starting of the operator part*/
}
for(op_id = 0; op_id < no_operators; op_id++) {
boundary(operator_list[op_id].kappa);
g_kappa = operator_list[op_id].kappa;
g_mu = operator_list[op_id].mu;
g_c_sw = operator_list[op_id].c_sw;
// DFLGCR and DFLFGMRES
if(operator_list[op_id].solver == DFLGCR || operator_list[op_id].solver == DFLFGMRES) {
generate_dfl_subspace(g_N_s, VOLUME, reproduce_randomnumber_flag);
}
if(use_preconditioning==1 && PRECWSOPERATORSELECT[operator_list[op_id].solver]!=PRECWS_NO ){
printf("# Using preconditioning with treelevel preconditioning operator: %s \n",
precWSOpToString(PRECWSOPERATORSELECT[operator_list[op_id].solver]));
/* initial preconditioning workspace */
operator_list[op_id].precWS=(spinorPrecWS*)malloc(sizeof(spinorPrecWS));
spinorPrecWS_Init(operator_list[op_id].precWS,
operator_list[op_id].kappa,
operator_list[op_id].mu/2./operator_list[op_id].kappa,
-(0.5/operator_list[op_id].kappa-4.),
PRECWSOPERATORSELECT[operator_list[op_id].solver]);
g_precWS = operator_list[op_id].precWS;
if(PRECWSOPERATORSELECT[operator_list[op_id].solver] == PRECWS_D_DAGGER_D) {
fitPrecParams(op_id);
}
}
for(isample = 0; isample < no_samples; isample++) {
for (ix = index_start; ix < index_end; ix++) {
if (g_cart_id == 0) {
fprintf(stdout, "#\n"); /*Indicate starting of new index*/
}
/* we use g_spinor_field[0-7] for sources and props for the moment */
/* 0-3 in case of 1 flavour */
/* 0-7 in case of 2 flavours */
prepare_source(nstore, isample, ix, op_id, read_source_flag, source_location, random_seed);
//randmize initial guess for eigcg if needed-----experimental
if( (operator_list[op_id].solver == INCREIGCG) && (operator_list[op_id].solver_params.eigcg_rand_guess_opt) ){ //randomize the initial guess
gaussian_volume_source( operator_list[op_id].prop0, operator_list[op_id].prop1,isample,ix,0); //need to check this
}
operator_list[op_id].inverter(op_id, index_start, 1);
}
}
if(use_preconditioning==1 && operator_list[op_id].precWS!=NULL ){
/* free preconditioning workspace */
spinorPrecWS_Free(operator_list[op_id].precWS);
free(operator_list[op_id].precWS);
}
if(operator_list[op_id].type == OVERLAP){
free_Dov_WS();
}
}
nstore += Nsave;
}
#ifdef TM_USE_OMP
free_omp_accumulators();
#endif
free_blocks();
free_dfl_subspace();
free_gauge_field();
free_gauge_field_32();
free_geometry_indices();
free_spinor_field();
free_spinor_field_32();
free_moment_field();
free_chi_spinor_field();
free(filename);
free(input_filename);
free(SourceInfo.basename);
free(PropInfo.basename);
#ifdef TM_USE_QUDA
_endQuda();
#endif
#ifdef TM_USE_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
#endif
return(0);
#ifdef _KOJAK_INST
#pragma pomp inst end(main)
#endif
}
