int tmLQCD_finalise() {
#ifdef OMP
free_omp_accumulators();
#endif
#ifdef HAVE_GPU
if(usegpu_flag){
finalize_mixedsolve();
# ifdef GPU_DOUBLE
finalize_gpu_fields();
# endif
# ifdef TEMPORALGAUGE
finalize_temporalgauge();
# endif
}
#endif
free_gauge_field();
free_geometry_indices();
free_spinor_field();
free_moment_field();
free_chi_spinor_field();
#ifdef MPI
MPI_Barrier(MPI_COMM_WORLD);
#endif
return(0);
}
void gtrafo_eo_nd(spinor * const Even_s, spinor * const Odd_s, spinor * const Even_c, spinor * const Odd_c,
spinor * const Even_new_s, spinor * const Odd_new_s, spinor * const Even_new_c, spinor * const Odd_new_c,
GTRAFO_TYPE type){
/* initialize temporal gauge here */
int retval;
double dret1, dret2;
static double plaquette1 = 0.0;
static double plaquette2 = 0.0;
if(type==GTRAFO_APPLY){
/* need VOLUME here (not N=VOLUME/2)*/
if ((retval = init_temporalgauge_trafo(VOLUME, g_gauge_field)) != 0 ) { // initializes the transformation matrices
if (g_proc_id == 0) printf("Error while gauge fixing to temporal gauge. Aborting...\n"); // g_tempgauge_field as a copy of g_gauge_field
exit(200);
}
/* do trafo */
plaquette1 = measure_plaquette(g_gauge_field);
apply_gtrafo(g_gauge_field, g_trafo); // transformation of the gauge field
plaquette2 = measure_plaquette(g_gauge_field);
if (g_proc_id == 0) printf("\tPlaquette before gauge fixing: %.16e\n", plaquette1/6./VOLUME);
if (g_proc_id == 0) printf("\tPlaquette after gauge fixing: %.16e\n", plaquette2/6./VOLUME);
/* do trafo to odd_s part of source */
dret1 = square_norm(Odd_s, VOLUME/2 , 1);
apply_gtrafo_spinor_odd(Odd_s, g_trafo); // odd spinor transformation, strange
dret2 = square_norm(Odd_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* do trafo to odd_c part of source */
dret1 = square_norm(Odd_c, VOLUME/2 , 1);
apply_gtrafo_spinor_odd(Odd_c, g_trafo); // odd spinor transformation, charm
dret2 = square_norm(Odd_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* do trafo to even_s part of source */
dret1 = square_norm(Even_s, VOLUME/2 , 1);
apply_gtrafo_spinor_even(Even_s, g_trafo); // even spinor transformation, strange
dret2 = square_norm(Even_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* do trafo to even_c part of source */
dret1 = square_norm(Even_c, VOLUME/2 , 1);
apply_gtrafo_spinor_even(Even_c, g_trafo); // even spinor transformation, charm
dret2 = square_norm(Even_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
} else {
/* undo trafo */
/* apply_inv_gtrafo(g_gauge_field, g_trafo);*/
/* copy back the saved original field located in g_tempgauge_field -> update necessary*/
plaquette1 = measure_plaquette(g_gauge_field);
copy_gauge_field(g_gauge_field, g_tempgauge_field);
g_update_gauge_copy = 1;
plaquette2 = measure_plaquette(g_gauge_field);
if (g_proc_id == 0) printf("\tPlaquette before inverse gauge fixing: %.16e\n", plaquette1/6./VOLUME);
if (g_proc_id == 0) printf("\tPlaquette after inverse gauge fixing: %.16e\n", plaquette2/6./VOLUME);
/* undo trafo to source Even_s */
dret1 = square_norm(Even_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_s, g_trafo);
dret2 = square_norm(Even_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* undo trafo to source Even_c */
dret1 = square_norm(Even_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_c, g_trafo);
dret2 = square_norm(Even_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* undo trafo to source Odd_s */
dret1 = square_norm(Odd_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_s, g_trafo);
dret2 = square_norm(Odd_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* undo trafo to source Odd_c */
dret1 = square_norm(Odd_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_c, g_trafo);
dret2 = square_norm(Odd_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Even_new_s
dret1 = square_norm(Even_new_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_new_s, g_trafo);
dret2 = square_norm(Even_new_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Even_new_c
dret1 = square_norm(Even_new_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_new_c, g_trafo);
dret2 = square_norm(Even_new_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Odd_new_s
dret1 = square_norm(Odd_new_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_new_s, g_trafo);
dret2 = square_norm(Odd_new_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Odd_new_c
dret1 = square_norm(Odd_new_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_new_c, g_trafo);
dret2 = square_norm(Odd_new_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
finalize_temporalgauge();
}
# ifdef TM_USE_MPI
xchange_gauge(g_gauge_field);
# endif
}
int invert_doublet_eo(spinor * const Even_new_s, spinor * const Odd_new_s,
spinor * const Even_new_c, spinor * const Odd_new_c,
spinor * const Even_s, spinor * const Odd_s,
spinor * const Even_c, spinor * const Odd_c,
const double precision, const int max_iter,
const int solver_flag, const int rel_prec) {
int iter = 0;
#ifdef HAVE_GPU
# ifdef TEMPORALGAUGE
/* initialize temporal gauge here */
int retval;
double dret1, dret2;
double plaquette1 = 0.0;
double plaquette2 = 0.0;
if (usegpu_flag) {
/* need VOLUME here (not N=VOLUME/2)*/
if ((retval = init_temporalgauge_trafo(VOLUME, g_gauge_field)) != 0 ) { // initializes the transformation matrices
if (g_proc_id == 0) printf("Error while gauge fixing to temporal gauge. Aborting...\n"); // g_tempgauge_field as a copy of g_gauge_field
exit(200);
}
/* do trafo */
plaquette1 = measure_plaquette(g_gauge_field);
apply_gtrafo(g_gauge_field, g_trafo); // transformation of the gauge field
plaquette2 = measure_plaquette(g_gauge_field);
if (g_proc_id == 0) printf("\tPlaquette before gauge fixing: %.16e\n", plaquette1/6./VOLUME);
if (g_proc_id == 0) printf("\tPlaquette after gauge fixing: %.16e\n", plaquette2/6./VOLUME);
/* do trafo to odd_s part of source */
dret1 = square_norm(Odd_s, VOLUME/2 , 1);
apply_gtrafo_spinor_odd(Odd_s, g_trafo); // odd spinor transformation, strange
dret2 = square_norm(Odd_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* do trafo to odd_c part of source */
dret1 = square_norm(Odd_c, VOLUME/2 , 1);
apply_gtrafo_spinor_odd(Odd_c, g_trafo); // odd spinor transformation, charm
dret2 = square_norm(Odd_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* do trafo to even_s part of source */
dret1 = square_norm(Even_s, VOLUME/2 , 1);
apply_gtrafo_spinor_even(Even_s, g_trafo); // even spinor transformation, strange
dret2 = square_norm(Even_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* do trafo to even_c part of source */
dret1 = square_norm(Even_c, VOLUME/2 , 1);
apply_gtrafo_spinor_even(Even_c, g_trafo); // even spinor transformation, charm
dret2 = square_norm(Even_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
# ifdef MPI
xchange_gauge(g_gauge_field);
# endif
}
# endif
#endif /* HAVE_GPU*/
/* here comes the inversion using even/odd preconditioning */
if(g_proc_id == 0) {printf("# Using even/odd preconditioning!\n"); fflush(stdout);}
M_ee_inv_ndpsi(Even_new_s, Even_new_c,
Even_s, Even_c,
g_mubar, g_epsbar);
Hopping_Matrix(OE, g_spinor_field[DUM_DERI], Even_new_s);
Hopping_Matrix(OE, g_spinor_field[DUM_DERI+1], Even_new_c);
/* The sign is plus, since in Hopping_Matrix */
/* the minus is missing */
assign_mul_add_r(g_spinor_field[DUM_DERI], +1., Odd_s, VOLUME/2);
assign_mul_add_r(g_spinor_field[DUM_DERI+1], +1., Odd_c, VOLUME/2);
/* Do the inversion with the preconditioned */
/* matrix to get the odd sites */
/* Here we invert the hermitean operator squared */
if(g_proc_id == 0) {
printf("# Using CG for TMWILSON flavour doublet!\n");
fflush(stdout);
}
gamma5(g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI], VOLUME/2);
gamma5(g_spinor_field[DUM_DERI+1], g_spinor_field[DUM_DERI+1], VOLUME/2);
#ifdef HAVE_GPU
if (usegpu_flag) { // GPU, mixed precision solver
# if defined(MPI) && defined(PARALLELT)
iter = mixedsolve_eo_nd(Odd_new_s, Odd_new_c, g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1],
max_iter, precision, rel_prec);
# elif !defined(MPI) && !defined(PARALLELT)
iter = mixedsolve_eo_nd(Odd_new_s, Odd_new_c, g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1],
max_iter, precision, rel_prec);
# else
printf("MPI and/or PARALLELT are not appropriately set for the GPU implementation. Aborting...\n");
exit(-1);
# endif
}
else { // CPU, conjugate gradient
iter = cg_her_nd(Odd_new_s, Odd_new_c, g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1],
max_iter, precision, rel_prec,
VOLUME/2, &Qtm_pm_ndpsi);
}
#else // CPU, conjugate gradient
iter = cg_her_nd(Odd_new_s, Odd_new_c, g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1],
max_iter, precision, rel_prec,
VOLUME/2, &Qtm_pm_ndpsi);
#endif
Qtm_dagger_ndpsi(Odd_new_s, Odd_new_c,
Odd_new_s, Odd_new_c);
/* Reconstruct the even sites */
Hopping_Matrix(EO, g_spinor_field[DUM_DERI], Odd_new_s);
Hopping_Matrix(EO, g_spinor_field[DUM_DERI+1], Odd_new_c);
M_ee_inv_ndpsi(g_spinor_field[DUM_DERI+2], g_spinor_field[DUM_DERI+3],
g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1],
g_mubar, g_epsbar);
/* The sign is plus, since in Hopping_Matrix */
/* the minus is missing */
assign_add_mul_r(Even_new_s, g_spinor_field[DUM_DERI+2], +1., VOLUME/2);
assign_add_mul_r(Even_new_c, g_spinor_field[DUM_DERI+3], +1., VOLUME/2);
#ifdef HAVE_GPU
/* return from temporal gauge again */
# ifdef TEMPORALGAUGE
if (usegpu_flag) {
/* undo trafo */
/* apply_inv_gtrafo(g_gauge_field, g_trafo);*/
/* copy back the saved original field located in g_tempgauge_field -> update necessary*/
plaquette1 = measure_plaquette(g_gauge_field);
copy_gauge_field(g_gauge_field, g_tempgauge_field);
g_update_gauge_copy = 1;
plaquette2 = measure_plaquette(g_gauge_field);
if (g_proc_id == 0) printf("\tPlaquette before inverse gauge fixing: %.16e\n", plaquette1/6./VOLUME);
if (g_proc_id == 0) printf("\tPlaquette after inverse gauge fixing: %.16e\n", plaquette2/6./VOLUME);
/* undo trafo to source Even_s */
dret1 = square_norm(Even_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_s, g_trafo);
dret2 = square_norm(Even_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* undo trafo to source Even_c */
dret1 = square_norm(Even_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_c, g_trafo);
dret2 = square_norm(Even_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* undo trafo to source Odd_s */
dret1 = square_norm(Odd_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_s, g_trafo);
dret2 = square_norm(Odd_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
/* undo trafo to source Odd_c */
dret1 = square_norm(Odd_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_c, g_trafo);
dret2 = square_norm(Odd_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Even_new_s
dret1 = square_norm(Even_new_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_new_s, g_trafo);
dret2 = square_norm(Even_new_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Even_new_c
dret1 = square_norm(Even_new_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_even(Even_new_c, g_trafo);
dret2 = square_norm(Even_new_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Odd_new_s
dret1 = square_norm(Odd_new_s, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_new_s, g_trafo);
dret2 = square_norm(Odd_new_s, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
// Odd_new_c
dret1 = square_norm(Odd_new_c, VOLUME/2 , 1);
apply_inv_gtrafo_spinor_odd(Odd_new_c, g_trafo);
dret2 = square_norm(Odd_new_c, VOLUME/2, 1);
if (g_proc_id == 0) printf("\tsquare norm before gauge fixing: %.16e\n", dret1);
if (g_proc_id == 0) printf("\tsquare norm after gauge fixing: %.16e\n", dret2);
finalize_temporalgauge();
# ifdef MPI
xchange_gauge(g_gauge_field);
# endif
}
# endif
#endif
return(iter);
}