void init_disres(FILE *fplog, const gmx_mtop_t *mtop,
t_inputrec *ir, const t_commrec *cr,
t_fcdata *fcd, t_state *state, gmx_bool bIsREMD)
{
int fa, nmol, npair, np;
t_disresdata *dd;
history_t *hist;
gmx_mtop_ilistloop_t iloop;
t_ilist *il;
char *ptr;
dd = &(fcd->disres);
if (gmx_mtop_ftype_count(mtop, F_DISRES) == 0)
{
dd->nres = 0;
return;
}
if (fplog)
{
fprintf(fplog, "Initializing the distance restraints\n");
}
if (ir->eDisre == edrEnsemble)
{
gmx_fatal(FARGS, "Sorry, distance restraints with ensemble averaging over multiple molecules in one system are not functional in this version of GROMACS");
}
dd->dr_weighting = ir->eDisreWeighting;
dd->dr_fc = ir->dr_fc;
if (EI_DYNAMICS(ir->eI))
{
dd->dr_tau = ir->dr_tau;
}
else
{
dd->dr_tau = 0.0;
}
if (dd->dr_tau == 0.0)
{
dd->dr_bMixed = FALSE;
dd->ETerm = 0.0;
}
else
{
dd->dr_bMixed = ir->bDisreMixed;
dd->ETerm = std::exp(-(ir->delta_t/ir->dr_tau));
}
dd->ETerm1 = 1.0 - dd->ETerm;
dd->nres = 0;
dd->npair = 0;
iloop = gmx_mtop_ilistloop_init(mtop);
while (gmx_mtop_ilistloop_next(iloop, &il, &nmol))
{
np = 0;
for (fa = 0; fa < il[F_DISRES].nr; fa += 3)
{
np++;
npair = mtop->ffparams.iparams[il[F_DISRES].iatoms[fa]].disres.npair;
if (np == npair)
{
dd->nres += (ir->eDisre == edrEnsemble ? 1 : nmol)*npair;
dd->npair += nmol*npair;
np = 0;
}
}
}
if (cr && PAR(cr))
{
/* Temporary check, will be removed when disre is implemented with DD */
const char *notestr = "NOTE: atoms involved in distance restraints should be within the same domain. If this is not the case mdrun generates a fatal error. If you encounter this, use a single MPI rank (Verlet+OpenMP+GPUs work fine).";
if (MASTER(cr))
{
fprintf(stderr, "\n%s\n\n", notestr);
}
if (fplog)
{
fprintf(fplog, "%s\n", notestr);
}
if (dd->dr_tau != 0 || ir->eDisre == edrEnsemble || cr->ms != NULL ||
dd->nres != dd->npair)
{
gmx_fatal(FARGS, "Time or ensemble averaged or multiple pair distance restraints do not work (yet) with domain decomposition, use a single MPI rank%s", cr->ms ? " per simulation" : "");
}
if (ir->nstdisreout != 0)
{
if (fplog)
{
fprintf(fplog, "\nWARNING: Can not write distance restraint data to energy file with domain decomposition\n\n");
}
if (MASTER(cr))
{
fprintf(stderr, "\nWARNING: Can not write distance restraint data to energy file with domain decomposition\n");
}
ir->nstdisreout = 0;
}
}
snew(dd->rt, dd->npair);
if (dd->dr_tau != 0.0)
{
hist = &state->hist;
/* Set the "history lack" factor to 1 */
state->flags |= (1<<estDISRE_INITF);
hist->disre_initf = 1.0;
/* Allocate space for the r^-3 time averages */
state->flags |= (1<<estDISRE_RM3TAV);
hist->ndisrepairs = dd->npair;
snew(hist->disre_rm3tav, hist->ndisrepairs);
}
/* Allocate space for a copy of rm3tav,
* so we can call do_force without modifying the state.
*/
snew(dd->rm3tav, dd->npair);
/* Allocate Rt_6 and Rtav_6 consecutively in memory so they can be
* averaged over the processors in one call (in calc_disre_R_6)
*/
snew(dd->Rt_6, 2*dd->nres);
dd->Rtav_6 = &(dd->Rt_6[dd->nres]);
ptr = getenv("GMX_DISRE_ENSEMBLE_SIZE");
if (cr && cr->ms != NULL && ptr != NULL && !bIsREMD)
{
#if GMX_MPI
dd->nsystems = 0;
sscanf(ptr, "%d", &dd->nsystems);
if (fplog)
{
fprintf(fplog, "Found GMX_DISRE_ENSEMBLE_SIZE set to %d systems per ensemble\n", dd->nsystems);
}
/* This check is only valid on MASTER(cr), so probably
* ensemble-averaged distance restraints are broken on more
* than one processor per simulation system. */
if (MASTER(cr))
{
check_multi_int(fplog, cr->ms, dd->nsystems,
"the number of systems per ensemble",
FALSE);
}
gmx_bcast_sim(sizeof(int), &dd->nsystems, cr);
/* We use to allow any value of nsystems which was a divisor
* of ms->nsim. But this required an extra communicator which
* was stored in t_fcdata. This pulled in mpi.h in nearly all C files.
*/
if (!(cr->ms->nsim == 1 || cr->ms->nsim == dd->nsystems))
{
gmx_fatal(FARGS, "GMX_DISRE_ENSEMBLE_SIZE (%d) is not equal to 1 or the number of systems (option -multi) %d", dd->nsystems, cr->ms->nsim);
}
if (fplog)
{
fprintf(fplog, "Our ensemble consists of systems:");
for (int i = 0; i < dd->nsystems; i++)
{
fprintf(fplog, " %d",
(cr->ms->sim/dd->nsystems)*dd->nsystems+i);
}
fprintf(fplog, "\n");
}
snew(dd->Rtl_6, dd->nres);
#endif
}
else
{
dd->nsystems = 1;
dd->Rtl_6 = dd->Rt_6;
}
if (dd->npair > 0)
{
if (fplog)
{
fprintf(fplog, "There are %d distance restraints involving %d atom pairs\n", dd->nres, dd->npair);
}
/* Have to avoid g_disre de-referencing cr blindly, mdrun not
* doing consistency checks for ensemble-averaged distance
* restraints when that's not happening, and only doing those
* checks from appropriate processes (since check_multi_int is
* too broken to check whether the communication will
* succeed...) */
if (cr && cr->ms && dd->nsystems > 1 && MASTER(cr))
{
check_multi_int(fplog, cr->ms, fcd->disres.nres,
"the number of distance restraints",
FALSE);
}
please_cite(fplog, "Tropp80a");
please_cite(fplog, "Torda89a");
}
}
void
do_md_trajectory_writing(FILE *fplog,
t_commrec *cr,
int nfile,
const t_filenm fnm[],
gmx_int64_t step,
gmx_int64_t step_rel,
double t,
t_inputrec *ir,
t_state *state,
t_state *state_global,
gmx_mtop_t *top_global,
t_forcerec *fr,
gmx_mdoutf_t outf,
t_mdebin *mdebin,
gmx_ekindata_t *ekind,
rvec *f,
rvec *f_global,
int *nchkpt,
gmx_bool bCPT,
gmx_bool bRerunMD,
gmx_bool bLastStep,
gmx_bool bDoConfOut,
gmx_bool bSumEkinhOld
)
{
int mdof_flags;
mdof_flags = 0;
if (do_per_step(step, ir->nstxout))
{
mdof_flags |= MDOF_X;
}
if (do_per_step(step, ir->nstvout))
{
mdof_flags |= MDOF_V;
}
if (do_per_step(step, ir->nstfout))
{
mdof_flags |= MDOF_F;
}
if (do_per_step(step, ir->nstxout_compressed))
{
mdof_flags |= MDOF_X_COMPRESSED;
}
if (bCPT)
{
mdof_flags |= MDOF_CPT;
}
;
#if defined(GMX_FAHCORE) || defined(GMX_WRITELASTSTEP)
if (bLastStep)
{
/* Enforce writing positions and velocities at end of run */
mdof_flags |= (MDOF_X | MDOF_V);
}
#endif
#ifdef GMX_FAHCORE
if (MASTER(cr))
{
fcReportProgress( ir->nsteps, step );
}
#if defined(__native_client__)
fcCheckin(MASTER(cr));
#endif
/* sync bCPT and fc record-keeping */
if (bCPT && MASTER(cr))
{
fcRequestCheckPoint();
}
#endif
if (mdof_flags != 0)
{
wallcycle_start(mdoutf_get_wcycle(outf), ewcTRAJ);
if (bCPT)
{
if (MASTER(cr))
{
if (bSumEkinhOld)
{
state_global->ekinstate.bUpToDate = FALSE;
}
else
{
update_ekinstate(&state_global->ekinstate, ekind);
state_global->ekinstate.bUpToDate = TRUE;
}
update_energyhistory(&state_global->enerhist, mdebin);
}
}
mdoutf_write_to_trajectory_files(fplog, cr, outf, mdof_flags, top_global,
step, t, state, state_global, f, f_global);
if (bCPT)
{
(*nchkpt)++;
bCPT = FALSE;
}
debug_gmx();
if (bLastStep && step_rel == ir->nsteps &&
bDoConfOut && MASTER(cr) &&
!bRerunMD)
{
/* x and v have been collected in mdoutf_write_to_trajectory_files,
* because a checkpoint file will always be written
* at the last step.
*/
fprintf(stderr, "\nWriting final coordinates.\n");
if (fr->bMolPBC)
{
/* Make molecules whole only for confout writing */
do_pbc_mtop(fplog, ir->ePBC, state->box, top_global, state_global->x);
}
write_sto_conf_mtop(ftp2fn(efSTO, nfile, fnm),
*top_global->name, top_global,
state_global->x, state_global->v,
ir->ePBC, state->box);
debug_gmx();
}
wallcycle_stop(mdoutf_get_wcycle(outf), ewcTRAJ);
}
}
int relax_shell_flexcon(FILE *fplog, t_commrec *cr, gmx_bool bVerbose,
gmx_large_int_t mdstep, t_inputrec *inputrec,
gmx_bool bDoNS, int force_flags,
gmx_bool bStopCM,
gmx_localtop_t *top,
gmx_mtop_t* mtop,
gmx_constr_t constr,
gmx_enerdata_t *enerd, t_fcdata *fcd,
t_state *state, rvec f[],
tensor force_vir,
t_mdatoms *md,
t_nrnb *nrnb, gmx_wallcycle_t wcycle,
t_graph *graph,
gmx_groups_t *groups,
struct gmx_shellfc *shfc,
t_forcerec *fr,
gmx_bool bBornRadii,
double t, rvec mu_tot,
int natoms, gmx_bool *bConverged,
gmx_vsite_t *vsite,
FILE *fp_field)
{
int nshell;
t_shell *shell;
t_idef *idef;
rvec *pos[2], *force[2], *acc_dir = NULL, *x_old = NULL;
real Epot[2], df[2];
rvec dx;
real sf_dir, invdt;
real ftol, xiH, xiS, dum = 0;
char sbuf[22];
gmx_bool bCont, bInit;
int nat, dd_ac0, dd_ac1 = 0, i;
int start = md->start, homenr = md->homenr, end = start+homenr, cg0, cg1;
int nflexcon, g, number_steps, d, Min = 0, count = 0;
#define Try (1-Min) /* At start Try = 1 */
bCont = (mdstep == inputrec->init_step) && inputrec->bContinuation;
bInit = (mdstep == inputrec->init_step) || shfc->bRequireInit;
ftol = inputrec->em_tol;
number_steps = inputrec->niter;
nshell = shfc->nshell;
shell = shfc->shell;
nflexcon = shfc->nflexcon;
idef = &top->idef;
if (DOMAINDECOMP(cr))
{
nat = dd_natoms_vsite(cr->dd);
if (nflexcon > 0)
{
dd_get_constraint_range(cr->dd, &dd_ac0, &dd_ac1);
nat = max(nat, dd_ac1);
}
}
else
{
nat = state->natoms;
}
if (nat > shfc->x_nalloc)
{
/* Allocate local arrays */
shfc->x_nalloc = over_alloc_dd(nat);
for (i = 0; (i < 2); i++)
{
srenew(shfc->x[i], shfc->x_nalloc);
srenew(shfc->f[i], shfc->x_nalloc);
}
}
for (i = 0; (i < 2); i++)
{
pos[i] = shfc->x[i];
force[i] = shfc->f[i];
}
/* With particle decomposition this code only works
* when all particles involved with each shell are in the same cg.
*/
if (bDoNS && inputrec->ePBC != epbcNONE && !DOMAINDECOMP(cr))
{
/* This is the only time where the coordinates are used
* before do_force is called, which normally puts all
* charge groups in the box.
*/
if (PARTDECOMP(cr))
{
pd_cg_range(cr, &cg0, &cg1);
}
else
{
cg0 = 0;
cg1 = top->cgs.nr;
}
put_charge_groups_in_box(fplog, cg0, cg1, fr->ePBC, state->box,
&(top->cgs), state->x, fr->cg_cm);
if (graph)
{
mk_mshift(fplog, graph, fr->ePBC, state->box, state->x);
}
}
/* After this all coordinate arrays will contain whole molecules */
if (graph)
{
shift_self(graph, state->box, state->x);
}
if (nflexcon)
{
if (nat > shfc->flex_nalloc)
{
shfc->flex_nalloc = over_alloc_dd(nat);
srenew(shfc->acc_dir, shfc->flex_nalloc);
srenew(shfc->x_old, shfc->flex_nalloc);
}
acc_dir = shfc->acc_dir;
x_old = shfc->x_old;
for (i = 0; i < homenr; i++)
{
for (d = 0; d < DIM; d++)
{
shfc->x_old[i][d] =
state->x[start+i][d] - state->v[start+i][d]*inputrec->delta_t;
}
}
}
/* Do a prediction of the shell positions */
if (shfc->bPredict && !bCont)
{
predict_shells(fplog, state->x, state->v, inputrec->delta_t, nshell, shell,
md->massT, NULL, bInit);
}
/* do_force expected the charge groups to be in the box */
if (graph)
{
unshift_self(graph, state->box, state->x);
}
/* Calculate the forces first time around */
if (gmx_debug_at)
{
pr_rvecs(debug, 0, "x b4 do_force", state->x + start, homenr);
}
do_force(fplog, cr, inputrec, mdstep, nrnb, wcycle, top, mtop, groups,
state->box, state->x, &state->hist,
force[Min], force_vir, md, enerd, fcd,
state->lambda, graph,
fr, vsite, mu_tot, t, fp_field, NULL, bBornRadii,
(bDoNS ? GMX_FORCE_NS : 0) | force_flags);
sf_dir = 0;
if (nflexcon)
{
init_adir(fplog, shfc,
constr, idef, inputrec, cr, dd_ac1, mdstep, md, start, end,
shfc->x_old-start, state->x, state->x, force[Min],
shfc->acc_dir-start,
fr->bMolPBC, state->box, state->lambda, &dum, nrnb);
for (i = start; i < end; i++)
{
sf_dir += md->massT[i]*norm2(shfc->acc_dir[i-start]);
}
}
Epot[Min] = enerd->term[F_EPOT];
df[Min] = rms_force(cr, shfc->f[Min], nshell, shell, nflexcon, &sf_dir, &Epot[Min]);
df[Try] = 0;
if (debug)
{
fprintf(debug, "df = %g %g\n", df[Min], df[Try]);
}
if (gmx_debug_at)
{
pr_rvecs(debug, 0, "force0", force[Min], md->nr);
}
if (nshell+nflexcon > 0)
{
/* Copy x to pos[Min] & pos[Try]: during minimization only the
* shell positions are updated, therefore the other particles must
* be set here.
*/
memcpy(pos[Min], state->x, nat*sizeof(state->x[0]));
memcpy(pos[Try], state->x, nat*sizeof(state->x[0]));
}
if (bVerbose && MASTER(cr))
{
print_epot(stdout, mdstep, 0, Epot[Min], df[Min], nflexcon, sf_dir);
}
if (debug)
{
fprintf(debug, "%17s: %14.10e\n",
interaction_function[F_EKIN].longname, enerd->term[F_EKIN]);
fprintf(debug, "%17s: %14.10e\n",
interaction_function[F_EPOT].longname, enerd->term[F_EPOT]);
fprintf(debug, "%17s: %14.10e\n",
interaction_function[F_ETOT].longname, enerd->term[F_ETOT]);
fprintf(debug, "SHELLSTEP %s\n", gmx_step_str(mdstep, sbuf));
}
/* First check whether we should do shells, or whether the force is
* low enough even without minimization.
*/
*bConverged = (df[Min] < ftol);
for (count = 1; (!(*bConverged) && (count < number_steps)); count++)
{
if (vsite)
{
construct_vsites(fplog, vsite, pos[Min], nrnb, inputrec->delta_t, state->v,
idef->iparams, idef->il,
fr->ePBC, fr->bMolPBC, graph, cr, state->box);
}
if (nflexcon)
{
init_adir(fplog, shfc,
constr, idef, inputrec, cr, dd_ac1, mdstep, md, start, end,
x_old-start, state->x, pos[Min], force[Min], acc_dir-start,
fr->bMolPBC, state->box, state->lambda, &dum, nrnb);
directional_sd(fplog, pos[Min], pos[Try], acc_dir-start, start, end,
fr->fc_stepsize);
}
/* New positions, Steepest descent */
shell_pos_sd(fplog, pos[Min], pos[Try], force[Min], nshell, shell, count);
/* do_force expected the charge groups to be in the box */
if (graph)
{
unshift_self(graph, state->box, pos[Try]);
}
if (gmx_debug_at)
{
pr_rvecs(debug, 0, "RELAX: pos[Min] ", pos[Min] + start, homenr);
pr_rvecs(debug, 0, "RELAX: pos[Try] ", pos[Try] + start, homenr);
}
/* Try the new positions */
do_force(fplog, cr, inputrec, 1, nrnb, wcycle,
top, mtop, groups, state->box, pos[Try], &state->hist,
force[Try], force_vir,
md, enerd, fcd, state->lambda, graph,
fr, vsite, mu_tot, t, fp_field, NULL, bBornRadii,
force_flags);
if (gmx_debug_at)
{
pr_rvecs(debug, 0, "RELAX: force[Min]", force[Min] + start, homenr);
pr_rvecs(debug, 0, "RELAX: force[Try]", force[Try] + start, homenr);
}
sf_dir = 0;
if (nflexcon)
{
init_adir(fplog, shfc,
constr, idef, inputrec, cr, dd_ac1, mdstep, md, start, end,
x_old-start, state->x, pos[Try], force[Try], acc_dir-start,
fr->bMolPBC, state->box, state->lambda, &dum, nrnb);
for (i = start; i < end; i++)
{
sf_dir += md->massT[i]*norm2(acc_dir[i-start]);
}
}
Epot[Try] = enerd->term[F_EPOT];
df[Try] = rms_force(cr, force[Try], nshell, shell, nflexcon, &sf_dir, &Epot[Try]);
if (debug)
{
fprintf(debug, "df = %g %g\n", df[Min], df[Try]);
}
if (debug)
{
if (gmx_debug_at)
{
pr_rvecs(debug, 0, "F na do_force", force[Try] + start, homenr);
}
if (gmx_debug_at)
{
fprintf(debug, "SHELL ITER %d\n", count);
dump_shells(debug, pos[Try], force[Try], ftol, nshell, shell);
}
}
if (bVerbose && MASTER(cr))
{
print_epot(stdout, mdstep, count, Epot[Try], df[Try], nflexcon, sf_dir);
}
*bConverged = (df[Try] < ftol);
if ((df[Try] < df[Min]))
{
if (debug)
{
fprintf(debug, "Swapping Min and Try\n");
}
if (nflexcon)
{
/* Correct the velocities for the flexible constraints */
invdt = 1/inputrec->delta_t;
for (i = start; i < end; i++)
{
for (d = 0; d < DIM; d++)
{
state->v[i][d] += (pos[Try][i][d] - pos[Min][i][d])*invdt;
}
}
}
Min = Try;
}
else
{
decrease_step_size(nshell, shell);
}
}
if (MASTER(cr) && !(*bConverged))
{
/* Note that the energies and virial are incorrect when not converged */
if (fplog)
{
fprintf(fplog,
"step %s: EM did not converge in %d iterations, RMS force %.3f\n",
gmx_step_str(mdstep, sbuf), number_steps, df[Min]);
}
fprintf(stderr,
"step %s: EM did not converge in %d iterations, RMS force %.3f\n",
gmx_step_str(mdstep, sbuf), number_steps, df[Min]);
}
/* Copy back the coordinates and the forces */
memcpy(state->x, pos[Min], nat*sizeof(state->x[0]));
memcpy(f, force[Min], nat*sizeof(f[0]));
return count;
}
//! Implements C-style main function for mdrun
int gmx_mdrun(int argc, char *argv[])
{
const char *desc[] = {
"[THISMODULE] is the main computational chemistry engine",
"within GROMACS. Obviously, it performs Molecular Dynamics simulations,",
"but it can also perform Stochastic Dynamics, Energy Minimization,",
"test particle insertion or (re)calculation of energies.",
"Normal mode analysis is another option. In this case [TT]mdrun[tt]",
"builds a Hessian matrix from single conformation.",
"For usual Normal Modes-like calculations, make sure that",
"the structure provided is properly energy-minimized.",
"The generated matrix can be diagonalized by [gmx-nmeig].[PAR]",
"The [TT]mdrun[tt] program reads the run input file ([TT]-s[tt])",
"and distributes the topology over ranks if needed.",
"[TT]mdrun[tt] produces at least four output files.",
"A single log file ([TT]-g[tt]) is written.",
"The trajectory file ([TT]-o[tt]), contains coordinates, velocities and",
"optionally forces.",
"The structure file ([TT]-c[tt]) contains the coordinates and",
"velocities of the last step.",
"The energy file ([TT]-e[tt]) contains energies, the temperature,",
"pressure, etc, a lot of these things are also printed in the log file.",
"Optionally coordinates can be written to a compressed trajectory file",
"([TT]-x[tt]).[PAR]",
"The option [TT]-dhdl[tt] is only used when free energy calculation is",
"turned on.[PAR]",
"Running mdrun efficiently in parallel is a complex topic topic,",
"many aspects of which are covered in the online User Guide. You",
"should look there for practical advice on using many of the options",
"available in mdrun.[PAR]",
"ED (essential dynamics) sampling and/or additional flooding potentials",
"are switched on by using the [TT]-ei[tt] flag followed by an [REF].edi[ref]",
"file. The [REF].edi[ref] file can be produced with the [TT]make_edi[tt] tool",
"or by using options in the essdyn menu of the WHAT IF program.",
"[TT]mdrun[tt] produces a [REF].xvg[ref] output file that",
"contains projections of positions, velocities and forces onto selected",
"eigenvectors.[PAR]",
"When user-defined potential functions have been selected in the",
"[REF].mdp[ref] file the [TT]-table[tt] option is used to pass [TT]mdrun[tt]",
"a formatted table with potential functions. The file is read from",
"either the current directory or from the [TT]GMXLIB[tt] directory.",
"A number of pre-formatted tables are presented in the [TT]GMXLIB[tt] dir,",
"for 6-8, 6-9, 6-10, 6-11, 6-12 Lennard-Jones potentials with",
"normal Coulomb.",
"When pair interactions are present, a separate table for pair interaction",
"functions is read using the [TT]-tablep[tt] option.[PAR]",
"When tabulated bonded functions are present in the topology,",
"interaction functions are read using the [TT]-tableb[tt] option.",
"For each different tabulated interaction type the table file name is",
"modified in a different way: before the file extension an underscore is",
"appended, then a 'b' for bonds, an 'a' for angles or a 'd' for dihedrals",
"and finally the table number of the interaction type.[PAR]",
"The options [TT]-px[tt] and [TT]-pf[tt] are used for writing pull COM",
"coordinates and forces when pulling is selected",
"in the [REF].mdp[ref] file.[PAR]",
"Finally some experimental algorithms can be tested when the",
"appropriate options have been given. Currently under",
"investigation are: polarizability.",
"[PAR]",
"The option [TT]-membed[tt] does what used to be g_membed, i.e. embed",
"a protein into a membrane. This module requires a number of settings",
"that are provided in a data file that is the argument of this option.",
"For more details in membrane embedding, see the documentation in the",
"user guide. The options [TT]-mn[tt] and [TT]-mp[tt] are used to provide",
"the index and topology files used for the embedding.",
"[PAR]",
"The option [TT]-pforce[tt] is useful when you suspect a simulation",
"crashes due to too large forces. With this option coordinates and",
"forces of atoms with a force larger than a certain value will",
"be printed to stderr.",
"[PAR]",
"Checkpoints containing the complete state of the system are written",
"at regular intervals (option [TT]-cpt[tt]) to the file [TT]-cpo[tt],",
"unless option [TT]-cpt[tt] is set to -1.",
"The previous checkpoint is backed up to [TT]state_prev.cpt[tt] to",
"make sure that a recent state of the system is always available,",
"even when the simulation is terminated while writing a checkpoint.",
"With [TT]-cpnum[tt] all checkpoint files are kept and appended",
"with the step number.",
"A simulation can be continued by reading the full state from file",
"with option [TT]-cpi[tt]. This option is intelligent in the way that",
"if no checkpoint file is found, GROMACS just assumes a normal run and",
"starts from the first step of the [REF].tpr[ref] file. By default the output",
"will be appending to the existing output files. The checkpoint file",
"contains checksums of all output files, such that you will never",
"loose data when some output files are modified, corrupt or removed.",
"There are three scenarios with [TT]-cpi[tt]:[PAR]",
"[TT]*[tt] no files with matching names are present: new output files are written[PAR]",
"[TT]*[tt] all files are present with names and checksums matching those stored",
"in the checkpoint file: files are appended[PAR]",
"[TT]*[tt] otherwise no files are modified and a fatal error is generated[PAR]",
"With [TT]-noappend[tt] new output files are opened and the simulation",
"part number is added to all output file names.",
"Note that in all cases the checkpoint file itself is not renamed",
"and will be overwritten, unless its name does not match",
"the [TT]-cpo[tt] option.",
"[PAR]",
"With checkpointing the output is appended to previously written",
"output files, unless [TT]-noappend[tt] is used or none of the previous",
"output files are present (except for the checkpoint file).",
"The integrity of the files to be appended is verified using checksums",
"which are stored in the checkpoint file. This ensures that output can",
"not be mixed up or corrupted due to file appending. When only some",
"of the previous output files are present, a fatal error is generated",
"and no old output files are modified and no new output files are opened.",
"The result with appending will be the same as from a single run.",
"The contents will be binary identical, unless you use a different number",
"of ranks or dynamic load balancing or the FFT library uses optimizations",
"through timing.",
"[PAR]",
"With option [TT]-maxh[tt] a simulation is terminated and a checkpoint",
"file is written at the first neighbor search step where the run time",
"exceeds [TT]-maxh[tt]\\*0.99 hours. This option is particularly useful in",
"combination with setting [TT]nsteps[tt] to -1 either in the mdp or using the",
"similarly named command line option. This results in an infinite run,",
"terminated only when the time limit set by [TT]-maxh[tt] is reached (if any)"
"or upon receiving a signal."
"[PAR]",
"When [TT]mdrun[tt] receives a TERM signal, it will set nsteps to the current",
"step plus one. When [TT]mdrun[tt] receives an INT signal (e.g. when ctrl+C is",
"pressed), it will stop after the next neighbor search step ",
"(with nstlist=0 at the next step).",
"In both cases all the usual output will be written to file.",
"When running with MPI, a signal to one of the [TT]mdrun[tt] ranks",
"is sufficient, this signal should not be sent to mpirun or",
"the [TT]mdrun[tt] process that is the parent of the others.",
"[PAR]",
"Interactive molecular dynamics (IMD) can be activated by using at least one",
"of the three IMD switches: The [TT]-imdterm[tt] switch allows to terminate the",
"simulation from the molecular viewer (e.g. VMD). With [TT]-imdwait[tt],",
"[TT]mdrun[tt] pauses whenever no IMD client is connected. Pulling from the",
"IMD remote can be turned on by [TT]-imdpull[tt].",
"The port [TT]mdrun[tt] listens to can be altered by [TT]-imdport[tt].The",
"file pointed to by [TT]-if[tt] contains atom indices and forces if IMD",
"pulling is used."
"[PAR]",
"When [TT]mdrun[tt] is started with MPI, it does not run niced by default."
};
t_commrec *cr;
t_filenm fnm[] = {
{ efTPR, NULL, NULL, ffREAD },
{ efTRN, "-o", NULL, ffWRITE },
{ efCOMPRESSED, "-x", NULL, ffOPTWR },
{ efCPT, "-cpi", NULL, ffOPTRD | ffALLOW_MISSING },
{ efCPT, "-cpo", NULL, ffOPTWR },
{ efSTO, "-c", "confout", ffWRITE },
{ efEDR, "-e", "ener", ffWRITE },
{ efLOG, "-g", "md", ffWRITE },
{ efXVG, "-dhdl", "dhdl", ffOPTWR },
{ efXVG, "-field", "field", ffOPTWR },
{ efXVG, "-table", "table", ffOPTRD },
{ efXVG, "-tabletf", "tabletf", ffOPTRD },
{ efXVG, "-tablep", "tablep", ffOPTRD },
{ efXVG, "-tableb", "table", ffOPTRD },
{ efTRX, "-rerun", "rerun", ffOPTRD },
{ efXVG, "-tpi", "tpi", ffOPTWR },
{ efXVG, "-tpid", "tpidist", ffOPTWR },
{ efEDI, "-ei", "sam", ffOPTRD },
{ efXVG, "-eo", "edsam", ffOPTWR },
{ efXVG, "-devout", "deviatie", ffOPTWR },
{ efXVG, "-runav", "runaver", ffOPTWR },
{ efXVG, "-px", "pullx", ffOPTWR },
{ efXVG, "-pf", "pullf", ffOPTWR },
{ efXVG, "-ro", "rotation", ffOPTWR },
{ efLOG, "-ra", "rotangles", ffOPTWR },
{ efLOG, "-rs", "rotslabs", ffOPTWR },
{ efLOG, "-rt", "rottorque", ffOPTWR },
{ efMTX, "-mtx", "nm", ffOPTWR },
{ efNDX, "-dn", "dipole", ffOPTWR },
{ efRND, "-multidir", NULL, ffOPTRDMULT},
{ efDAT, "-membed", "membed", ffOPTRD },
{ efTOP, "-mp", "membed", ffOPTRD },
{ efNDX, "-mn", "membed", ffOPTRD },
{ efXVG, "-if", "imdforces", ffOPTWR },
{ efXVG, "-swap", "swapions", ffOPTWR }
};
const int NFILE = asize(fnm);
/* Command line options ! */
gmx_bool bDDBondCheck = TRUE;
gmx_bool bDDBondComm = TRUE;
gmx_bool bTunePME = TRUE;
gmx_bool bVerbose = FALSE;
gmx_bool bCompact = TRUE;
gmx_bool bRerunVSite = FALSE;
gmx_bool bConfout = TRUE;
gmx_bool bReproducible = FALSE;
gmx_bool bIMDwait = FALSE;
gmx_bool bIMDterm = FALSE;
gmx_bool bIMDpull = FALSE;
int npme = -1;
int nstlist = 0;
int nmultisim = 0;
int nstglobalcomm = -1;
int repl_ex_nst = 0;
int repl_ex_seed = -1;
int repl_ex_nex = 0;
int nstepout = 100;
int resetstep = -1;
gmx_int64_t nsteps = -2; /* the value -2 means that the mdp option will be used */
int imdport = 8888; /* can be almost anything, 8888 is easy to remember */
rvec realddxyz = {0, 0, 0};
const char *ddno_opt[ddnoNR+1] =
{ NULL, "interleave", "pp_pme", "cartesian", NULL };
const char *dddlb_opt[] =
{ NULL, "auto", "no", "yes", NULL };
const char *thread_aff_opt[threadaffNR+1] =
{ NULL, "auto", "on", "off", NULL };
const char *nbpu_opt[] =
{ NULL, "auto", "cpu", "gpu", "gpu_cpu", NULL };
real rdd = 0.0, rconstr = 0.0, dlb_scale = 0.8, pforce = -1;
char *ddcsx = NULL, *ddcsy = NULL, *ddcsz = NULL;
real cpt_period = 15.0, max_hours = -1;
gmx_bool bTryToAppendFiles = TRUE;
gmx_bool bKeepAndNumCPT = FALSE;
gmx_bool bResetCountersHalfWay = FALSE;
gmx_output_env_t *oenv = NULL;
/* Non transparent initialization of a complex gmx_hw_opt_t struct.
* But unfortunately we are not allowed to call a function here,
* since declarations follow below.
*/
gmx_hw_opt_t hw_opt = {
0, 0, 0, 0, threadaffSEL, 0, 0,
{ NULL, FALSE, 0, NULL }
};
t_pargs pa[] = {
{ "-dd", FALSE, etRVEC, {&realddxyz},
"Domain decomposition grid, 0 is optimize" },
{ "-ddorder", FALSE, etENUM, {ddno_opt},
"DD rank order" },
{ "-npme", FALSE, etINT, {&npme},
"Number of separate ranks to be used for PME, -1 is guess" },
{ "-nt", FALSE, etINT, {&hw_opt.nthreads_tot},
"Total number of threads to start (0 is guess)" },
{ "-ntmpi", FALSE, etINT, {&hw_opt.nthreads_tmpi},
"Number of thread-MPI threads to start (0 is guess)" },
{ "-ntomp", FALSE, etINT, {&hw_opt.nthreads_omp},
"Number of OpenMP threads per MPI rank to start (0 is guess)" },
{ "-ntomp_pme", FALSE, etINT, {&hw_opt.nthreads_omp_pme},
"Number of OpenMP threads per MPI rank to start (0 is -ntomp)" },
{ "-pin", FALSE, etENUM, {thread_aff_opt},
"Whether mdrun should try to set thread affinities" },
{ "-pinoffset", FALSE, etINT, {&hw_opt.core_pinning_offset},
"The lowest logical core number to which mdrun should pin the first thread" },
{ "-pinstride", FALSE, etINT, {&hw_opt.core_pinning_stride},
"Pinning distance in logical cores for threads, use 0 to minimize the number of threads per physical core" },
{ "-gpu_id", FALSE, etSTR, {&hw_opt.gpu_opt.gpu_id},
"List of GPU device id-s to use, specifies the per-node PP rank to GPU mapping" },
{ "-ddcheck", FALSE, etBOOL, {&bDDBondCheck},
"Check for all bonded interactions with DD" },
{ "-ddbondcomm", FALSE, etBOOL, {&bDDBondComm},
"HIDDENUse special bonded atom communication when [TT]-rdd[tt] > cut-off" },
{ "-rdd", FALSE, etREAL, {&rdd},
"The maximum distance for bonded interactions with DD (nm), 0 is determine from initial coordinates" },
{ "-rcon", FALSE, etREAL, {&rconstr},
"Maximum distance for P-LINCS (nm), 0 is estimate" },
{ "-dlb", FALSE, etENUM, {dddlb_opt},
"Dynamic load balancing (with DD)" },
{ "-dds", FALSE, etREAL, {&dlb_scale},
"Fraction in (0,1) by whose reciprocal the initial DD cell size will be increased in order to "
"provide a margin in which dynamic load balancing can act while preserving the minimum cell size." },
{ "-ddcsx", FALSE, etSTR, {&ddcsx},
"HIDDENA string containing a vector of the relative sizes in the x "
"direction of the corresponding DD cells. Only effective with static "
"load balancing." },
{ "-ddcsy", FALSE, etSTR, {&ddcsy},
"HIDDENA string containing a vector of the relative sizes in the y "
"direction of the corresponding DD cells. Only effective with static "
"load balancing." },
{ "-ddcsz", FALSE, etSTR, {&ddcsz},
"HIDDENA string containing a vector of the relative sizes in the z "
"direction of the corresponding DD cells. Only effective with static "
"load balancing." },
{ "-gcom", FALSE, etINT, {&nstglobalcomm},
"Global communication frequency" },
{ "-nb", FALSE, etENUM, {&nbpu_opt},
"Calculate non-bonded interactions on" },
{ "-nstlist", FALSE, etINT, {&nstlist},
"Set nstlist when using a Verlet buffer tolerance (0 is guess)" },
{ "-tunepme", FALSE, etBOOL, {&bTunePME},
"Optimize PME load between PP/PME ranks or GPU/CPU" },
{ "-v", FALSE, etBOOL, {&bVerbose},
"Be loud and noisy" },
{ "-compact", FALSE, etBOOL, {&bCompact},
"Write a compact log file" },
{ "-pforce", FALSE, etREAL, {&pforce},
"Print all forces larger than this (kJ/mol nm)" },
{ "-reprod", FALSE, etBOOL, {&bReproducible},
"Try to avoid optimizations that affect binary reproducibility" },
{ "-cpt", FALSE, etREAL, {&cpt_period},
"Checkpoint interval (minutes)" },
{ "-cpnum", FALSE, etBOOL, {&bKeepAndNumCPT},
"Keep and number checkpoint files" },
{ "-append", FALSE, etBOOL, {&bTryToAppendFiles},
"Append to previous output files when continuing from checkpoint instead of adding the simulation part number to all file names" },
{ "-nsteps", FALSE, etINT64, {&nsteps},
"Run this number of steps, overrides .mdp file option (-1 means infinite, -2 means use mdp option, smaller is invalid)" },
{ "-maxh", FALSE, etREAL, {&max_hours},
"Terminate after 0.99 times this time (hours)" },
{ "-multi", FALSE, etINT, {&nmultisim},
"Do multiple simulations in parallel" },
{ "-replex", FALSE, etINT, {&repl_ex_nst},
"Attempt replica exchange periodically with this period (steps)" },
{ "-nex", FALSE, etINT, {&repl_ex_nex},
"Number of random exchanges to carry out each exchange interval (N^3 is one suggestion). -nex zero or not specified gives neighbor replica exchange." },
{ "-reseed", FALSE, etINT, {&repl_ex_seed},
"Seed for replica exchange, -1 is generate a seed" },
{ "-imdport", FALSE, etINT, {&imdport},
"HIDDENIMD listening port" },
{ "-imdwait", FALSE, etBOOL, {&bIMDwait},
"HIDDENPause the simulation while no IMD client is connected" },
{ "-imdterm", FALSE, etBOOL, {&bIMDterm},
"HIDDENAllow termination of the simulation from IMD client" },
{ "-imdpull", FALSE, etBOOL, {&bIMDpull},
"HIDDENAllow pulling in the simulation from IMD client" },
{ "-rerunvsite", FALSE, etBOOL, {&bRerunVSite},
"HIDDENRecalculate virtual site coordinates with [TT]-rerun[tt]" },
{ "-confout", FALSE, etBOOL, {&bConfout},
"HIDDENWrite the last configuration with [TT]-c[tt] and force checkpointing at the last step" },
{ "-stepout", FALSE, etINT, {&nstepout},
"HIDDENFrequency of writing the remaining wall clock time for the run" },
{ "-resetstep", FALSE, etINT, {&resetstep},
"HIDDENReset cycle counters after these many time steps" },
{ "-resethway", FALSE, etBOOL, {&bResetCountersHalfWay},
"HIDDENReset the cycle counters after half the number of steps or halfway [TT]-maxh[tt]" }
};
unsigned long Flags;
ivec ddxyz;
int dd_node_order;
gmx_bool bDoAppendFiles, bStartFromCpt;
FILE *fplog;
int rc;
char **multidir = NULL;
cr = init_commrec();
unsigned long PCA_Flags = PCA_CAN_SET_DEFFNM;
// With -multi or -multidir, the file names are going to get processed
// further (or the working directory changed), so we can't check for their
// existence during parsing. It isn't useful to do any completion based on
// file system contents, either.
if (is_multisim_option_set(argc, argv))
{
PCA_Flags |= PCA_DISABLE_INPUT_FILE_CHECKING;
}
/* Comment this in to do fexist calls only on master
* works not with rerun or tables at the moment
* also comment out the version of init_forcerec in md.c
* with NULL instead of opt2fn
*/
/*
if (!MASTER(cr))
{
PCA_Flags |= PCA_NOT_READ_NODE;
}
*/
if (!parse_common_args(&argc, argv, PCA_Flags, NFILE, fnm, asize(pa), pa,
asize(desc), desc, 0, NULL, &oenv))
{
return 0;
}
/* we set these early because they might be used in init_multisystem()
Note that there is the potential for npme>nnodes until the number of
threads is set later on, if there's thread parallelization. That shouldn't
lead to problems. */
dd_node_order = nenum(ddno_opt);
cr->npmenodes = npme;
hw_opt.thread_affinity = nenum(thread_aff_opt);
/* now check the -multi and -multidir option */
if (opt2bSet("-multidir", NFILE, fnm))
{
if (nmultisim > 0)
{
gmx_fatal(FARGS, "mdrun -multi and -multidir options are mutually exclusive.");
}
nmultisim = opt2fns(&multidir, "-multidir", NFILE, fnm);
}
if (repl_ex_nst != 0 && nmultisim < 2)
{
gmx_fatal(FARGS, "Need at least two replicas for replica exchange (option -multi)");
}
if (repl_ex_nex < 0)
{
gmx_fatal(FARGS, "Replica exchange number of exchanges needs to be positive");
}
if (nmultisim >= 1)
{
#ifndef GMX_THREAD_MPI
gmx_bool bParFn = (multidir == NULL);
init_multisystem(cr, nmultisim, multidir, NFILE, fnm, bParFn);
#else
gmx_fatal(FARGS, "mdrun -multi or -multidir are not supported with the thread-MPI library. "
"Please compile GROMACS with a proper external MPI library.");
#endif
}
handleRestart(cr, bTryToAppendFiles, NFILE, fnm,
&bDoAppendFiles, &bStartFromCpt);
Flags = opt2bSet("-rerun", NFILE, fnm) ? MD_RERUN : 0;
Flags = Flags | (bDDBondCheck ? MD_DDBONDCHECK : 0);
Flags = Flags | (bDDBondComm ? MD_DDBONDCOMM : 0);
Flags = Flags | (bTunePME ? MD_TUNEPME : 0);
Flags = Flags | (bConfout ? MD_CONFOUT : 0);
Flags = Flags | (bRerunVSite ? MD_RERUN_VSITE : 0);
Flags = Flags | (bReproducible ? MD_REPRODUCIBLE : 0);
Flags = Flags | (bDoAppendFiles ? MD_APPENDFILES : 0);
Flags = Flags | (opt2parg_bSet("-append", asize(pa), pa) ? MD_APPENDFILESSET : 0);
Flags = Flags | (bKeepAndNumCPT ? MD_KEEPANDNUMCPT : 0);
Flags = Flags | (bStartFromCpt ? MD_STARTFROMCPT : 0);
Flags = Flags | (bResetCountersHalfWay ? MD_RESETCOUNTERSHALFWAY : 0);
Flags = Flags | (opt2parg_bSet("-ntomp", asize(pa), pa) ? MD_NTOMPSET : 0);
Flags = Flags | (bIMDwait ? MD_IMDWAIT : 0);
Flags = Flags | (bIMDterm ? MD_IMDTERM : 0);
Flags = Flags | (bIMDpull ? MD_IMDPULL : 0);
/* We postpone opening the log file if we are appending, so we can
first truncate the old log file and append to the correct position
there instead. */
if (MASTER(cr) && !bDoAppendFiles)
{
gmx_log_open(ftp2fn(efLOG, NFILE, fnm), cr,
Flags & MD_APPENDFILES, &fplog);
}
else
{
fplog = NULL;
}
ddxyz[XX] = (int)(realddxyz[XX] + 0.5);
ddxyz[YY] = (int)(realddxyz[YY] + 0.5);
ddxyz[ZZ] = (int)(realddxyz[ZZ] + 0.5);
rc = gmx::mdrunner(&hw_opt, fplog, cr, NFILE, fnm, oenv, bVerbose, bCompact,
nstglobalcomm, ddxyz, dd_node_order, rdd, rconstr,
dddlb_opt[0], dlb_scale, ddcsx, ddcsy, ddcsz,
nbpu_opt[0], nstlist,
nsteps, nstepout, resetstep,
nmultisim, repl_ex_nst, repl_ex_nex, repl_ex_seed,
pforce, cpt_period, max_hours, imdport, Flags);
/* Log file has to be closed in mdrunner if we are appending to it
(fplog not set here) */
if (MASTER(cr) && !bDoAppendFiles)
{
gmx_log_close(fplog);
}
return rc;
}
int mdrunner(gmx_hw_opt_t *hw_opt,
FILE *fplog, t_commrec *cr, int nfile,
const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
gmx_bool bCompact, int nstglobalcomm,
ivec ddxyz, int dd_node_order, real rdd, real rconstr,
const char *dddlb_opt, real dlb_scale,
const char *ddcsx, const char *ddcsy, const char *ddcsz,
const char *nbpu_opt, int nstlist_cmdline,
gmx_int64_t nsteps_cmdline, int nstepout, int resetstep,
int gmx_unused nmultisim, int repl_ex_nst, int repl_ex_nex,
int repl_ex_seed, real pforce, real cpt_period, real max_hours,
int imdport, unsigned long Flags)
{
gmx_bool bForceUseGPU, bTryUseGPU, bRerunMD;
t_inputrec *inputrec;
t_state *state = NULL;
matrix box;
gmx_ddbox_t ddbox = {0};
int npme_major, npme_minor;
t_nrnb *nrnb;
gmx_mtop_t *mtop = NULL;
t_mdatoms *mdatoms = NULL;
t_forcerec *fr = NULL;
t_fcdata *fcd = NULL;
real ewaldcoeff_q = 0;
real ewaldcoeff_lj = 0;
struct gmx_pme_t **pmedata = NULL;
gmx_vsite_t *vsite = NULL;
gmx_constr_t constr;
int nChargePerturbed = -1, nTypePerturbed = 0, status;
gmx_wallcycle_t wcycle;
gmx_bool bReadEkin;
gmx_walltime_accounting_t walltime_accounting = NULL;
int rc;
gmx_int64_t reset_counters;
gmx_edsam_t ed = NULL;
int nthreads_pme = 1;
int nthreads_pp = 1;
gmx_membed_t membed = NULL;
gmx_hw_info_t *hwinfo = NULL;
/* The master rank decides early on bUseGPU and broadcasts this later */
gmx_bool bUseGPU = FALSE;
/* CAUTION: threads may be started later on in this function, so
cr doesn't reflect the final parallel state right now */
snew(inputrec, 1);
snew(mtop, 1);
if (Flags & MD_APPENDFILES)
{
fplog = NULL;
}
bRerunMD = (Flags & MD_RERUN);
bForceUseGPU = (strncmp(nbpu_opt, "gpu", 3) == 0);
bTryUseGPU = (strncmp(nbpu_opt, "auto", 4) == 0) || bForceUseGPU;
/* Detect hardware, gather information. This is an operation that is
* global for this process (MPI rank). */
hwinfo = gmx_detect_hardware(fplog, cr, bTryUseGPU);
gmx_print_detected_hardware(fplog, cr, hwinfo);
if (fplog != NULL)
{
/* Print references after all software/hardware printing */
please_cite(fplog, "Abraham2015");
please_cite(fplog, "Pall2015");
please_cite(fplog, "Pronk2013");
please_cite(fplog, "Hess2008b");
please_cite(fplog, "Spoel2005a");
please_cite(fplog, "Lindahl2001a");
please_cite(fplog, "Berendsen95a");
}
snew(state, 1);
if (SIMMASTER(cr))
{
/* Read (nearly) all data required for the simulation */
read_tpx_state(ftp2fn(efTPR, nfile, fnm), inputrec, state, NULL, mtop);
if (inputrec->cutoff_scheme == ecutsVERLET)
{
/* Here the master rank decides if all ranks will use GPUs */
bUseGPU = (hwinfo->gpu_info.n_dev_compatible > 0 ||
getenv("GMX_EMULATE_GPU") != NULL);
/* TODO add GPU kernels for this and replace this check by:
* (bUseGPU && (ir->vdwtype == evdwPME &&
* ir->ljpme_combination_rule == eljpmeLB))
* update the message text and the content of nbnxn_acceleration_supported.
*/
if (bUseGPU &&
!nbnxn_gpu_acceleration_supported(fplog, cr, inputrec, bRerunMD))
{
/* Fallback message printed by nbnxn_acceleration_supported */
if (bForceUseGPU)
{
gmx_fatal(FARGS, "GPU acceleration requested, but not supported with the given input settings");
}
bUseGPU = FALSE;
}
prepare_verlet_scheme(fplog, cr,
inputrec, nstlist_cmdline, mtop, state->box,
bUseGPU);
}
else
{
if (nstlist_cmdline > 0)
{
gmx_fatal(FARGS, "Can not set nstlist with the group cut-off scheme");
}
if (hwinfo->gpu_info.n_dev_compatible > 0)
{
md_print_warn(cr, fplog,
"NOTE: GPU(s) found, but the current simulation can not use GPUs\n"
" To use a GPU, set the mdp option: cutoff-scheme = Verlet\n");
}
if (bForceUseGPU)
{
gmx_fatal(FARGS, "GPU requested, but can't be used without cutoff-scheme=Verlet");
}
#ifdef GMX_TARGET_BGQ
md_print_warn(cr, fplog,
"NOTE: There is no SIMD implementation of the group scheme kernels on\n"
" BlueGene/Q. You will observe better performance from using the\n"
" Verlet cut-off scheme.\n");
#endif
}
if (inputrec->eI == eiSD2)
{
md_print_warn(cr, fplog, "The stochastic dynamics integrator %s is deprecated, since\n"
"it is slower than integrator %s and is slightly less accurate\n"
"with constraints. Use the %s integrator.",
ei_names[inputrec->eI], ei_names[eiSD1], ei_names[eiSD1]);
}
}
/* Check and update the hardware options for internal consistency */
check_and_update_hw_opt_1(hw_opt, cr);
/* Early check for externally set process affinity. */
gmx_check_thread_affinity_set(fplog, cr,
hw_opt, hwinfo->nthreads_hw_avail, FALSE);
#ifdef GMX_THREAD_MPI
if (SIMMASTER(cr))
{
if (cr->npmenodes > 0 && hw_opt->nthreads_tmpi <= 0)
{
gmx_fatal(FARGS, "You need to explicitly specify the number of MPI threads (-ntmpi) when using separate PME ranks");
}
/* Since the master knows the cut-off scheme, update hw_opt for this.
* This is done later for normal MPI and also once more with tMPI
* for all tMPI ranks.
*/
check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
/* NOW the threads will be started: */
hw_opt->nthreads_tmpi = get_nthreads_mpi(hwinfo,
hw_opt,
inputrec, mtop,
cr, fplog, bUseGPU);
if (hw_opt->nthreads_tmpi > 1)
{
t_commrec *cr_old = cr;
/* now start the threads. */
cr = mdrunner_start_threads(hw_opt, fplog, cr_old, nfile, fnm,
oenv, bVerbose, bCompact, nstglobalcomm,
ddxyz, dd_node_order, rdd, rconstr,
dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz,
nbpu_opt, nstlist_cmdline,
nsteps_cmdline, nstepout, resetstep, nmultisim,
repl_ex_nst, repl_ex_nex, repl_ex_seed, pforce,
cpt_period, max_hours,
Flags);
/* the main thread continues here with a new cr. We don't deallocate
the old cr because other threads may still be reading it. */
if (cr == NULL)
{
gmx_comm("Failed to spawn threads");
}
}
}
#endif
/* END OF CAUTION: cr is now reliable */
/* g_membed initialisation *
* Because we change the mtop, init_membed is called before the init_parallel *
* (in case we ever want to make it run in parallel) */
if (opt2bSet("-membed", nfile, fnm))
{
if (MASTER(cr))
{
fprintf(stderr, "Initializing membed");
}
membed = init_membed(fplog, nfile, fnm, mtop, inputrec, state, cr, &cpt_period);
}
if (PAR(cr))
{
/* now broadcast everything to the non-master nodes/threads: */
init_parallel(cr, inputrec, mtop);
/* The master rank decided on the use of GPUs,
* broadcast this information to all ranks.
*/
gmx_bcast_sim(sizeof(bUseGPU), &bUseGPU, cr);
}
if (fplog != NULL)
{
pr_inputrec(fplog, 0, "Input Parameters", inputrec, FALSE);
fprintf(fplog, "\n");
}
/* now make sure the state is initialized and propagated */
set_state_entries(state, inputrec);
/* A parallel command line option consistency check that we can
only do after any threads have started. */
if (!PAR(cr) &&
(ddxyz[XX] > 1 || ddxyz[YY] > 1 || ddxyz[ZZ] > 1 || cr->npmenodes > 0))
{
gmx_fatal(FARGS,
"The -dd or -npme option request a parallel simulation, "
#ifndef GMX_MPI
"but %s was compiled without threads or MPI enabled"
#else
#ifdef GMX_THREAD_MPI
"but the number of threads (option -nt) is 1"
#else
"but %s was not started through mpirun/mpiexec or only one rank was requested through mpirun/mpiexec"
#endif
#endif
, output_env_get_program_display_name(oenv)
);
}
if (bRerunMD &&
(EI_ENERGY_MINIMIZATION(inputrec->eI) || eiNM == inputrec->eI))
{
gmx_fatal(FARGS, "The .mdp file specified an energy mininization or normal mode algorithm, and these are not compatible with mdrun -rerun");
}
if (can_use_allvsall(inputrec, TRUE, cr, fplog) && DOMAINDECOMP(cr))
{
gmx_fatal(FARGS, "All-vs-all loops do not work with domain decomposition, use a single MPI rank");
}
if (!(EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype)))
{
if (cr->npmenodes > 0)
{
gmx_fatal_collective(FARGS, cr, NULL,
"PME-only ranks are requested, but the system does not use PME for electrostatics or LJ");
}
cr->npmenodes = 0;
}
if (bUseGPU && cr->npmenodes < 0)
{
/* With GPUs we don't automatically use PME-only ranks. PME ranks can
* improve performance with many threads per GPU, since our OpenMP
* scaling is bad, but it's difficult to automate the setup.
*/
cr->npmenodes = 0;
}
#ifdef GMX_FAHCORE
if (MASTER(cr))
{
fcRegisterSteps(inputrec->nsteps, inputrec->init_step);
}
#endif
/* NMR restraints must be initialized before load_checkpoint,
* since with time averaging the history is added to t_state.
* For proper consistency check we therefore need to extend
* t_state here.
* So the PME-only nodes (if present) will also initialize
* the distance restraints.
*/
snew(fcd, 1);
/* This needs to be called before read_checkpoint to extend the state */
init_disres(fplog, mtop, inputrec, cr, fcd, state, repl_ex_nst > 0);
init_orires(fplog, mtop, state->x, inputrec, cr, &(fcd->orires),
state);
if (DEFORM(*inputrec))
{
/* Store the deform reference box before reading the checkpoint */
if (SIMMASTER(cr))
{
copy_mat(state->box, box);
}
if (PAR(cr))
{
gmx_bcast(sizeof(box), box, cr);
}
/* Because we do not have the update struct available yet
* in which the reference values should be stored,
* we store them temporarily in static variables.
* This should be thread safe, since they are only written once
* and with identical values.
*/
tMPI_Thread_mutex_lock(&deform_init_box_mutex);
deform_init_init_step_tpx = inputrec->init_step;
copy_mat(box, deform_init_box_tpx);
tMPI_Thread_mutex_unlock(&deform_init_box_mutex);
}
if (opt2bSet("-cpi", nfile, fnm))
{
/* Check if checkpoint file exists before doing continuation.
* This way we can use identical input options for the first and subsequent runs...
*/
if (gmx_fexist_master(opt2fn_master("-cpi", nfile, fnm, cr), cr) )
{
load_checkpoint(opt2fn_master("-cpi", nfile, fnm, cr), &fplog,
cr, ddxyz,
inputrec, state, &bReadEkin,
(Flags & MD_APPENDFILES),
(Flags & MD_APPENDFILESSET));
if (bReadEkin)
{
Flags |= MD_READ_EKIN;
}
}
}
if (MASTER(cr) && (Flags & MD_APPENDFILES))
{
gmx_log_open(ftp2fn(efLOG, nfile, fnm), cr,
Flags, &fplog);
}
/* override nsteps with value from cmdline */
override_nsteps_cmdline(fplog, nsteps_cmdline, inputrec, cr);
if (SIMMASTER(cr))
{
copy_mat(state->box, box);
}
if (PAR(cr))
{
gmx_bcast(sizeof(box), box, cr);
}
/* Essential dynamics */
if (opt2bSet("-ei", nfile, fnm))
{
/* Open input and output files, allocate space for ED data structure */
ed = ed_open(mtop->natoms, &state->edsamstate, nfile, fnm, Flags, oenv, cr);
}
if (PAR(cr) && !(EI_TPI(inputrec->eI) ||
inputrec->eI == eiNM))
{
cr->dd = init_domain_decomposition(fplog, cr, Flags, ddxyz, rdd, rconstr,
dddlb_opt, dlb_scale,
ddcsx, ddcsy, ddcsz,
mtop, inputrec,
box, state->x,
&ddbox, &npme_major, &npme_minor);
make_dd_communicators(fplog, cr, dd_node_order);
/* Set overallocation to avoid frequent reallocation of arrays */
set_over_alloc_dd(TRUE);
}
else
{
/* PME, if used, is done on all nodes with 1D decomposition */
cr->npmenodes = 0;
cr->duty = (DUTY_PP | DUTY_PME);
npme_major = 1;
npme_minor = 1;
if (inputrec->ePBC == epbcSCREW)
{
gmx_fatal(FARGS,
"pbc=%s is only implemented with domain decomposition",
epbc_names[inputrec->ePBC]);
}
}
if (PAR(cr))
{
/* After possible communicator splitting in make_dd_communicators.
* we can set up the intra/inter node communication.
*/
gmx_setup_nodecomm(fplog, cr);
}
/* Initialize per-physical-node MPI process/thread ID and counters. */
gmx_init_intranode_counters(cr);
#ifdef GMX_MPI
if (MULTISIM(cr))
{
md_print_info(cr, fplog,
"This is simulation %d out of %d running as a composite GROMACS\n"
"multi-simulation job. Setup for this simulation:\n\n",
cr->ms->sim, cr->ms->nsim);
}
md_print_info(cr, fplog, "Using %d MPI %s\n",
cr->nnodes,
#ifdef GMX_THREAD_MPI
cr->nnodes == 1 ? "thread" : "threads"
#else
cr->nnodes == 1 ? "process" : "processes"
#endif
);
fflush(stderr);
#endif
/* Check and update hw_opt for the cut-off scheme */
check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
/* Check and update hw_opt for the number of MPI ranks */
check_and_update_hw_opt_3(hw_opt);
gmx_omp_nthreads_init(fplog, cr,
hwinfo->nthreads_hw_avail,
hw_opt->nthreads_omp,
hw_opt->nthreads_omp_pme,
(cr->duty & DUTY_PP) == 0,
inputrec->cutoff_scheme == ecutsVERLET);
#ifndef NDEBUG
if (integrator[inputrec->eI].func != do_tpi &&
inputrec->cutoff_scheme == ecutsVERLET)
{
gmx_feenableexcept();
}
#endif
if (bUseGPU)
{
/* Select GPU id's to use */
gmx_select_gpu_ids(fplog, cr, &hwinfo->gpu_info, bForceUseGPU,
&hw_opt->gpu_opt);
}
else
{
/* Ignore (potentially) manually selected GPUs */
hw_opt->gpu_opt.n_dev_use = 0;
}
/* check consistency across ranks of things like SIMD
* support and number of GPUs selected */
gmx_check_hw_runconf_consistency(fplog, hwinfo, cr, hw_opt, bUseGPU);
/* Now that we know the setup is consistent, check for efficiency */
check_resource_division_efficiency(hwinfo, hw_opt, Flags & MD_NTOMPSET,
cr, fplog);
if (DOMAINDECOMP(cr))
{
/* When we share GPUs over ranks, we need to know this for the DLB */
dd_setup_dlb_resource_sharing(cr, hwinfo, hw_opt);
}
/* getting number of PP/PME threads
PME: env variable should be read only on one node to make sure it is
identical everywhere;
*/
/* TODO nthreads_pp is only used for pinning threads.
* This is a temporary solution until we have a hw topology library.
*/
nthreads_pp = gmx_omp_nthreads_get(emntNonbonded);
nthreads_pme = gmx_omp_nthreads_get(emntPME);
wcycle = wallcycle_init(fplog, resetstep, cr, nthreads_pp, nthreads_pme);
if (PAR(cr))
{
/* Master synchronizes its value of reset_counters with all nodes
* including PME only nodes */
reset_counters = wcycle_get_reset_counters(wcycle);
gmx_bcast_sim(sizeof(reset_counters), &reset_counters, cr);
wcycle_set_reset_counters(wcycle, reset_counters);
}
snew(nrnb, 1);
if (cr->duty & DUTY_PP)
{
bcast_state(cr, state);
/* Initiate forcerecord */
fr = mk_forcerec();
fr->hwinfo = hwinfo;
fr->gpu_opt = &hw_opt->gpu_opt;
init_forcerec(fplog, oenv, fr, fcd, inputrec, mtop, cr, box,
opt2fn("-table", nfile, fnm),
opt2fn("-tabletf", nfile, fnm),
opt2fn("-tablep", nfile, fnm),
opt2fn("-tableb", nfile, fnm),
nbpu_opt,
FALSE,
pforce);
/* version for PCA_NOT_READ_NODE (see md.c) */
/*init_forcerec(fplog,fr,fcd,inputrec,mtop,cr,box,FALSE,
"nofile","nofile","nofile","nofile",FALSE,pforce);
*/
/* Initialize QM-MM */
if (fr->bQMMM)
{
init_QMMMrec(cr, mtop, inputrec, fr);
}
/* Initialize the mdatoms structure.
* mdatoms is not filled with atom data,
* as this can not be done now with domain decomposition.
*/
mdatoms = init_mdatoms(fplog, mtop, inputrec->efep != efepNO);
/* Initialize the virtual site communication */
vsite = init_vsite(mtop, cr, FALSE);
calc_shifts(box, fr->shift_vec);
/* With periodic molecules the charge groups should be whole at start up
* and the virtual sites should not be far from their proper positions.
*/
if (!inputrec->bContinuation && MASTER(cr) &&
!(inputrec->ePBC != epbcNONE && inputrec->bPeriodicMols))
{
/* Make molecules whole at start of run */
if (fr->ePBC != epbcNONE)
{
do_pbc_first_mtop(fplog, inputrec->ePBC, box, mtop, state->x);
}
if (vsite)
{
/* Correct initial vsite positions are required
* for the initial distribution in the domain decomposition
* and for the initial shell prediction.
*/
construct_vsites_mtop(vsite, mtop, state->x);
}
}
if (EEL_PME(fr->eeltype) || EVDW_PME(fr->vdwtype))
{
ewaldcoeff_q = fr->ewaldcoeff_q;
ewaldcoeff_lj = fr->ewaldcoeff_lj;
pmedata = &fr->pmedata;
}
else
{
pmedata = NULL;
}
}
else
{
/* This is a PME only node */
/* We don't need the state */
done_state(state);
ewaldcoeff_q = calc_ewaldcoeff_q(inputrec->rcoulomb, inputrec->ewald_rtol);
ewaldcoeff_lj = calc_ewaldcoeff_lj(inputrec->rvdw, inputrec->ewald_rtol_lj);
snew(pmedata, 1);
}
if (hw_opt->thread_affinity != threadaffOFF)
{
/* Before setting affinity, check whether the affinity has changed
* - which indicates that probably the OpenMP library has changed it
* since we first checked).
*/
gmx_check_thread_affinity_set(fplog, cr,
hw_opt, hwinfo->nthreads_hw_avail, TRUE);
/* Set the CPU affinity */
gmx_set_thread_affinity(fplog, cr, hw_opt, hwinfo);
}
/* Initiate PME if necessary,
* either on all nodes or on dedicated PME nodes only. */
if (EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype))
{
if (mdatoms)
{
nChargePerturbed = mdatoms->nChargePerturbed;
if (EVDW_PME(inputrec->vdwtype))
{
nTypePerturbed = mdatoms->nTypePerturbed;
}
}
if (cr->npmenodes > 0)
{
/* The PME only nodes need to know nChargePerturbed(FEP on Q) and nTypePerturbed(FEP on LJ)*/
gmx_bcast_sim(sizeof(nChargePerturbed), &nChargePerturbed, cr);
gmx_bcast_sim(sizeof(nTypePerturbed), &nTypePerturbed, cr);
}
if (cr->duty & DUTY_PME)
{
status = gmx_pme_init(pmedata, cr, npme_major, npme_minor, inputrec,
mtop ? mtop->natoms : 0, nChargePerturbed, nTypePerturbed,
(Flags & MD_REPRODUCIBLE), nthreads_pme);
if (status != 0)
{
gmx_fatal(FARGS, "Error %d initializing PME", status);
}
}
}
if (integrator[inputrec->eI].func == do_md)
{
/* Turn on signal handling on all nodes */
/*
* (A user signal from the PME nodes (if any)
* is communicated to the PP nodes.
*/
signal_handler_install();
}
if (cr->duty & DUTY_PP)
{
/* Assumes uniform use of the number of OpenMP threads */
walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntDefault));
if (inputrec->bPull)
{
/* Initialize pull code */
inputrec->pull_work =
init_pull(fplog, inputrec->pull, inputrec, nfile, fnm,
mtop, cr, oenv, inputrec->fepvals->init_lambda,
EI_DYNAMICS(inputrec->eI) && MASTER(cr), Flags);
}
if (inputrec->bRot)
{
/* Initialize enforced rotation code */
init_rot(fplog, inputrec, nfile, fnm, cr, state->x, box, mtop, oenv,
bVerbose, Flags);
}
if (inputrec->eSwapCoords != eswapNO)
{
/* Initialize ion swapping code */
init_swapcoords(fplog, bVerbose, inputrec, opt2fn_master("-swap", nfile, fnm, cr),
mtop, state->x, state->box, &state->swapstate, cr, oenv, Flags);
}
constr = init_constraints(fplog, mtop, inputrec, ed, state, cr);
if (DOMAINDECOMP(cr))
{
GMX_RELEASE_ASSERT(fr, "fr was NULL while cr->duty was DUTY_PP");
dd_init_bondeds(fplog, cr->dd, mtop, vsite, inputrec,
Flags & MD_DDBONDCHECK, fr->cginfo_mb);
set_dd_parameters(fplog, cr->dd, dlb_scale, inputrec, &ddbox);
setup_dd_grid(fplog, cr->dd);
}
/* Now do whatever the user wants us to do (how flexible...) */
integrator[inputrec->eI].func(fplog, cr, nfile, fnm,
oenv, bVerbose, bCompact,
nstglobalcomm,
vsite, constr,
nstepout, inputrec, mtop,
fcd, state,
mdatoms, nrnb, wcycle, ed, fr,
repl_ex_nst, repl_ex_nex, repl_ex_seed,
membed,
cpt_period, max_hours,
imdport,
Flags,
walltime_accounting);
if (inputrec->bPull)
{
finish_pull(inputrec->pull_work);
}
if (inputrec->bRot)
{
finish_rot(inputrec->rot);
}
}
else
{
GMX_RELEASE_ASSERT(pmedata, "pmedata was NULL while cr->duty was not DUTY_PP");
/* do PME only */
walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntPME));
gmx_pmeonly(*pmedata, cr, nrnb, wcycle, walltime_accounting, ewaldcoeff_q, ewaldcoeff_lj, inputrec);
}
wallcycle_stop(wcycle, ewcRUN);
/* Finish up, write some stuff
* if rerunMD, don't write last frame again
*/
finish_run(fplog, cr,
inputrec, nrnb, wcycle, walltime_accounting,
fr ? fr->nbv : NULL,
EI_DYNAMICS(inputrec->eI) && !MULTISIM(cr));
/* Free GPU memory and context */
free_gpu_resources(fr, cr, &hwinfo->gpu_info, fr ? fr->gpu_opt : NULL);
if (opt2bSet("-membed", nfile, fnm))
{
sfree(membed);
}
gmx_hardware_info_free(hwinfo);
/* Does what it says */
print_date_and_time(fplog, cr->nodeid, "Finished mdrun", gmx_gettime());
walltime_accounting_destroy(walltime_accounting);
/* PLUMED */
if(plumedswitch){
plumed_finalize(plumedmain);
}
/* END PLUMED */
/* Close logfile already here if we were appending to it */
if (MASTER(cr) && (Flags & MD_APPENDFILES))
{
gmx_log_close(fplog);
}
rc = (int)gmx_get_stop_condition();
done_ed(&ed);
#ifdef GMX_THREAD_MPI
/* we need to join all threads. The sub-threads join when they
exit this function, but the master thread needs to be told to
wait for that. */
if (PAR(cr) && MASTER(cr))
{
tMPI_Finalize();
}
#endif
return rc;
}
int RowDescription(POOL_CONNECTION *frontend,
POOL_CONNECTION_POOL *backend,
short *result)
{
short num_fields, num_fields1 = 0;
int oid, mod;
int oid1, mod1;
short size, size1;
char *string;
int len, len1;
int i;
pool_read(MASTER(backend), &num_fields, sizeof(short));
num_fields1 = num_fields;
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i) && !IS_MASTER_NODE_ID(i))
{
/* # of fields (could be 0) */
pool_read(CONNECTION(backend, i), &num_fields, sizeof(short));
if (num_fields != num_fields1)
{
pool_error("RowDescription: num_fields does not match between backends master(%d) and %d th backend(%d)",
num_fields, i, num_fields1);
return POOL_FATAL;
}
}
}
/* forward it to the frontend */
pool_write(frontend, "T", 1);
pool_write(frontend, &num_fields, sizeof(short));
num_fields = ntohs(num_fields);
for (i = 0;i<num_fields;i++)
{
int j;
/* field name */
string = pool_read_string(MASTER(backend), &len, 0);
if (string == NULL)
return POOL_END;
len1 = len;
if (pool_write(frontend, string, len) < 0)
return POOL_END;
for (j=0;j<NUM_BACKENDS;j++)
{
if (VALID_BACKEND(j) && !IS_MASTER_NODE_ID(j))
{
string = pool_read_string(CONNECTION(backend, j), &len, 0);
if (string == NULL)
return POOL_END;
if (len != len1)
{
pool_error("RowDescription: field length does not match between backends master(%d) and %d th backend(%d)",
ntohl(len), j, ntohl(len1));
return POOL_FATAL;
}
}
}
/* type oid */
pool_read(MASTER(backend), &oid, sizeof(int));
oid1 = oid;
pool_debug("RowDescription: type oid: %d", ntohl(oid));
for (j=0;j<NUM_BACKENDS;j++)
{
if (VALID_BACKEND(j) && !IS_MASTER_NODE_ID(j))
{
pool_read(CONNECTION(backend, j), &oid, sizeof(int));
/* we do not regard oid mismatch as fatal */
if (oid != oid1)
{
pool_debug("RowDescription: field oid does not match between backends master(%d) and %d th backend(%d)",
ntohl(oid), j, ntohl(oid1));
}
}
}
if (pool_write(frontend, &oid1, sizeof(int)) < 0)
return POOL_END;
/* size */
pool_read(MASTER(backend), &size, sizeof(short));
size1 = size;
for (j=0;j<NUM_BACKENDS;j++)
{
if (VALID_BACKEND(j) && !IS_MASTER_NODE_ID(j))
{
pool_read(CONNECTION(backend, j), &size, sizeof(short));
if (size1 != size1)
{
pool_error("RowDescription: field size does not match between backends master(%d) and %d th backend(%d)",
ntohs(size), j, ntohs(size1));
return POOL_FATAL;
}
}
}
pool_debug("RowDescription: field size: %d", ntohs(size));
pool_write(frontend, &size1, sizeof(short));
/* modifier */
pool_read(MASTER(backend), &mod, sizeof(int));
pool_debug("RowDescription: modifier: %d", ntohs(mod));
mod1 = mod;
for (j=0;j<NUM_BACKENDS;j++)
{
if (VALID_BACKEND(j) && !IS_MASTER_NODE_ID(j))
{
pool_read(CONNECTION(backend, j), &mod, sizeof(int));
if (mod != mod1)
{
pool_debug("RowDescription: modifier does not match between backends master(%d) and %d th backend(%d)",
ntohl(mod), j, ntohl(mod1));
}
}
}
if (pool_write(frontend, &mod1, sizeof(int)) < 0)
return POOL_END;
}
*result = num_fields;
return pool_flush(frontend);
}
void mdoutf_write_to_trajectory_files(FILE *fplog, t_commrec *cr,
gmx_mdoutf_t of,
int mdof_flags,
gmx_mtop_t *top_global,
gmx_int64_t step, double t,
t_state *state_local, t_state *state_global,
rvec *f_local, rvec *f_global)
{
rvec *local_v;
rvec *global_v;
/* MRS -- defining these variables is to manage the difference
* between half step and full step velocities, but there must be a better way . . . */
local_v = state_local->v;
global_v = state_global->v;
if (DOMAINDECOMP(cr))
{
if (mdof_flags & MDOF_CPT)
{
dd_collect_state(cr->dd, state_local, state_global);
}
else
{
if (mdof_flags & (MDOF_X | MDOF_X_COMPRESSED))
{
dd_collect_vec(cr->dd, state_local, state_local->x,
state_global->x);
}
if (mdof_flags & MDOF_V)
{
dd_collect_vec(cr->dd, state_local, local_v,
global_v);
}
}
if (mdof_flags & MDOF_F)
{
dd_collect_vec(cr->dd, state_local, f_local, f_global);
}
}
else
{
if (mdof_flags & MDOF_CPT)
{
/* All pointers in state_local are equal to state_global,
* but we need to copy the non-pointer entries.
*/
state_global->lambda = state_local->lambda;
state_global->veta = state_local->veta;
state_global->vol0 = state_local->vol0;
copy_mat(state_local->box, state_global->box);
copy_mat(state_local->boxv, state_global->boxv);
copy_mat(state_local->svir_prev, state_global->svir_prev);
copy_mat(state_local->fvir_prev, state_global->fvir_prev);
copy_mat(state_local->pres_prev, state_global->pres_prev);
}
}
if (MASTER(cr))
{
if (mdof_flags & MDOF_CPT)
{
fflush_tng(of->tng);
fflush_tng(of->tng_low_prec);
write_checkpoint(of->fn_cpt, of->bKeepAndNumCPT,
fplog, cr, of->eIntegrator, of->simulation_part,
of->bExpanded, of->elamstats, step, t, state_global);
}
if (mdof_flags & (MDOF_X | MDOF_V | MDOF_F))
{
if (of->fp_trn)
{
gmx_trr_write_frame(of->fp_trn, step, t, state_local->lambda[efptFEP],
state_local->box, top_global->natoms,
(mdof_flags & MDOF_X) ? state_global->x : NULL,
(mdof_flags & MDOF_V) ? global_v : NULL,
(mdof_flags & MDOF_F) ? f_global : NULL);
if (gmx_fio_flush(of->fp_trn) != 0)
{
gmx_file("Cannot write trajectory; maybe you are out of disk space?");
}
}
gmx_fwrite_tng(of->tng, FALSE, step, t, state_local->lambda[efptFEP],
state_local->box,
top_global->natoms,
(mdof_flags & MDOF_X) ? state_global->x : NULL,
(mdof_flags & MDOF_V) ? global_v : NULL,
(mdof_flags & MDOF_F) ? f_global : NULL);
}
if (mdof_flags & MDOF_X_COMPRESSED)
{
rvec *xxtc = NULL;
if (of->natoms_x_compressed == of->natoms_global)
{
/* We are writing the positions of all of the atoms to
the compressed output */
xxtc = state_global->x;
}
else
{
/* We are writing the positions of only a subset of
the atoms to the compressed output, so we have to
make a copy of the subset of coordinates. */
int i, j;
snew(xxtc, of->natoms_x_compressed);
for (i = 0, j = 0; (i < of->natoms_global); i++)
{
if (ggrpnr(of->groups, egcCompressedX, i) == 0)
{
copy_rvec(state_global->x[i], xxtc[j++]);
}
}
}
if (write_xtc(of->fp_xtc, of->natoms_x_compressed, step, t,
state_local->box, xxtc, of->x_compression_precision) == 0)
{
gmx_fatal(FARGS, "XTC error - maybe you are out of disk space?");
}
gmx_fwrite_tng(of->tng_low_prec,
TRUE,
step,
t,
state_local->lambda[efptFEP],
state_local->box,
of->natoms_x_compressed,
xxtc,
NULL,
NULL);
if (of->natoms_x_compressed != of->natoms_global)
{
sfree(xxtc);
}
}
}
}
void init_forcerec(FILE *fp,
t_forcerec *fr,
t_inputrec *ir,
t_topology *top,
t_commrec *cr,
t_mdatoms *mdatoms,
t_nsborder *nsb,
matrix box,
bool bMolEpot,
char *tabfn,
bool bNoSolvOpt)
{
int i,j,m,natoms,ngrp,tabelemsize;
real q,zsq,nrdf,T;
rvec box_size;
double rtab;
t_block *mols,*cgs;
t_idef *idef;
if (check_box(box))
fatal_error(0,check_box(box));
cgs = &(top->blocks[ebCGS]);
mols = &(top->blocks[ebMOLS]);
idef = &(top->idef);
natoms = mdatoms->nr;
/* Shell stuff */
fr->fc_stepsize = ir->fc_stepsize;
/* Free energy */
fr->efep = ir->efep;
fr->sc_alpha = ir->sc_alpha;
fr->sc_sigma6 = pow(ir->sc_sigma,6);
/* Neighbour searching stuff */
fr->bGrid = (ir->ns_type == ensGRID);
fr->ndelta = ir->ndelta;
fr->ePBC = ir->ePBC;
fr->rlist = ir->rlist;
fr->rlistlong = max(ir->rlist,max(ir->rcoulomb,ir->rvdw));
fr->eeltype = ir->coulombtype;
fr->vdwtype = ir->vdwtype;
fr->bTwinRange = fr->rlistlong > fr->rlist;
fr->bEwald = fr->eeltype==eelPME || fr->eeltype==eelEWALD;
fr->bvdwtab = fr->vdwtype != evdwCUT;
fr->bRF = (fr->eeltype==eelRF || fr->eeltype==eelGRF) &&
fr->vdwtype==evdwCUT;
fr->bcoultab = (fr->eeltype!=eelCUT && !fr->bRF) || fr->bEwald;
#ifndef SPEC_CPU
if (getenv("GMX_FORCE_TABLES")) {
fr->bvdwtab = TRUE;
fr->bcoultab = TRUE;
}
#endif
if (fp) {
fprintf(fp,"Table routines are used for coulomb: %s\n",bool_names[fr->bcoultab]);
fprintf(fp,"Table routines are used for vdw: %s\n",bool_names[fr->bvdwtab ]);
}
/* Tables are used for direct ewald sum */
if(fr->bEwald) {
fr->ewaldcoeff=calc_ewaldcoeff(ir->rcoulomb, ir->ewald_rtol);
if (fp)
fprintf(fp,"Using a Gaussian width (1/beta) of %g nm for Ewald\n",
1/fr->ewaldcoeff);
}
/* Domain decomposition parallellism... */
fr->bDomDecomp = ir->bDomDecomp;
fr->Dimension = ir->decomp_dir;
/* Electrostatics */
fr->epsilon_r = ir->epsilon_r;
fr->fudgeQQ = ir->fudgeQQ;
fr->rcoulomb_switch = ir->rcoulomb_switch;
fr->rcoulomb = ir->rcoulomb;
#ifndef SPEC_CPU
if (bNoSolvOpt || getenv("GMX_NO_SOLV_OPT"))
fr->bSolvOpt = FALSE;
else
#endif
fr->bSolvOpt = TRUE;
/* Parameters for generalized RF */
fr->zsquare = 0.0;
fr->temp = 0.0;
if (fr->eeltype == eelGRF) {
zsq = 0.0;
for (i=0; (i<cgs->nr); i++) {
q = 0;
for(j=cgs->index[i]; (j<cgs->index[i+1]); j++)
q+=mdatoms->chargeT[cgs->a[j]];
if (fabs(q) > GMX_REAL_MIN)
/* Changed from square to fabs 990314 DvdS
* Does not make a difference for monovalent ions, but doe for
* divalent ions (Ca2+!!)
*/
zsq += fabs(q);
}
fr->zsquare = zsq;
T = 0.0;
nrdf = 0.0;
for(i=0; (i<ir->opts.ngtc); i++) {
nrdf += ir->opts.nrdf[i];
T += (ir->opts.nrdf[i] * ir->opts.ref_t[i]);
}
if (nrdf < GMX_REAL_MIN)
fatal_error(0,"No degrees of freedom!");
fr->temp = T/nrdf;
}
else if (EEL_LR(fr->eeltype) || (fr->eeltype == eelSHIFT) ||
(fr->eeltype == eelUSER) || (fr->eeltype == eelSWITCH)) {
/* We must use the long range cut-off for neighboursearching...
* An extra range of e.g. 0.1 nm (half the size of a charge group)
* is necessary for neighboursearching. This allows diffusion
* into the cut-off range (between neighborlist updates),
* and gives more accurate forces because all atoms within the short-range
* cut-off rc must be taken into account, while the ns criterium takes
* only those with the center of geometry within the cut-off.
* (therefore we have to add half the size of a charge group, plus
* something to account for diffusion if we have nstlist > 1)
*/
for(m=0; (m<DIM); m++)
box_size[m]=box[m][m];
if (fr->phi == NULL)
snew(fr->phi,mdatoms->nr);
if ((fr->eeltype==eelPPPM) || (fr->eeltype==eelPOISSON) ||
(fr->eeltype == eelSHIFT && fr->rcoulomb > fr->rcoulomb_switch))
set_shift_consts(fp,fr->rcoulomb_switch,fr->rcoulomb,box_size,fr);
}
/* Initiate arrays */
if (fr->bTwinRange) {
snew(fr->f_twin,natoms);
snew(fr->fshift_twin,SHIFTS);
}
if (EEL_LR(fr->eeltype)) {
snew(fr->f_pme,natoms);
}
/* Mask that says whether or not this NBF list should be computed */
/* if (fr->bMask == NULL) {
ngrp = ir->opts.ngener*ir->opts.ngener;
snew(fr->bMask,ngrp);*/
/* Defaults to always */
/* for(i=0; (i<ngrp); i++)
fr->bMask[i] = TRUE;
}*/
if (fr->cg_cm == NULL)
snew(fr->cg_cm,cgs->nr);
if (fr->shift_vec == NULL)
snew(fr->shift_vec,SHIFTS);
if (fr->fshift == NULL)
snew(fr->fshift,SHIFTS);
if (bMolEpot && (fr->nmol==0)) {
fr->nmol=mols->nr;
fr->mol_nr=make_invblock(mols,natoms);
snew(fr->mol_epot,fr->nmol);
fr->nstcalc=ir->nstenergy;
}
if (fr->nbfp == NULL) {
fr->ntype = idef->atnr;
fr->bBHAM = (idef->functype[0] == F_BHAM);
fr->nbfp = mk_nbfp(idef,fr->bBHAM);
}
/* Copy the energy group exclusions */
fr->eg_excl = ir->opts.eg_excl;
/* Van der Waals stuff */
fr->rvdw = ir->rvdw;
fr->rvdw_switch = ir->rvdw_switch;
if ((fr->vdwtype != evdwCUT) && (fr->vdwtype != evdwUSER) && !fr->bBHAM) {
if (fr->rvdw_switch >= fr->rvdw)
fatal_error(0,"rvdw_switch (%g) must be < rvdw (%g)",
fr->rvdw_switch,fr->rvdw);
if (fp)
fprintf(fp,"Using %s Lennard-Jones, switch between %g and %g nm\n",
(fr->eeltype==eelSWITCH) ? "switched":"shifted",
fr->rvdw_switch,fr->rvdw);
}
if (fp)
fprintf(fp,"Cut-off's: NS: %g Coulomb: %g %s: %g\n",
fr->rlist,fr->rcoulomb,fr->bBHAM ? "BHAM":"LJ",fr->rvdw);
if (ir->eDispCorr != edispcNO)
set_avcsix(fp,fr,mdatoms);
if (fr->bBHAM)
set_bham_b_max(fp,fr,mdatoms);
/* Now update the rest of the vars */
update_forcerec(fp,fr,box);
/* if we are using LR electrostatics, and they are tabulated,
* the tables will contain shifted coulomb interactions.
* Since we want to use the non-shifted ones for 1-4
* coulombic interactions, we must have an extra set of
* tables. This should be done in tables.c, instead of this
* ugly hack, but it works for now...
*/
#define MAX_14_DIST 1.0
/* Shell to account for the maximum chargegroup radius (2*0.2 nm) *
* and diffusion during nstlist steps (0.2 nm) */
#define TAB_EXT 0.6
/* Construct tables.
* A little unnecessary to make both vdw and coul tables sometimes,
* but what the heck... */
if (fr->bcoultab || fr->bvdwtab) {
if (EEL_LR(fr->eeltype)) {
bool bcoulsave,bvdwsave;
/* generate extra tables for 1-4 interactions only
* fake the forcerec so make_tables thinks it should
* just create the non shifted version
*/
bcoulsave=fr->bcoultab;
bvdwsave=fr->bvdwtab;
fr->bcoultab=FALSE;
fr->bvdwtab=FALSE;
fr->rtab=MAX_14_DIST;
make_tables(fp,fr,MASTER(cr),tabfn);
fr->bcoultab=bcoulsave;
fr->bvdwtab=bvdwsave;
fr->coulvdw14tab=fr->coulvdwtab;
fr->coulvdwtab=NULL;
}
fr->rtab = max(fr->rlistlong+TAB_EXT,MAX_14_DIST);
}
else if (fr->efep != efepNO) {
if (fr->rlistlong < GMX_REAL_MIN) {
char *ptr,*envvar="FEP_TABLE_LENGTH";
fr->rtab = 5;
#ifdef SPEC_CPU
ptr = NULL;
#else
ptr = getenv(envvar);
#endif
if (ptr) {
sscanf(ptr,"%lf",&rtab);
fr->rtab = rtab;
}
if (fp)
fprintf(fp,"\nNote: Setting the free energy table length to %g nm\n"
" You can set this value with the environment variable %s"
"\n\n",fr->rtab,envvar);
}
else
fr->rtab = max(fr->rlistlong+TAB_EXT,MAX_14_DIST);
}
else
fr->rtab = MAX_14_DIST;
/* make tables for ordinary interactions */
make_tables(fp,fr,MASTER(cr),tabfn);
if(!(EEL_LR(fr->eeltype) && (fr->bcoultab || fr->bvdwtab)))
fr->coulvdw14tab=fr->coulvdwtab;
/* Copy the contents of the table to separate coulomb and LJ
* tables too, to improve cache performance.
*/
tabelemsize=fr->bBHAM ? 16 : 12;
snew(fr->coultab,4*(fr->ntab+1)*sizeof(real));
snew(fr->vdwtab,(tabelemsize-4)*(fr->ntab+1)*sizeof(real));
for(i=0; i<=fr->ntab; i++) {
for(j=0; j<4; j++)
fr->coultab[4*i+j]=fr->coulvdwtab[tabelemsize*i+j];
for(j=0; j<tabelemsize-4; j++)
fr->vdwtab[(tabelemsize-4)*i+j]=fr->coulvdwtab[tabelemsize*i+4+j];
}
if (!fr->mno_index)
check_solvent(fp,top,fr,mdatoms,nsb);
}
void bcast_state(const t_commrec *cr, t_state *state)
{
int i, nnht, nnhtp;
gmx_bool bAlloc;
if (!PAR(cr))
{
return;
}
/* Broadcasts the state sizes and flags from the master to all nodes
* in cr->mpi_comm_mygroup. The arrays are not broadcasted. */
block_bc(cr, state->natoms);
block_bc(cr, state->ngtc);
block_bc(cr, state->nnhpres);
block_bc(cr, state->nhchainlength);
block_bc(cr, state->flags);
if (state->lambda == NULL)
{
snew_bc(cr, state->lambda, efptNR)
}
if (cr->dd)
{
/* We allocate dynamically in dd_partition_system. */
return;
}
/* The code below is reachable only by TPI and NM, so it is not
tested by anything. */
nnht = (state->ngtc)*(state->nhchainlength);
nnhtp = (state->nnhpres)*(state->nhchainlength);
/* We still need to allocate the arrays in state for non-master
* ranks, which is done (implicitly via bAlloc) in the dirty,
* dirty nblock_abc macro. */
bAlloc = !MASTER(cr);
if (bAlloc)
{
state->nalloc = state->natoms;
}
for (i = 0; i < estNR; i++)
{
if (state->flags & (1<<i))
{
switch (i)
{
case estLAMBDA: nblock_bc(cr, efptNR, state->lambda); break;
case estFEPSTATE: block_bc(cr, state->fep_state); break;
case estBOX: block_bc(cr, state->box); break;
case estBOX_REL: block_bc(cr, state->box_rel); break;
case estBOXV: block_bc(cr, state->boxv); break;
case estPRES_PREV: block_bc(cr, state->pres_prev); break;
case estSVIR_PREV: block_bc(cr, state->svir_prev); break;
case estFVIR_PREV: block_bc(cr, state->fvir_prev); break;
case estNH_XI: nblock_abc(cr, nnht, state->nosehoover_xi); break;
case estNH_VXI: nblock_abc(cr, nnht, state->nosehoover_vxi); break;
case estNHPRES_XI: nblock_abc(cr, nnhtp, state->nhpres_xi); break;
case estNHPRES_VXI: nblock_abc(cr, nnhtp, state->nhpres_vxi); break;
case estTC_INT: nblock_abc(cr, state->ngtc, state->therm_integral); break;
case estVETA: block_bc(cr, state->veta); break;
case estVOL0: block_bc(cr, state->vol0); break;
case estX: nblock_abc(cr, state->natoms, state->x); break;
case estV: nblock_abc(cr, state->natoms, state->v); break;
case estSDX: nblock_abc(cr, state->natoms, state->sd_X); break;
case estCGP: nblock_abc(cr, state->natoms, state->cg_p); break;
case estDISRE_INITF: block_bc(cr, state->hist.disre_initf); break;
case estDISRE_RM3TAV:
block_bc(cr, state->hist.ndisrepairs);
nblock_abc(cr, state->hist.ndisrepairs, state->hist.disre_rm3tav);
break;
case estORIRE_INITF: block_bc(cr, state->hist.orire_initf); break;
case estORIRE_DTAV:
block_bc(cr, state->hist.norire_Dtav);
nblock_abc(cr, state->hist.norire_Dtav, state->hist.orire_Dtav);
break;
default:
gmx_fatal(FARGS,
"Communication is not implemented for %s in bcast_state",
est_names[i]);
}
}
}
}
gmx_constr_t init_constraints(FILE *fplog,
gmx_mtop_t *mtop,t_inputrec *ir,
gmx_edsam_t ed,t_state *state,
t_commrec *cr)
{
int ncon,nset,nmol,settle_type,i,natoms,mt,nflexcon;
struct gmx_constr *constr;
char *env;
t_ilist *ilist;
gmx_mtop_ilistloop_t iloop;
ncon = gmx_mtop_ftype_count(mtop,F_CONSTR);
nset = gmx_mtop_ftype_count(mtop,F_SETTLE);
if (ncon+nset == 0 && ir->ePull != epullCONSTRAINT && ed == NULL) {
return NULL;
}
snew(constr,1);
constr->ncon_tot = ncon;
constr->nflexcon = 0;
if (ncon > 0) {
constr->n_at2con_mt = mtop->nmoltype;
snew(constr->at2con_mt,constr->n_at2con_mt);
for(mt=0; mt<mtop->nmoltype; mt++) {
ilist = &mtop->moltype[mt].ilist[F_CONSTR];
constr->at2con_mt[mt] = make_at2con(0,mtop->moltype[mt].atoms.nr,
ilist,mtop->ffparams.iparams,
EI_DYNAMICS(ir->eI),&nflexcon);
for(i=0; i<mtop->nmolblock; i++) {
if (mtop->molblock[i].type == mt) {
constr->nflexcon += mtop->molblock[i].nmol*nflexcon;
}
}
}
if (constr->nflexcon > 0) {
if (fplog) {
fprintf(fplog,"There are %d flexible constraints\n",
constr->nflexcon);
if (ir->fc_stepsize == 0) {
fprintf(fplog,"\n"
"WARNING: step size for flexible constraining = 0\n"
" All flexible constraints will be rigid.\n"
" Will try to keep all flexible constraints at their original length,\n"
" but the lengths may exhibit some drift.\n\n");
constr->nflexcon = 0;
}
}
if (constr->nflexcon > 0)
please_cite(fplog,"Hess2002");
}
if (ir->eConstrAlg == econtLINCS) {
constr->lincsd = init_lincs(fplog,mtop,
constr->nflexcon,constr->at2con_mt,
DOMAINDECOMP(cr) && cr->dd->bInterCGcons,
ir->nLincsIter,ir->nProjOrder);
}
if (ir->eConstrAlg == econtSHAKE) {
if (DOMAINDECOMP(cr) && cr->dd->bInterCGcons)
gmx_fatal(FARGS,"SHAKE is not supported with domain decomposition and constraint that cross charge group boundaries, use LINCS");
if (constr->nflexcon)
gmx_fatal(FARGS,"For this system also velocities and/or forces need to be constrained, this can not be done with SHAKE, you should select LINCS");
please_cite(fplog,"Ryckaert77a");
if (ir->bShakeSOR)
please_cite(fplog,"Barth95a");
}
}
if (nset > 0) {
please_cite(fplog,"Miyamoto92a");
/* Check that we have only one settle type */
settle_type = -1;
iloop = gmx_mtop_ilistloop_init(mtop);
while (gmx_mtop_ilistloop_next(iloop,&ilist,&nmol)) {
for (i=0; i<ilist[F_SETTLE].nr; i+=2) {
if (settle_type == -1) {
settle_type = ilist[F_SETTLE].iatoms[i];
} else if (ilist[F_SETTLE].iatoms[i] != settle_type) {
gmx_fatal(FARGS,"More than one settle type.\n"
"Suggestion: change the least use settle constraints into 3 normal constraints.");
}
}
}
}
constr->maxwarn = 999;
env = getenv("GMX_MAXCONSTRWARN");
if (env) {
constr->maxwarn = 0;
sscanf(env,"%d",&constr->maxwarn);
if (fplog) {
fprintf(fplog,
"Setting the maximum number of constraint warnings to %d\n",
constr->maxwarn);
}
if (MASTER(cr)) {
fprintf(stderr,
"Setting the maximum number of constraint warnings to %d\n",
constr->maxwarn);
}
}
if (constr->maxwarn < 0 && fplog) {
fprintf(fplog,"maxwarn < 0, will not stop on constraint errors\n");
}
constr->warncount_lincs = 0;
constr->warncount_settle = 0;
/* Initialize the essential dynamics sampling.
* Put the pointer to the ED struct in constr */
constr->ed = ed;
if (ed != NULL) {
init_edsam(mtop,ir,cr,ed,state->x,state->box);
}
constr->warn_mtop = mtop;
return constr;
}
int main(int argc,char *argv[])
{
static char *desc[] = {
"The [TT]pmetest[tt] program tests the scaling of the PME code. When only given",
"a [TT].tpr[tt] file it will compute PME for one frame. When given a trajectory",
"it will do so for all the frames in the trajectory. Before the PME",
"routine is called the coordinates are sorted along the X-axis.[PAR]",
"As an extra service to the public the program can also compute",
"long-range Coulomb energies for components of the system. When the",
"[TT]-groups[tt] flag is given to the program the energy groups",
"from the [TT].tpr[tt] file will be read, and half an energy matrix computed."
};
t_commrec *cr,*mcr;
static t_filenm fnm[] = {
{ efTPX, NULL, NULL, ffREAD },
{ efTRN, "-o", NULL, ffWRITE },
{ efLOG, "-g", "pme", ffWRITE },
{ efTRX, "-f", NULL, ffOPTRD },
{ efXVG, "-x", "ener-pme", ffWRITE }
};
#define NFILE asize(fnm)
/* Command line options ! */
static gmx_bool bVerbose=FALSE;
static gmx_bool bOptFFT=FALSE;
static gmx_bool bSort=FALSE;
static int ewald_geometry=eewg3D;
static int nnodes=1;
static int pme_order=0;
static rvec grid = { -1, -1, -1 };
static real rc = 0.0;
static real dtol = 0.0;
static gmx_bool bGroups = FALSE;
static t_pargs pa[] = {
{ "-np", FALSE, etINT, {&nnodes},
"Number of nodes, must be the same as used for [TT]grompp[tt]" },
{ "-v", FALSE, etBOOL,{&bVerbose},
"Be loud and noisy" },
{ "-sort", FALSE, etBOOL,{&bSort},
"Sort coordinates. Crucial for domain decomposition." },
{ "-grid", FALSE, etRVEC,{&grid},
"Number of grid cells in X, Y, Z dimension (if -1 use from [TT].tpr[tt])" },
{ "-order", FALSE, etINT, {&pme_order},
"Order of the PME spreading algorithm" },
{ "-groups", FALSE, etBOOL, {&bGroups},
"Compute half an energy matrix based on the energy groups in your [TT].tpr[tt] file" },
{ "-rc", FALSE, etREAL, {&rc},
"Rcoulomb for Ewald summation" },
{ "-tol", FALSE, etREAL, {&dtol},
"Tolerance for Ewald summation" }
};
FILE *fp;
t_inputrec *ir;
t_topology top;
t_tpxheader tpx;
t_nrnb nrnb;
t_nsborder *nsb;
t_forcerec *fr;
t_mdatoms *mdatoms;
char title[STRLEN];
int natoms,step,status,i,ncg,root;
real t,lambda,ewaldcoeff,qtot;
rvec *x,*f,*xbuf;
int *index;
gmx_bool bCont;
real *charge,*qbuf,*qqbuf;
matrix box;
/* Start the actual parallel code if necessary */
cr = init_par(&argc,&argv);
root = 0;
if (MASTER(cr))
CopyRight(stderr,argv[0]);
/* Parse command line on all processors, arguments are passed on in
* init_par (see above)
*/
parse_common_args(&argc,argv,
PCA_KEEP_ARGS | PCA_NOEXIT_ON_ARGS | PCA_BE_NICE |
PCA_CAN_SET_DEFFNM | (MASTER(cr) ? 0 : PCA_QUIET),
NFILE,fnm,asize(pa),pa,asize(desc),desc,0,NULL);
#ifndef GMX_MPI
if (nnodes > 1)
gmx_fatal(FARGS,"GROMACS compiled without MPI support - can't do parallel runs");
#endif
/* Open log files on all processors */
open_log(ftp2fn(efLOG,NFILE,fnm),cr);
snew(ir,1);
if (MASTER(cr)) {
/* Read tpr file etc. */
read_tpxheader(ftp2fn(efTPX,NFILE,fnm),&tpx,FALSE,NULL,NULL);
snew(x,tpx.natoms);
read_tpx(ftp2fn(efTPX,NFILE,fnm),&step,&t,&lambda,ir,
box,&natoms,x,NULL,NULL,&top);
/* Charges */
qtot = 0;
snew(charge,natoms);
for(i=0; (i<natoms); i++) {
charge[i] = top.atoms.atom[i].q;
qtot += charge[i];
}
/* Grid stuff */
if (opt2parg_bSet("-grid",asize(pa),pa)) {
ir->nkx = grid[XX];
ir->nky = grid[YY];
ir->nkz = grid[ZZ];
}
/* Check command line parameters for consistency */
if ((ir->nkx <= 0) || (ir->nky <= 0) || (ir->nkz <= 0))
gmx_fatal(FARGS,"PME grid = %d %d %d",ir->nkx,ir->nky,ir->nkz);
if (opt2parg_bSet("-rc",asize(pa),pa))
ir->rcoulomb = rc;
if (ir->rcoulomb <= 0)
gmx_fatal(FARGS,"rcoulomb should be > 0 (not %f)",ir->rcoulomb);
if (opt2parg_bSet("-order",asize(pa),pa))
ir->pme_order = pme_order;
if (ir->pme_order <= 0)
gmx_fatal(FARGS,"pme_order should be > 0 (not %d)",ir->pme_order);
if (opt2parg_bSet("-tol",asize(pa),pa))
ir->ewald_rtol = dtol;
if (ir->ewald_rtol <= 0)
gmx_fatal(FARGS,"ewald_tol should be > 0 (not %f)",ir->ewald_rtol);
}
else {
init_top(&top);
}
/* Add parallellization code here */
snew(nsb,1);
if (MASTER(cr)) {
ncg = top.blocks[ebCGS].multinr[0];
for(i=0; (i<cr->nnodes-1); i++)
top.blocks[ebCGS].multinr[i] = min(ncg,(ncg*(i+1))/cr->nnodes);
for( ; (i<MAXNODES); i++)
top.blocks[ebCGS].multinr[i] = ncg;
}
if (PAR(cr)) {
/* Set some variables to zero to avoid core dumps */
ir->opts.ngtc = ir->opts.ngacc = ir->opts.ngfrz = ir->opts.ngener = 0;
#ifdef GMX_MPI
/* Distribute the data over processors */
MPI_Bcast(&natoms,1,MPI_INT,root,MPI_COMM_WORLD);
MPI_Bcast(ir,sizeof(*ir),MPI_BYTE,root,MPI_COMM_WORLD);
MPI_Bcast(&qtot,1,GMX_MPI_REAL,root,MPI_COMM_WORLD);
#endif
/* Call some dedicated communication routines, master sends n-1 times */
if (MASTER(cr)) {
for(i=1; (i<cr->nnodes); i++) {
mv_block(i,&(top.blocks[ebCGS]));
mv_block(i,&(top.atoms.excl));
}
}
else {
ld_block(root,&(top.blocks[ebCGS]));
ld_block(root,&(top.atoms.excl));
}
if (!MASTER(cr)) {
snew(charge,natoms);
snew(x,natoms);
}
#ifdef GMX_MPI
MPI_Bcast(charge,natoms,GMX_MPI_REAL,root,MPI_COMM_WORLD);
#endif
}
ewaldcoeff = calc_ewaldcoeff(ir->rcoulomb,ir->ewald_rtol);
if (bVerbose)
pr_inputrec(stdlog,0,"Inputrec",ir);
/* Allocate memory for temp arrays etc. */
snew(xbuf,natoms);
snew(f,natoms);
snew(qbuf,natoms);
snew(qqbuf,natoms);
snew(index,natoms);
/* Initialize the PME code */
init_pme(stdlog,cr,ir->nkx,ir->nky,ir->nkz,ir->pme_order,
natoms,FALSE,bOptFFT,ewald_geometry);
/* MFlops accounting */
init_nrnb(&nrnb);
/* Initialize the work division */
calc_nsb(stdlog,&(top.blocks[ebCGS]),cr->nnodes,nsb,0);
nsb->nodeid = cr->nodeid;
print_nsb(stdlog,"pmetest",nsb);
/* Initiate forcerec */
mdatoms = atoms2md(stdlog,&top.atoms,ir->opts.nFreeze,ir->eI,
ir->delta_t,0,ir->opts.tau_t,FALSE,FALSE);
snew(fr,1);
init_forcerec(stdlog,fr,ir,&top,cr,mdatoms,nsb,box,FALSE,NULL,NULL,FALSE);
/* First do PME based on coordinates in tpr file, send them to
* other processors if needed.
*/
if (MASTER(cr))
fprintf(stdlog,"-----\n"
"Results based on tpr file %s\n",ftp2fn(efTPX,NFILE,fnm));
#ifdef GMX_MPI
if (PAR(cr)) {
MPI_Bcast(x[0],natoms*DIM,GMX_MPI_REAL,root,MPI_COMM_WORLD);
MPI_Bcast(box[0],DIM*DIM,GMX_MPI_REAL,root,MPI_COMM_WORLD);
MPI_Bcast(&t,1,GMX_MPI_REAL,root,MPI_COMM_WORLD);
}
#endif
do_my_pme(stdlog,0,bVerbose,ir,x,xbuf,f,charge,qbuf,qqbuf,box,bSort,
cr,nsb,&nrnb,&(top.atoms.excl),qtot,fr,index,NULL,
bGroups ? ir->opts.ngener : 1,mdatoms->cENER);
/* If we have a trajectry file, we will read the frames in it and compute
* the PME energy.
*/
if (ftp2bSet(efTRX,NFILE,fnm)) {
fprintf(stdlog,"-----\n"
"Results based on trx file %s\n",ftp2fn(efTRX,NFILE,fnm));
if (MASTER(cr)) {
sfree(x);
natoms = read_first_x(&status,ftp2fn(efTRX,NFILE,fnm),&t,&x,box);
if (natoms != top.atoms.nr)
gmx_fatal(FARGS,"natoms in trx = %d, in tpr = %d",natoms,top.atoms.nr);
fp = xvgropen(ftp2fn(efXVG,NFILE,fnm),"PME Energy","Time (ps)","E (kJ/mol)");
}
else
fp = NULL;
do {
/* Send coordinates, box and time to the other nodes */
#ifdef GMX_MPI
if (PAR(cr)) {
MPI_Bcast(x[0],natoms*DIM,GMX_MPI_REAL,root,MPI_COMM_WORLD);
MPI_Bcast(box[0],DIM*DIM,GMX_MPI_REAL,root,MPI_COMM_WORLD);
MPI_Bcast(&t,1,GMX_MPI_REAL,root,MPI_COMM_WORLD);
}
#endif
rm_pbc(&top.idef,nsb->natoms,box,x,x);
/* Call the PME wrapper function */
do_my_pme(stdlog,t,bVerbose,ir,x,xbuf,f,charge,qbuf,qqbuf,box,bSort,cr,
nsb,&nrnb,&(top.atoms.excl),qtot,fr,index,fp,
bGroups ? ir->opts.ngener : 1,mdatoms->cENER);
/* Only the master processor reads more data */
if (MASTER(cr))
bCont = read_next_x(status,&t,natoms,x,box);
/* Check whether we need to continue */
#ifdef GMX_MPI
if (PAR(cr))
MPI_Bcast(&bCont,1,MPI_INT,root,MPI_COMM_WORLD);
#endif
} while (bCont);
/* Finish I/O, close files */
if (MASTER(cr)) {
close_trx(status);
ffclose(fp);
}
}
if (bVerbose) {
/* Do some final I/O about performance, might be useful in debugging */
fprintf(stdlog,"-----\n");
print_nrnb(stdlog,&nrnb);
}
/* Finish the parallel stuff */
if (gmx_parallel_env_initialized())
gmx_finalize(cr);
/* Thank the audience, as usual */
if (MASTER(cr))
gmx_thanx(stderr);
return 0;
}
int main(int argc,char *argv[])
{
int mmm[] = { 8, 10, 12, 15, 16, 18, 20, 24, 25, 27, 30, 32, 36, 40,
45, 48, 50, 54, 60, 64, 72, 75, 80, 81, 90, 100 };
int nnn[] = { 24, 32, 48, 60, 72, 84, 96 };
#define NNN asize(nnn)
FILE *fp,*fplog;
int *niter;
int i,j,n,nit,ntot,n3,rsize;
double t,nflop,start;
double *rt,*ct;
t_fftgrid *g;
t_commrec *cr;
static gmx_bool bReproducible = FALSE;
static int nnode = 1;
static int nitfac = 1;
t_pargs pa[] = {
{ "-reproducible", FALSE, etBOOL, {&bReproducible},
"Request binary reproducible results" },
{ "-np", FALSE, etINT, {&nnode},
"Number of NODEs" },
{ "-itfac", FALSE, etINT, {&nitfac},
"Multiply number of iterations by this" }
};
static t_filenm fnm[] = {
{ efLOG, "-g", "fft", ffWRITE },
{ efXVG, "-o", "fft", ffWRITE }
};
#define NFILE asize(fnm)
cr = init_par(&argc,&argv);
if (MASTER(cr))
CopyRight(stdout,argv[0]);
parse_common_args(&argc,argv,
PCA_CAN_SET_DEFFNM | (MASTER(cr) ? 0 : PCA_QUIET),
NFILE,fnm,asize(pa),pa,0,NULL,0,NULL);
gmx_log_open(ftp2fn(efLOG,NFILE,fnm),cr,1,0,&fplog);
snew(niter,NNN);
snew(ct,NNN);
snew(rt,NNN);
rsize = sizeof(real);
for(i=0; (i<NNN); i++) {
n = nnn[i];
if (n < 16)
niter[i] = 50;
else if (n < 26)
niter[i] = 20;
else if (n < 51)
niter[i] = 10;
else
niter[i] = 5;
niter[i] *= nitfac;
nit = niter[i];
if (MASTER(cr))
fprintf(stderr,"\r3D FFT (%s precision) %3d^3, niter %3d ",
(rsize == 8) ? "Double" : "Single",n,nit);
g = mk_fftgrid(n,n,n,NULL,NULL,cr,bReproducible);
if (PAR(cr))
start = time(NULL);
else
start_time();
for(j=0; (j<nit); j++) {
gmxfft3D(g,GMX_FFT_REAL_TO_COMPLEX,cr);
gmxfft3D(g,GMX_FFT_COMPLEX_TO_REAL,cr);
}
if (PAR(cr))
rt[i] = time(NULL)-start;
else {
update_time();
rt[i] = node_time();
}
done_fftgrid(g);
sfree(g);
}
if (MASTER(cr)) {
fprintf(stderr,"\n");
fp=xvgropen(ftp2fn(efXVG,NFILE,fnm),
"FFT timings","n^3","t (s)");
for(i=0; (i<NNN); i++) {
n3 = 2*niter[i]*nnn[i]*nnn[i]*nnn[i];
fprintf(fp,"%10d %10g\n",nnn[i],rt[i]/(2*niter[i]));
}
gmx_fio_fclose(fp);
}
return 0;
}
void gmx_log_open(const char *lognm, const t_commrec *cr, gmx_bool bMasterOnly,
gmx_bool bAppendFiles, FILE** fplog)
{
int len, testlen, pid;
char buf[256], host[256];
time_t t;
char timebuf[STRLEN];
FILE *fp = *fplog;
char *tmpnm;
debug_gmx();
/* Communicate the filename for logfile */
if (cr->nnodes > 1 && !bMasterOnly
#ifdef GMX_THREAD_MPI
/* With thread MPI the non-master log files are opened later
* when the files names are already known on all nodes.
*/
&& FALSE
#endif
)
{
if (MASTER(cr))
{
len = strlen(lognm) + 1;
}
gmx_bcast(sizeof(len), &len, cr);
if (!MASTER(cr))
{
snew(tmpnm, len+8);
}
else
{
tmpnm = gmx_strdup(lognm);
}
gmx_bcast(len*sizeof(*tmpnm), tmpnm, cr);
}
else
{
tmpnm = gmx_strdup(lognm);
}
debug_gmx();
if (!bMasterOnly && !MASTER(cr))
{
/* Since log always ends with '.log' let's use this info */
par_fn(tmpnm, efLOG, cr, FALSE, !bMasterOnly, buf, 255);
fp = gmx_fio_fopen(buf, bAppendFiles ? "a+" : "w+" );
}
else if (!bAppendFiles)
{
fp = gmx_fio_fopen(tmpnm, bAppendFiles ? "a+" : "w+" );
}
sfree(tmpnm);
gmx_fatal_set_log_file(fp);
/* Get some machine parameters */
gmx_gethostname(host, 256);
time(&t);
#ifndef NO_GETPID
# ifdef GMX_NATIVE_WINDOWS
pid = _getpid();
# else
pid = getpid();
# endif
#else
pid = 0;
#endif
if (bAppendFiles)
{
fprintf(fp,
"\n"
"\n"
"-----------------------------------------------------------\n"
"Restarting from checkpoint, appending to previous log file.\n"
"\n"
);
}
gmx_ctime_r(&t, timebuf, STRLEN);
fprintf(fp,
"Log file opened on %s"
"Host: %s pid: %d nodeid: %d nnodes: %d\n",
timebuf, host, pid, cr->nodeid, cr->nnodes);
gmx_print_version_info(fp);
fprintf(fp, "\n\n");
fflush(fp);
debug_gmx();
*fplog = fp;
}
int main(int argc,char *argv[])
{
static char *desc[] = {
"The ffscan program performs a single point energy and force calculation",
"in which the force field is modified. This way a range of parameters can",
"be changed and tested for reproduction of e.g. quantum chemical or",
"experimental data. A grid scan over the parameters is done as specified",
"using command line arguments. All parameters that reproduce the energy",
"within a given absolute tolerance are printed to a log file.[PAR]",
"Obviously polarizable models can be used, and shell optimisation is",
"performed if necessary. Also, like in [TT]mdrun[tt] table functions can be used",
"for user defined potential functions.[PAR]",
"If the option -ga with appropriate file is passed, a genetic algorithm will",
"be used rather than a grid scan."
};
t_commrec *cr;
static t_filenm fnm[] = {
{ efTPX, NULL, NULL, ffREAD },
{ efLOG, "-g", "md", ffWRITE },
{ efXVG, "-table", "table", ffOPTRD },
{ efDAT, "-parm", "params", ffREAD },
{ efDAT, "-ga", "genalg", ffOPTRD },
{ efGRO, "-c", "junk", ffWRITE },
{ efEDR, "-e", "junk", ffWRITE },
{ efTRN, "-o", "junk", ffWRITE }
};
#define NFILE asize(fnm)
/* Command line options ! */
static t_ffscan ff = {
/* tol */ 0.1,
/* f_max */ 100.0,
/* npow */ 12.0,
/* epot */ 0.0,
/* fac_epot */ 1.0,
/* fac_pres */ 0.1,
/* fac_msf */ 0.1,
/* pres */ 1.0,
/* molsize */ 1,
/* nmol */ 1,
/* bComb */ TRUE,
/* bVerbose */ FALSE,
/* bLogEps */ FALSE
};
static char *loadx=NULL,*loady=NULL,*loadz=NULL;
static t_pargs pa[] = {
{ "-tol", FALSE, etREAL, {&ff.tol}, "Energy tolerance (kJ/mol) (zero means everything is printed)" },
{ "-fmax", FALSE, etREAL, {&ff.f_max}, "Force tolerance (zero means everything is printed)" },
{ "-comb", FALSE, etBOOL, {&ff.bComb}, "Use combination rules" },
{ "-npow", FALSE, etREAL, {&ff.npow}, "Power for LJ in case of table use" },
{ "-logeps",FALSE, etBOOL, {&ff.bLogEps}, "Use a logarithmic scale for epsilon" },
{ "-v", FALSE, etBOOL, {&ff.bVerbose}, "Be loud and noisy" },
{ "-epot", FALSE, etREAL, {&ff.epot}, "Target energy (kJ/mol)" },
{ "-fepot", FALSE, etREAL, {&ff.fac_epot}, "Factor for scaling energy violations (0 turns energy contribution off)" },
{ "-pres", FALSE, etREAL, {&ff.pres}, "Value for reference pressure" },
{ "-fpres", FALSE, etREAL, {&ff.fac_pres}, "Factor for scaling pressure violations (0 turns pressure contribution off)" },
{ "-fmsf", FALSE, etREAL, {&ff.fac_msf}, "Factor for scaling mean square force violations (0 turns MSF contribution off)" },
{ "-molsize",FALSE,etINT, {&ff.molsize}, "Number of atoms per molecule" },
{ "-nmol", FALSE, etINT, {&ff.nmol}, "Number of molecules (Epot is divided by this value!)" }
};
#define NPA asize(pa)
unsigned long Flags = 0;
gmx_edsam_t ed=NULL;
FILE *fplog;
ivec ddxyz = { 1,1,1 };
cr = init_par(&argc,&argv);
ff.bVerbose = ff.bVerbose && MASTER(cr);
#if 0
snew(ed,1);
ed->eEDtype=eEDnone;
#endif
if (MASTER(cr))
CopyRight(stderr,argv[0]);
parse_common_args(&argc,argv,PCA_BE_NICE,NFILE,fnm,
NPA,pa,asize(desc),desc,0,NULL);
if (ff.npow <= 6.0)
gmx_fatal(FARGS,"Can not have repulsion with smaller exponent than 6");
if (ff.nmol < 1)
gmx_fatal(FARGS,"Can not fit %d molecules",ff.nmol);
gmx_log_open(ftp2fn(efLOG,NFILE,fnm),cr,FALSE,0,&fplog);
if (MASTER(cr)) {
CopyRight(fplog,argv[0]);
please_cite(fplog,"Lindahl2001a");
please_cite(fplog,"Berendsen95a");
}
set_ffvars(&ff);
Flags = (Flags | MD_FFSCAN);
mdrunner(fplog,cr,NFILE,fnm,ff.bVerbose,FALSE,
ddxyz,0,0,0,loadx,loady,loadz,1,
0,0,Flags);
if (gmx_parallel_env_initialized())
gmx_finalize(cr);
gmx_log_close(fplog);
return 0;
}
POOL_STATUS AsciiRow(POOL_CONNECTION *frontend,
POOL_CONNECTION_POOL *backend,
short num_fields)
{
static char nullmap[8192], nullmap1[8192];
int nbytes;
int i, j;
unsigned char mask;
int size, size1 = 0;
char *buf = NULL, *sendbuf = NULL;
char msgbuf[1024];
pool_write(frontend, "D", 1);
nbytes = (num_fields + 7)/8;
if (nbytes <= 0)
return POOL_CONTINUE;
/* NULL map */
pool_read(MASTER(backend), nullmap, nbytes);
memcpy(nullmap1, nullmap, nbytes);
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i) && !IS_MASTER_NODE_ID(i))
{
pool_read(CONNECTION(backend, i), nullmap, nbytes);
if (memcmp(nullmap, nullmap1, nbytes))
{
/* XXX: NULLMAP maybe different among
backends. If we were a paranoid, we have to treat
this as a fatal error. However in the real world
we'd better to adapt this situation. Just throw a
log... */
pool_debug("AsciiRow: NULLMAP differ between master and %d th backend", i);
}
}
}
if (pool_write(frontend, nullmap1, nbytes) < 0)
return POOL_END;
mask = 0;
for (i = 0;i<num_fields;i++)
{
if (mask == 0)
mask = 0x80;
/* NOT NULL? */
if (mask & nullmap[i/8])
{
/* field size */
if (pool_read(MASTER(backend), &size, sizeof(int)) < 0)
return POOL_END;
size1 = ntohl(size) - 4;
/* read and send actual data only when size > 0 */
if (size1 > 0)
{
sendbuf = pool_read2(MASTER(backend), size1);
if (sendbuf == NULL)
return POOL_END;
}
/* forward to frontend */
pool_write(frontend, &size, sizeof(int));
pool_write(frontend, sendbuf, size1);
snprintf(msgbuf, Min(sizeof(msgbuf), size1+1), "%s", sendbuf);
pool_debug("AsciiRow: len: %d data: %s", size1, msgbuf);
for (j=0;j<NUM_BACKENDS;j++)
{
if (VALID_BACKEND(j) && !IS_MASTER_NODE_ID(j))
{
/* field size */
if (pool_read(CONNECTION(backend, j), &size, sizeof(int)) < 0)
return POOL_END;
buf = NULL;
size = ntohl(size) - 4;
/* XXX: field size maybe different among
backends. If we were a paranoid, we have to treat
this as a fatal error. However in the real world
we'd better to adapt this situation. Just throw a
log... */
if (size != size1)
pool_debug("AsciiRow: %d th field size does not match between master(%d) and %d th backend(%d)",
i, ntohl(size), j, ntohl(size1));
/* read and send actual data only when size > 0 */
if (size > 0)
{
buf = pool_read2(CONNECTION(backend, j), size);
if (buf == NULL)
return POOL_END;
}
}
}
}
mask >>= 1;
}
if (pool_flush(frontend))
return POOL_END;
return POOL_CONTINUE;
}
int gmx_pme_error(int argc, char *argv[])
{
const char *desc[] = {
"[THISMODULE] estimates the error of the electrostatic forces",
"if using the sPME algorithm. The flag [TT]-tune[tt] will determine",
"the splitting parameter such that the error is equally",
"distributed over the real and reciprocal space part.",
"The part of the error that stems from self interaction of the particles "
"is computationally demanding. However, a good a approximation is to",
"just use a fraction of the particles for this term which can be",
"indicated by the flag [TT]-self[tt].[PAR]",
};
real fs = 0.0; /* 0 indicates: not set by the user */
real user_beta = -1.0;
real fracself = 1.0;
t_inputinfo info;
t_state state; /* The state from the tpr input file */
gmx_mtop_t mtop; /* The topology from the tpr input file */
t_inputrec *ir = NULL; /* The inputrec from the tpr file */
FILE *fp = NULL;
t_commrec *cr;
unsigned long PCA_Flags;
gmx_bool bTUNE = FALSE;
gmx_bool bVerbose = FALSE;
int seed = 0;
static t_filenm fnm[] = {
{ efTPR, "-s", NULL, ffREAD },
{ efOUT, "-o", "error", ffWRITE },
{ efTPR, "-so", "tuned", ffOPTWR }
};
output_env_t oenv = NULL;
t_pargs pa[] = {
{ "-beta", FALSE, etREAL, {&user_beta},
"If positive, overwrite ewald_beta from [TT].tpr[tt] file with this value" },
{ "-tune", FALSE, etBOOL, {&bTUNE},
"Tune the splitting parameter such that the error is equally distributed between real and reciprocal space" },
{ "-self", FALSE, etREAL, {&fracself},
"If between 0.0 and 1.0, determine self interaction error from just this fraction of the charged particles" },
{ "-seed", FALSE, etINT, {&seed},
"Random number seed used for Monte Carlo algorithm when [TT]-self[tt] is set to a value between 0.0 and 1.0" },
{ "-v", FALSE, etBOOL, {&bVerbose},
"Be loud and noisy" }
};
#define NFILE asize(fnm)
cr = init_commrec();
PCA_Flags = PCA_NOEXIT_ON_ARGS;
if (!parse_common_args(&argc, argv, PCA_Flags,
NFILE, fnm, asize(pa), pa, asize(desc), desc,
0, NULL, &oenv))
{
return 0;
}
if (!bTUNE)
{
bTUNE = opt2bSet("-so", NFILE, fnm);
}
info.n_entries = 1;
/* Allocate memory for the inputinfo struct: */
create_info(&info);
info.fourier_sp[0] = fs;
/* Read in the tpr file and open logfile for reading */
if (MASTER(cr))
{
snew(ir, 1);
read_tpr_file(opt2fn("-s", NFILE, fnm), &info, &state, &mtop, ir, user_beta, fracself);
fp = fopen(opt2fn("-o", NFILE, fnm), "w");
}
/* Check consistency if the user provided fourierspacing */
if (fs > 0 && MASTER(cr))
{
/* Recalculate the grid dimensions using fourierspacing from user input */
info.nkx[0] = 0;
info.nky[0] = 0;
info.nkz[0] = 0;
calc_grid(stdout, state.box, info.fourier_sp[0], &(info.nkx[0]), &(info.nky[0]), &(info.nkz[0]));
if ( (ir->nkx != info.nkx[0]) || (ir->nky != info.nky[0]) || (ir->nkz != info.nkz[0]) )
{
gmx_fatal(FARGS, "Wrong fourierspacing %f nm, input file grid = %d x %d x %d, computed grid = %d x %d x %d",
fs, ir->nkx, ir->nky, ir->nkz, info.nkx[0], info.nky[0], info.nkz[0]);
}
}
/* Estimate (S)PME force error */
/* Determine the volume of the simulation box */
if (MASTER(cr))
{
info.volume = det(state.box);
calc_recipbox(state.box, info.recipbox);
info.natoms = mtop.natoms;
info.bTUNE = bTUNE;
}
if (PAR(cr))
{
bcast_info(&info, cr);
}
/* Get an error estimate of the input tpr file and do some tuning if requested */
estimate_PME_error(&info, &state, &mtop, fp, bVerbose, seed, cr);
if (MASTER(cr))
{
/* Write out optimized tpr file if requested */
if (opt2bSet("-so", NFILE, fnm) || bTUNE)
{
ir->ewald_rtol = info.ewald_rtol[0];
write_tpx_state(opt2fn("-so", NFILE, fnm), ir, &state, &mtop);
}
please_cite(fp, "Wang2010");
fclose(fp);
}
return 0;
}
POOL_STATUS FunctionResultResponse(POOL_CONNECTION *frontend,
POOL_CONNECTION_POOL *backend)
{
char dummy;
int len;
char *result = 0;
int i;
pool_write(frontend, "V", 1);
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i))
{
if (pool_read(CONNECTION(backend, i), &dummy, 1) < 0)
return POOL_ERROR;
}
}
pool_write(frontend, &dummy, 1);
/* non empty result? */
if (dummy == 'G')
{
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i))
{
/* length of result in bytes */
if (pool_read(CONNECTION(backend, i), &len, sizeof(len)) < 0)
return POOL_ERROR;
}
}
pool_write(frontend, &len, sizeof(len));
len = ntohl(len);
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i))
{
/* result value itself */
if ((result = pool_read2(MASTER(backend), len)) == NULL)
return POOL_ERROR;
}
}
pool_write(frontend, result, len);
}
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i))
{
/* unused ('0') */
if (pool_read(MASTER(backend), &dummy, 1) < 0)
return POOL_ERROR;
}
}
pool_write(frontend, "0", 1);
return pool_flush(frontend);
}
/* Estimate the reciprocal space part error of the SPME Ewald sum. */
static real estimate_reciprocal(
t_inputinfo *info,
rvec x[], /* array of particles */
real q[], /* array of charges */
int nr, /* number of charges = size of the charge array */
FILE gmx_unused *fp_out,
gmx_bool bVerbose,
unsigned int seed, /* The seed for the random number generator */
int *nsamples, /* Return the number of samples used if Monte Carlo
* algorithm is used for self energy error estimate */
t_commrec *cr)
{
real e_rec = 0; /* reciprocal error estimate */
real e_rec1 = 0; /* Error estimate term 1*/
real e_rec2 = 0; /* Error estimate term 2*/
real e_rec3 = 0; /* Error estimate term 3 */
real e_rec3x = 0; /* part of Error estimate term 3 in x */
real e_rec3y = 0; /* part of Error estimate term 3 in y */
real e_rec3z = 0; /* part of Error estimate term 3 in z */
int i, ci;
int nx, ny, nz; /* grid coordinates */
real q2_all = 0; /* sum of squared charges */
rvec gridpx; /* reciprocal grid point in x direction*/
rvec gridpxy; /* reciprocal grid point in x and y direction*/
rvec gridp; /* complete reciprocal grid point in 3 directions*/
rvec tmpvec; /* template to create points from basis vectors */
rvec tmpvec2; /* template to create points from basis vectors */
real coeff = 0; /* variable to compute coefficients of the error estimate */
real coeff2 = 0; /* variable to compute coefficients of the error estimate */
real tmp = 0; /* variables to compute different factors from vectors */
real tmp1 = 0;
real tmp2 = 0;
gmx_bool bFraction;
/* Random number generator */
gmx_rng_t rng = NULL;
int *numbers = NULL;
/* Index variables for parallel work distribution */
int startglobal, stopglobal;
int startlocal, stoplocal;
int x_per_core;
int xtot;
#ifdef TAKETIME
double t0 = 0.0;
double t1 = 0.0;
#endif
rng = gmx_rng_init(seed);
clear_rvec(gridpx);
clear_rvec(gridpxy);
clear_rvec(gridp);
clear_rvec(tmpvec);
clear_rvec(tmpvec2);
for (i = 0; i < nr; i++)
{
q2_all += q[i]*q[i];
}
/* Calculate indices for work distribution */
startglobal = -info->nkx[0]/2;
stopglobal = info->nkx[0]/2;
xtot = stopglobal*2+1;
if (PAR(cr))
{
x_per_core = static_cast<int>(ceil(static_cast<real>(xtot) / cr->nnodes));
startlocal = startglobal + x_per_core*cr->nodeid;
stoplocal = startlocal + x_per_core -1;
if (stoplocal > stopglobal)
{
stoplocal = stopglobal;
}
}
else
{
startlocal = startglobal;
stoplocal = stopglobal;
x_per_core = xtot;
}
/*
#ifdef GMX_LIB_MPI
MPI_Barrier(MPI_COMM_WORLD);
#endif
*/
#ifdef GMX_LIB_MPI
#ifdef TAKETIME
if (MASTER(cr))
{
t0 = MPI_Wtime();
}
#endif
#endif
if (MASTER(cr))
{
fprintf(stderr, "Calculating reciprocal error part 1 ...");
}
for (nx = startlocal; nx <= stoplocal; nx++)
{
svmul(nx, info->recipbox[XX], gridpx);
for (ny = -info->nky[0]/2; ny < info->nky[0]/2+1; ny++)
{
svmul(ny, info->recipbox[YY], tmpvec);
rvec_add(gridpx, tmpvec, gridpxy);
for (nz = -info->nkz[0]/2; nz < info->nkz[0]/2+1; nz++)
{
if (0 == nx && 0 == ny && 0 == nz)
{
continue;
}
svmul(nz, info->recipbox[ZZ], tmpvec);
rvec_add(gridpxy, tmpvec, gridp);
tmp = norm2(gridp);
coeff = exp(-1.0 * M_PI * M_PI * tmp / info->ewald_beta[0] / info->ewald_beta[0] );
coeff /= 2.0 * M_PI * info->volume * tmp;
coeff2 = tmp;
tmp = eps_poly2(nx, info->nkx[0], info->pme_order[0]);
tmp += eps_poly2(ny, info->nkx[0], info->pme_order[0]);
tmp += eps_poly2(nz, info->nkx[0], info->pme_order[0]);
tmp1 = eps_poly1(nx, info->nkx[0], info->pme_order[0]);
tmp2 = eps_poly1(ny, info->nky[0], info->pme_order[0]);
tmp += 2.0 * tmp1 * tmp2;
tmp1 = eps_poly1(nz, info->nkz[0], info->pme_order[0]);
tmp2 = eps_poly1(ny, info->nky[0], info->pme_order[0]);
tmp += 2.0 * tmp1 * tmp2;
tmp1 = eps_poly1(nz, info->nkz[0], info->pme_order[0]);
tmp2 = eps_poly1(nx, info->nkx[0], info->pme_order[0]);
tmp += 2.0 * tmp1 * tmp2;
tmp1 = eps_poly1(nx, info->nkx[0], info->pme_order[0]);
tmp1 += eps_poly1(ny, info->nky[0], info->pme_order[0]);
tmp1 += eps_poly1(nz, info->nkz[0], info->pme_order[0]);
tmp += tmp1 * tmp1;
e_rec1 += 32.0 * M_PI * M_PI * coeff * coeff * coeff2 * tmp * q2_all * q2_all / nr;
tmp1 = eps_poly3(nx, info->nkx[0], info->pme_order[0]);
tmp1 *= info->nkx[0];
tmp2 = iprod(gridp, info->recipbox[XX]);
tmp = tmp1*tmp2;
tmp1 = eps_poly3(ny, info->nky[0], info->pme_order[0]);
tmp1 *= info->nky[0];
tmp2 = iprod(gridp, info->recipbox[YY]);
tmp += tmp1*tmp2;
tmp1 = eps_poly3(nz, info->nkz[0], info->pme_order[0]);
tmp1 *= info->nkz[0];
tmp2 = iprod(gridp, info->recipbox[ZZ]);
tmp += tmp1*tmp2;
tmp *= 4.0 * M_PI;
tmp1 = eps_poly4(nx, info->nkx[0], info->pme_order[0]);
tmp1 *= norm2(info->recipbox[XX]);
tmp1 *= info->nkx[0] * info->nkx[0];
tmp += tmp1;
tmp1 = eps_poly4(ny, info->nky[0], info->pme_order[0]);
tmp1 *= norm2(info->recipbox[YY]);
tmp1 *= info->nky[0] * info->nky[0];
tmp += tmp1;
tmp1 = eps_poly4(nz, info->nkz[0], info->pme_order[0]);
tmp1 *= norm2(info->recipbox[ZZ]);
tmp1 *= info->nkz[0] * info->nkz[0];
tmp += tmp1;
e_rec2 += 4.0 * coeff * coeff * tmp * q2_all * q2_all / nr;
}
}
if (MASTER(cr))
{
fprintf(stderr, "\rCalculating reciprocal error part 1 ... %3.0f%%", 100.0*(nx-startlocal+1)/(x_per_core));
}
}
if (MASTER(cr))
{
fprintf(stderr, "\n");
}
/* Use just a fraction of all charges to estimate the self energy error term? */
bFraction = (info->fracself > 0.0) && (info->fracself < 1.0);
if (bFraction)
{
/* Here xtot is the number of samples taken for the Monte Carlo calculation
* of the average of term IV of equation 35 in Wang2010. Round up to a
* number of samples that is divisible by the number of nodes */
x_per_core = static_cast<int>(ceil(info->fracself * nr / cr->nnodes));
xtot = x_per_core * cr->nnodes;
}
else
{
/* In this case we use all nr particle positions */
xtot = nr;
x_per_core = static_cast<int>(ceil(static_cast<real>(xtot) / cr->nnodes));
}
startlocal = x_per_core * cr->nodeid;
stoplocal = std::min(startlocal + x_per_core, xtot); /* min needed if xtot == nr */
if (bFraction)
{
/* Make shure we get identical results in serial and parallel. Therefore,
* take the sample indices from a single, global random number array that
* is constructed on the master node and that only depends on the seed */
snew(numbers, xtot);
if (MASTER(cr))
{
for (i = 0; i < xtot; i++)
{
numbers[i] = static_cast<int>(floor(gmx_rng_uniform_real(rng) * nr));
}
}
/* Broadcast the random number array to the other nodes */
if (PAR(cr))
{
nblock_bc(cr, xtot, numbers);
}
if (bVerbose && MASTER(cr))
{
fprintf(stdout, "Using %d sample%s to approximate the self interaction error term",
xtot, xtot == 1 ? "" : "s");
if (PAR(cr))
{
fprintf(stdout, " (%d sample%s per rank)", x_per_core, x_per_core == 1 ? "" : "s");
}
fprintf(stdout, ".\n");
}
}
/* Return the number of positions used for the Monte Carlo algorithm */
*nsamples = xtot;
for (i = startlocal; i < stoplocal; i++)
{
e_rec3x = 0;
e_rec3y = 0;
e_rec3z = 0;
if (bFraction)
{
/* Randomly pick a charge */
ci = numbers[i];
}
else
{
/* Use all charges */
ci = i;
}
/* for(nx=startlocal; nx<=stoplocal; nx++)*/
for (nx = -info->nkx[0]/2; nx < info->nkx[0]/2+1; nx++)
{
svmul(nx, info->recipbox[XX], gridpx);
for (ny = -info->nky[0]/2; ny < info->nky[0]/2+1; ny++)
{
svmul(ny, info->recipbox[YY], tmpvec);
rvec_add(gridpx, tmpvec, gridpxy);
for (nz = -info->nkz[0]/2; nz < info->nkz[0]/2+1; nz++)
{
if (0 == nx && 0 == ny && 0 == nz)
{
continue;
}
svmul(nz, info->recipbox[ZZ], tmpvec);
rvec_add(gridpxy, tmpvec, gridp);
tmp = norm2(gridp);
coeff = exp(-1.0 * M_PI * M_PI * tmp / info->ewald_beta[0] / info->ewald_beta[0] );
coeff /= tmp;
e_rec3x += coeff*eps_self(nx, info->nkx[0], info->recipbox[XX], info->pme_order[0], x[ci]);
e_rec3y += coeff*eps_self(ny, info->nky[0], info->recipbox[YY], info->pme_order[0], x[ci]);
e_rec3z += coeff*eps_self(nz, info->nkz[0], info->recipbox[ZZ], info->pme_order[0], x[ci]);
}
}
}
clear_rvec(tmpvec2);
svmul(e_rec3x, info->recipbox[XX], tmpvec);
rvec_inc(tmpvec2, tmpvec);
svmul(e_rec3y, info->recipbox[YY], tmpvec);
rvec_inc(tmpvec2, tmpvec);
svmul(e_rec3z, info->recipbox[ZZ], tmpvec);
rvec_inc(tmpvec2, tmpvec);
e_rec3 += q[ci]*q[ci]*q[ci]*q[ci]*norm2(tmpvec2) / ( xtot * M_PI * info->volume * M_PI * info->volume);
if (MASTER(cr))
{
fprintf(stderr, "\rCalculating reciprocal error part 2 ... %3.0f%%",
100.0*(i+1)/stoplocal);
}
}
if (MASTER(cr))
{
fprintf(stderr, "\n");
}
#ifdef GMX_LIB_MPI
#ifdef TAKETIME
if (MASTER(cr))
{
t1 = MPI_Wtime() - t0;
fprintf(fp_out, "Recip. err. est. took : %lf s\n", t1);
}
#endif
#endif
#ifdef DEBUG
if (PAR(cr))
{
fprintf(stderr, "Rank %3d: nx=[%3d...%3d] e_rec3=%e\n",
cr->nodeid, startlocal, stoplocal, e_rec3);
}
#endif
if (PAR(cr))
{
gmx_sum(1, &e_rec1, cr);
gmx_sum(1, &e_rec2, cr);
gmx_sum(1, &e_rec3, cr);
}
/* e_rec1*=8.0 * q2_all / info->volume / info->volume / nr ;
e_rec2*= q2_all / M_PI / M_PI / info->volume / info->volume / nr ;
e_rec3/= M_PI * M_PI * info->volume * info->volume * nr ;
*/
e_rec = sqrt(e_rec1+e_rec2+e_rec3);
return ONE_4PI_EPS0 * e_rec;
}
gmx_bool replica_exchange(FILE *fplog, const t_commrec *cr,
const gmx_multisim_t *ms, struct gmx_repl_ex *re,
t_state *state, const gmx_enerdata_t *enerd,
t_state *state_local, int64_t step, real time)
{
int j;
int replica_id = 0;
int exchange_partner;
int maxswap = 0;
/* Number of rounds of exchanges needed to deal with any multiple
* exchanges. */
/* Where each replica ends up after the exchange attempt(s). */
/* The order in which multiple exchanges will occur. */
gmx_bool bThisReplicaExchanged = FALSE;
if (MASTER(cr))
{
replica_id = re->repl;
test_for_replica_exchange(fplog, ms, re, enerd, det(state_local->box), step, time);
prepare_to_do_exchange(re, replica_id, &maxswap, &bThisReplicaExchanged);
}
/* Do intra-simulation broadcast so all processors belonging to
* each simulation know whether they need to participate in
* collecting the state. Otherwise, they might as well get on with
* the next thing to do. */
if (DOMAINDECOMP(cr))
{
#if GMX_MPI
MPI_Bcast(&bThisReplicaExchanged, sizeof(gmx_bool), MPI_BYTE, MASTERRANK(cr),
cr->mpi_comm_mygroup);
#endif
}
if (bThisReplicaExchanged)
{
/* Exchange the states */
/* Collect the global state on the master node */
if (DOMAINDECOMP(cr))
{
dd_collect_state(cr->dd, state_local, state);
}
else
{
copy_state_serial(state_local, state);
}
if (MASTER(cr))
{
/* There will be only one swap cycle with standard replica
* exchange, but there may be multiple swap cycles if we
* allow multiple swaps. */
for (j = 0; j < maxswap; j++)
{
exchange_partner = re->order[replica_id][j];
if (exchange_partner != replica_id)
{
/* Exchange the global states between the master nodes */
if (debug)
{
fprintf(debug, "Exchanging %d with %d\n", replica_id, exchange_partner);
}
exchange_state(ms, exchange_partner, state);
}
}
/* For temperature-type replica exchange, we need to scale
* the velocities. */
if (re->type == ereTEMP || re->type == ereTL)
{
scale_velocities(state->v,
std::sqrt(re->q[ereTEMP][replica_id]/re->q[ereTEMP][re->destinations[replica_id]]));
}
}
/* With domain decomposition the global state is distributed later */
if (!DOMAINDECOMP(cr))
{
/* Copy the global state to the local state data structure */
copy_state_serial(state, state_local);
}
}
return bThisReplicaExchanged;
}
/* Estimate the error of the SPME Ewald sum. This estimate is based upon
* a) a homogeneous distribution of the charges
* b) a total charge of zero.
*/
static void estimate_PME_error(t_inputinfo *info, t_state *state,
gmx_mtop_t *mtop, FILE *fp_out, gmx_bool bVerbose, unsigned int seed,
t_commrec *cr)
{
rvec *x = NULL; /* The coordinates */
real *q = NULL; /* The charges */
real edir = 0.0; /* real space error */
real erec = 0.0; /* reciprocal space error */
real derr = 0.0; /* difference of real and reciprocal space error */
real derr0 = 0.0; /* difference of real and reciprocal space error */
real beta = 0.0; /* splitting parameter beta */
real beta0 = 0.0; /* splitting parameter beta */
int ncharges; /* The number of atoms with charges */
int nsamples; /* The number of samples used for the calculation of the
* self-energy error term */
int i = 0;
if (MASTER(cr))
{
fprintf(fp_out, "\n--- PME ERROR ESTIMATE ---\n");
}
/* Prepare an x and q array with only the charged atoms */
ncharges = prepare_x_q(&q, &x, mtop, state->x, cr);
if (MASTER(cr))
{
calc_q2all(mtop, &(info->q2all), &(info->q2allnr));
info->ewald_rtol[0] = gmx_erfc(info->rcoulomb[0]*info->ewald_beta[0]);
/* Write some info to log file */
fprintf(fp_out, "Box volume : %g nm^3\n", info->volume);
fprintf(fp_out, "Number of charged atoms : %d (total atoms %d)\n", ncharges, info->natoms);
fprintf(fp_out, "Coulomb radius : %g nm\n", info->rcoulomb[0]);
fprintf(fp_out, "Ewald_rtol : %g\n", info->ewald_rtol[0]);
fprintf(fp_out, "Ewald parameter beta : %g\n", info->ewald_beta[0]);
fprintf(fp_out, "Interpolation order : %d\n", info->pme_order[0]);
fprintf(fp_out, "Fourier grid (nx,ny,nz) : %d x %d x %d\n",
info->nkx[0], info->nky[0], info->nkz[0]);
fflush(fp_out);
}
if (PAR(cr))
{
bcast_info(info, cr);
}
/* Calculate direct space error */
info->e_dir[0] = estimate_direct(info);
/* Calculate reciprocal space error */
info->e_rec[0] = estimate_reciprocal(info, x, q, ncharges, fp_out, bVerbose,
seed, &nsamples, cr);
if (PAR(cr))
{
bcast_info(info, cr);
}
if (MASTER(cr))
{
fprintf(fp_out, "Direct space error est. : %10.3e kJ/(mol*nm)\n", info->e_dir[0]);
fprintf(fp_out, "Reciprocal sp. err. est.: %10.3e kJ/(mol*nm)\n", info->e_rec[0]);
fprintf(fp_out, "Self-energy error term was estimated using %d samples\n", nsamples);
fflush(fp_out);
fprintf(stderr, "Direct space error est. : %10.3e kJ/(mol*nm)\n", info->e_dir[0]);
fprintf(stderr, "Reciprocal sp. err. est.: %10.3e kJ/(mol*nm)\n", info->e_rec[0]);
}
i = 0;
if (info->bTUNE)
{
if (MASTER(cr))
{
fprintf(stderr, "Starting tuning ...\n");
}
edir = info->e_dir[0];
erec = info->e_rec[0];
derr0 = edir-erec;
beta0 = info->ewald_beta[0];
if (derr > 0.0)
{
info->ewald_beta[0] += 0.1;
}
else
{
info->ewald_beta[0] -= 0.1;
}
info->e_dir[0] = estimate_direct(info);
info->e_rec[0] = estimate_reciprocal(info, x, q, ncharges, fp_out, bVerbose,
seed, &nsamples, cr);
if (PAR(cr))
{
bcast_info(info, cr);
}
edir = info->e_dir[0];
erec = info->e_rec[0];
derr = edir-erec;
while (fabs(derr/std::min(erec, edir)) > 1e-4)
{
beta = info->ewald_beta[0];
beta -= derr*(info->ewald_beta[0]-beta0)/(derr-derr0);
beta0 = info->ewald_beta[0];
info->ewald_beta[0] = beta;
derr0 = derr;
info->e_dir[0] = estimate_direct(info);
info->e_rec[0] = estimate_reciprocal(info, x, q, ncharges, fp_out, bVerbose,
seed, &nsamples, cr);
if (PAR(cr))
{
bcast_info(info, cr);
}
edir = info->e_dir[0];
erec = info->e_rec[0];
derr = edir-erec;
if (MASTER(cr))
{
i++;
fprintf(stderr, "difference between real and rec. space error (step %d): %g\n", i, fabs(derr));
fprintf(stderr, "old beta: %f\n", beta0);
fprintf(stderr, "new beta: %f\n", beta);
}
}
info->ewald_rtol[0] = gmx_erfc(info->rcoulomb[0]*info->ewald_beta[0]);
if (MASTER(cr))
{
/* Write some info to log file */
fflush(fp_out);
fprintf(fp_out, "========= After tuning ========\n");
fprintf(fp_out, "Direct space error est. : %10.3e kJ/(mol*nm)\n", info->e_dir[0]);
fprintf(fp_out, "Reciprocal sp. err. est.: %10.3e kJ/(mol*nm)\n", info->e_rec[0]);
fprintf(stderr, "Direct space error est. : %10.3e kJ/(mol*nm)\n", info->e_dir[0]);
fprintf(stderr, "Reciprocal sp. err. est.: %10.3e kJ/(mol*nm)\n", info->e_rec[0]);
fprintf(fp_out, "Ewald_rtol : %g\n", info->ewald_rtol[0]);
fprintf(fp_out, "Ewald parameter beta : %g\n", info->ewald_beta[0]);
fflush(fp_out);
}
}
}
gmx_mdoutf_t init_mdoutf(FILE *fplog, int nfile, const t_filenm fnm[],
int mdrun_flags, const t_commrec *cr,
const t_inputrec *ir, gmx_mtop_t *top_global,
const output_env_t oenv, gmx_wallcycle_t wcycle)
{
gmx_mdoutf_t of;
char filemode[3];
gmx_bool bAppendFiles, bCiteTng = FALSE;
int i;
snew(of, 1);
of->fp_trn = NULL;
of->fp_ene = NULL;
of->fp_xtc = NULL;
of->tng = NULL;
of->tng_low_prec = NULL;
of->fp_dhdl = NULL;
of->fp_field = NULL;
of->eIntegrator = ir->eI;
of->bExpanded = ir->bExpanded;
of->elamstats = ir->expandedvals->elamstats;
of->simulation_part = ir->simulation_part;
of->x_compression_precision = static_cast<int>(ir->x_compression_precision);
of->wcycle = wcycle;
if (MASTER(cr))
{
bAppendFiles = (mdrun_flags & MD_APPENDFILES);
of->bKeepAndNumCPT = (mdrun_flags & MD_KEEPANDNUMCPT);
sprintf(filemode, bAppendFiles ? "a+" : "w+");
if ((EI_DYNAMICS(ir->eI) || EI_ENERGY_MINIMIZATION(ir->eI))
#ifndef GMX_FAHCORE
&&
!(EI_DYNAMICS(ir->eI) &&
ir->nstxout == 0 &&
ir->nstvout == 0 &&
ir->nstfout == 0)
#endif
)
{
const char *filename;
filename = ftp2fn(efTRN, nfile, fnm);
switch (fn2ftp(filename))
{
case efTRR:
case efTRN:
of->fp_trn = gmx_trr_open(filename, filemode);
break;
case efTNG:
gmx_tng_open(filename, filemode[0], &of->tng);
if (filemode[0] == 'w')
{
gmx_tng_prepare_md_writing(of->tng, top_global, ir);
}
bCiteTng = TRUE;
break;
default:
gmx_incons("Invalid full precision file format");
}
}
if (EI_DYNAMICS(ir->eI) &&
ir->nstxout_compressed > 0)
{
const char *filename;
filename = ftp2fn(efCOMPRESSED, nfile, fnm);
switch (fn2ftp(filename))
{
case efXTC:
of->fp_xtc = open_xtc(filename, filemode);
break;
case efTNG:
gmx_tng_open(filename, filemode[0], &of->tng_low_prec);
if (filemode[0] == 'w')
{
gmx_tng_prepare_low_prec_writing(of->tng_low_prec, top_global, ir);
}
bCiteTng = TRUE;
break;
default:
gmx_incons("Invalid reduced precision file format");
}
}
if (EI_DYNAMICS(ir->eI) || EI_ENERGY_MINIMIZATION(ir->eI))
{
of->fp_ene = open_enx(ftp2fn(efEDR, nfile, fnm), filemode);
}
of->fn_cpt = opt2fn("-cpo", nfile, fnm);
if ((ir->efep != efepNO || ir->bSimTemp) && ir->fepvals->nstdhdl > 0 &&
(ir->fepvals->separate_dhdl_file == esepdhdlfileYES ) &&
EI_DYNAMICS(ir->eI))
{
if (bAppendFiles)
{
of->fp_dhdl = gmx_fio_fopen(opt2fn("-dhdl", nfile, fnm), filemode);
}
else
{
of->fp_dhdl = open_dhdl(opt2fn("-dhdl", nfile, fnm), ir, oenv);
}
}
if (opt2bSet("-field", nfile, fnm) &&
(ir->ex[XX].n || ir->ex[YY].n || ir->ex[ZZ].n))
{
if (bAppendFiles)
{
of->fp_field = gmx_fio_fopen(opt2fn("-field", nfile, fnm),
filemode);
}
else
{
of->fp_field = xvgropen(opt2fn("-field", nfile, fnm),
"Applied electric field", "Time (ps)",
"E (V/nm)", oenv);
}
}
/* Set up atom counts so they can be passed to actual
trajectory-writing routines later. Also, XTC writing needs
to know what (and how many) atoms might be in the XTC
groups, and how to look up later which ones they are. */
of->natoms_global = top_global->natoms;
of->groups = &top_global->groups;
of->natoms_x_compressed = 0;
for (i = 0; (i < top_global->natoms); i++)
{
if (ggrpnr(of->groups, egcCompressedX, i) == 0)
{
of->natoms_x_compressed++;
}
}
}
if (bCiteTng)
{
please_cite(fplog, "Lundborg2014");
}
return of;
}
void init_gamess(t_commrec *cr, t_QMrec *qm, t_MMrec *mm)
{
/* it works hopelessly complicated :-)
* first a file is written. Then the standard gamess input/output
* routine is called (no system()!) to set up all fortran arrays.
* this routine writes a punch file, like in a normal gamess run.
* via this punch file the other games routines, needed for gradient
* and energy evaluations are called. This setup works fine for
* dynamics simulations. 7-6-2002 (London)
*/
int
i, j;
FILE
*out;
char
periodic_system[37][3] = {
"XX", "H ", "He", "Li", "Be", "B ", "C ", "N ",
"O ", "F ", "Ne", "Na", "Mg", "Al", "Si", "P ",
"S ", "Cl", "Ar", "K ", "Ca", "Sc", "Ti", "V ",
"Cr", "Mn", "Fe", "Co", "Ni", "Cu", "Zn", "Ga",
"Ge", "As", "Se", "Br", "Kr"
};
if (PAR(cr))
{
if (MASTER(cr))
{
out = fopen("FOR009", "w");
/* of these options I am not completely sure.... the overall
* preformance on more than 4 cpu's is rather poor at the moment.
*/
fprintf(out, "memory 48000000\nPARALLEL IOMODE SCREENED\n");
fprintf(out, "ELEC %d\nMULT %d\nSUPER ON\nNOSYM\nGEOMETRY ANGSTROM\n",
qm->nelectrons, qm->multiplicity);
for (i = 0; i < qm->nrQMatoms; i++)
{
#ifdef DOUBLE
fprintf(out, "%10.7lf %10.7lf %10.7lf %5.3lf %2s\n",
i/2.,
i/3.,
i/4.,
qm->atomicnumberQM[i]*1.0,
periodic_system[qm->atomicnumberQM[i]]);
#else
fprintf(out, "%10.7f %10.7f %10.7f %5.3f %2s\n",
i/2.,
i/3.,
i/4.,
qm->atomicnumberQM[i]*1.0,
periodic_system[qm->atomicnumberQM[i]]);
#endif
}
if (mm->nrMMatoms)
{
for (j = i; j < i+2; j++)
{
#ifdef DOUBLE
fprintf(out, "%10.7lf %10.7lf %10.7lf %5.3lf BQ\n",
j/5.,
j/6.,
j/7.,
1.0);
#else
fprintf(out, "%10.7f %10.7f %10.7f %5.3f BQ\n",
j/5.,
j/6.,
j/7.,
2.0);
#endif
}
}
if (!qm->bTS)
{
fprintf(out, "END\nBASIS %s\nRUNTYPE GRADIENT\nSCFTYPE %s\n",
eQMbasis_names[qm->QMbasis],
eQMmethod_names[qm->QMmethod]); /* see enum.h */
}
else
{
fprintf(out, "END\nBASIS %s\nRUNTYPE SADDLE\nSCFTYPE %s\n",
eQMbasis_names[qm->QMbasis],
eQMmethod_names[qm->QMmethod]); /* see enum.h */
}
fclose(out);
}
gmx_barrier(cr);
F77_FUNC(inigms, IMIGMS) ();
}
else /* normal serial run */
{
out = fopen("FOR009", "w");
/* of these options I am not completely sure.... the overall
* preformance on more than 4 cpu's is rather poor at the moment.
*/
fprintf(out, "ELEC %d\nMULT %d\nSUPER ON\nNOSYM\nGEOMETRY ANGSTROM\n",
qm->nelectrons, qm->multiplicity);
for (i = 0; i < qm->nrQMatoms; i++)
{
#ifdef DOUBLE
fprintf(out, "%10.7lf %10.7lf %10.7lf %5.3lf %2s\n",
i/2.,
i/3.,
i/4.,
qm->atomicnumberQM[i]*1.0,
periodic_system[qm->atomicnumberQM[i]]);
#else
fprintf(out, "%10.7f %10.7f %10.7f %5.3f %2s\n",
i/2.,
i/3.,
i/4.,
qm->atomicnumberQM[i]*1.0,
periodic_system[qm->atomicnumberQM[i]]);
#endif
}
if (mm->nrMMatoms)
{
for (j = i; j < i+2; j++)
{
#ifdef DOUBLE
fprintf(out, "%10.7lf %10.7lf %10.7lf %5.3lf BQ\n",
j/5.,
j/6.,
j/7.,
1.0);
#else
fprintf(out, "%10.7f %10.7f %10.7f %5.3f BQ\n",
j/5.,
j/6.,
j/7.,
2.0);
#endif
}
}
if (!qm->bTS)
{
fprintf(out, "END\nBASIS %s\nRUNTYPE GRADIENT\nSCFTYPE %s\n",
eQMbasis_names[qm->QMbasis],
eQMmethod_names[qm->QMmethod]); /* see enum.h */
}
else
{
fprintf(out, "END\nBASIS %s\nRUNTYPE SADDLE\nSCFTYPE %s\n",
eQMbasis_names[qm->QMbasis],
eQMmethod_names[qm->QMmethod]); /* see enum.h */
}
F77_FUNC(inigms, IMIGMS) ();
}
}
/* Try to increase nstlist when using the Verlet cut-off scheme */
static void increase_nstlist(FILE *fp, t_commrec *cr,
t_inputrec *ir, int nstlist_cmdline,
const gmx_mtop_t *mtop, matrix box,
gmx_bool bGPU)
{
float listfac_ok, listfac_max;
int nstlist_orig, nstlist_prev;
verletbuf_list_setup_t ls;
real rlistWithReferenceNstlist, rlist_inc, rlist_ok, rlist_max;
real rlist_new, rlist_prev;
size_t nstlist_ind = 0;
t_state state_tmp;
gmx_bool bBox, bDD, bCont;
const char *nstl_gpu = "\nFor optimal performance with a GPU nstlist (now %d) should be larger.\nThe optimum depends on your CPU and GPU resources.\nYou might want to try several nstlist values.\n";
const char *nve_err = "Can not increase nstlist because an NVE ensemble is used";
const char *vbd_err = "Can not increase nstlist because verlet-buffer-tolerance is not set or used";
const char *box_err = "Can not increase nstlist because the box is too small";
const char *dd_err = "Can not increase nstlist because of domain decomposition limitations";
char buf[STRLEN];
const float oneThird = 1.0f / 3.0f;
if (nstlist_cmdline <= 0)
{
if (ir->nstlist == 1)
{
/* The user probably set nstlist=1 for a reason,
* don't mess with the settings.
*/
return;
}
if (fp != NULL && bGPU && ir->nstlist < nstlist_try[0])
{
fprintf(fp, nstl_gpu, ir->nstlist);
}
nstlist_ind = 0;
while (nstlist_ind < NNSTL && ir->nstlist >= nstlist_try[nstlist_ind])
{
nstlist_ind++;
}
if (nstlist_ind == NNSTL)
{
/* There are no larger nstlist value to try */
return;
}
}
if (EI_MD(ir->eI) && ir->etc == etcNO)
{
if (MASTER(cr))
{
fprintf(stderr, "%s\n", nve_err);
}
if (fp != NULL)
{
fprintf(fp, "%s\n", nve_err);
}
return;
}
if (ir->verletbuf_tol == 0 && bGPU)
{
gmx_fatal(FARGS, "You are using an old tpr file with a GPU, please generate a new tpr file with an up to date version of grompp");
}
if (ir->verletbuf_tol < 0)
{
if (MASTER(cr))
{
fprintf(stderr, "%s\n", vbd_err);
}
if (fp != NULL)
{
fprintf(fp, "%s\n", vbd_err);
}
return;
}
if (bGPU)
{
listfac_ok = nbnxn_gpu_listfac_ok;
listfac_max = nbnxn_gpu_listfac_max;
}
else
{
listfac_ok = nbnxn_cpu_listfac_ok;
listfac_max = nbnxn_cpu_listfac_max;
}
nstlist_orig = ir->nstlist;
if (nstlist_cmdline > 0)
{
if (fp)
{
sprintf(buf, "Getting nstlist=%d from command line option",
nstlist_cmdline);
}
ir->nstlist = nstlist_cmdline;
}
verletbuf_get_list_setup(TRUE, bGPU, &ls);
/* Allow rlist to make the list a given factor larger than the list
* would be with the reference value for nstlist (10).
*/
nstlist_prev = ir->nstlist;
ir->nstlist = nbnxnReferenceNstlist;
calc_verlet_buffer_size(mtop, det(box), ir, -1, &ls, NULL,
&rlistWithReferenceNstlist);
ir->nstlist = nstlist_prev;
/* Determine the pair list size increase due to zero interactions */
rlist_inc = nbnxn_get_rlist_effective_inc(ls.cluster_size_j,
mtop->natoms/det(box));
rlist_ok = (rlistWithReferenceNstlist + rlist_inc)*pow(listfac_ok, oneThird) - rlist_inc;
rlist_max = (rlistWithReferenceNstlist + rlist_inc)*pow(listfac_max, oneThird) - rlist_inc;
if (debug)
{
fprintf(debug, "nstlist tuning: rlist_inc %.3f rlist_ok %.3f rlist_max %.3f\n",
rlist_inc, rlist_ok, rlist_max);
}
nstlist_prev = nstlist_orig;
rlist_prev = ir->rlist;
do
{
if (nstlist_cmdline <= 0)
{
ir->nstlist = nstlist_try[nstlist_ind];
}
/* Set the pair-list buffer size in ir */
calc_verlet_buffer_size(mtop, det(box), ir, -1, &ls, NULL, &rlist_new);
/* Does rlist fit in the box? */
bBox = (sqr(rlist_new) < max_cutoff2(ir->ePBC, box));
bDD = TRUE;
if (bBox && DOMAINDECOMP(cr))
{
/* Check if rlist fits in the domain decomposition */
if (inputrec2nboundeddim(ir) < DIM)
{
gmx_incons("Changing nstlist with domain decomposition and unbounded dimensions is not implemented yet");
}
copy_mat(box, state_tmp.box);
bDD = change_dd_cutoff(cr, &state_tmp, ir, rlist_new);
}
if (debug)
{
fprintf(debug, "nstlist %d rlist %.3f bBox %d bDD %d\n",
ir->nstlist, rlist_new, bBox, bDD);
}
bCont = FALSE;
if (nstlist_cmdline <= 0)
{
if (bBox && bDD && rlist_new <= rlist_max)
{
/* Increase nstlist */
nstlist_prev = ir->nstlist;
rlist_prev = rlist_new;
bCont = (nstlist_ind+1 < NNSTL && rlist_new < rlist_ok);
}
else
{
/* Stick with the previous nstlist */
ir->nstlist = nstlist_prev;
rlist_new = rlist_prev;
bBox = TRUE;
bDD = TRUE;
}
}
nstlist_ind++;
}
while (bCont);
if (!bBox || !bDD)
{
gmx_warning(!bBox ? box_err : dd_err);
if (fp != NULL)
{
fprintf(fp, "\n%s\n", bBox ? box_err : dd_err);
}
ir->nstlist = nstlist_orig;
}
else if (ir->nstlist != nstlist_orig || rlist_new != ir->rlist)
{
sprintf(buf, "Changing nstlist from %d to %d, rlist from %g to %g",
nstlist_orig, ir->nstlist,
ir->rlist, rlist_new);
if (MASTER(cr))
{
fprintf(stderr, "%s\n\n", buf);
}
if (fp != NULL)
{
fprintf(fp, "%s\n\n", buf);
}
ir->rlist = rlist_new;
ir->rlistlong = rlist_new;
}
}
int cmain(int argc, char *argv[])
{
const char *desc[] = {
"The [TT]mdrun[tt] program is the main computational chemistry engine",
"within GROMACS. Obviously, it performs Molecular Dynamics simulations,",
"but it can also perform Stochastic Dynamics, Energy Minimization,",
"test particle insertion or (re)calculation of energies.",
"Normal mode analysis is another option. In this case [TT]mdrun[tt]",
"builds a Hessian matrix from single conformation.",
"For usual Normal Modes-like calculations, make sure that",
"the structure provided is properly energy-minimized.",
"The generated matrix can be diagonalized by [TT]g_nmeig[tt].[PAR]",
"The [TT]mdrun[tt] program reads the run input file ([TT]-s[tt])",
"and distributes the topology over nodes if needed.",
"[TT]mdrun[tt] produces at least four output files.",
"A single log file ([TT]-g[tt]) is written, unless the option",
"[TT]-seppot[tt] is used, in which case each node writes a log file.",
"The trajectory file ([TT]-o[tt]), contains coordinates, velocities and",
"optionally forces.",
"The structure file ([TT]-c[tt]) contains the coordinates and",
"velocities of the last step.",
"The energy file ([TT]-e[tt]) contains energies, the temperature,",
"pressure, etc, a lot of these things are also printed in the log file.",
"Optionally coordinates can be written to a compressed trajectory file",
"([TT]-x[tt]).[PAR]",
"The option [TT]-dhdl[tt] is only used when free energy calculation is",
"turned on.[PAR]",
"A simulation can be run in parallel using two different parallelization",
"schemes: MPI parallelization and/or OpenMP thread parallelization.",
"The MPI parallelization uses multiple processes when [TT]mdrun[tt] is",
"compiled with a normal MPI library or threads when [TT]mdrun[tt] is",
"compiled with the GROMACS built-in thread-MPI library. OpenMP threads",
"are supported when mdrun is compiled with OpenMP. Full OpenMP support",
"is only available with the Verlet cut-off scheme, with the (older)",
"group scheme only PME-only processes can use OpenMP parallelization.",
"In all cases [TT]mdrun[tt] will by default try to use all the available",
"hardware resources. With a normal MPI library only the options",
"[TT]-ntomp[tt] (with the Verlet cut-off scheme) and [TT]-ntomp_pme[tt],",
"for PME-only processes, can be used to control the number of threads.",
"With thread-MPI there are additional options [TT]-nt[tt], which sets",
"the total number of threads, and [TT]-ntmpi[tt], which sets the number",
"of thread-MPI threads.",
"Note that using combined MPI+OpenMP parallelization is almost always",
"slower than single parallelization, except at the scaling limit, where",
"especially OpenMP parallelization of PME reduces the communication cost.",
"OpenMP-only parallelization is much faster than MPI-only parallelization",
"on a single CPU(-die). Since we currently don't have proper hardware",
"topology detection, [TT]mdrun[tt] compiled with thread-MPI will only",
"automatically use OpenMP-only parallelization when you use up to 4",
"threads, up to 12 threads with Intel Nehalem/Westmere, or up to 16",
"threads with Intel Sandy Bridge or newer CPUs. Otherwise MPI-only",
"parallelization is used (except with GPUs, see below).",
"[PAR]",
"To quickly test the performance of the new Verlet cut-off scheme",
"with old [TT].tpr[tt] files, either on CPUs or CPUs+GPUs, you can use",
"the [TT]-testverlet[tt] option. This should not be used for production,",
"since it can slightly modify potentials and it will remove charge groups",
"making analysis difficult, as the [TT].tpr[tt] file will still contain",
"charge groups. For production simulations it is highly recommended",
"to specify [TT]cutoff-scheme = Verlet[tt] in the [TT].mdp[tt] file.",
"[PAR]",
"With GPUs (only supported with the Verlet cut-off scheme), the number",
"of GPUs should match the number of MPI processes or MPI threads,",
"excluding PME-only processes/threads. With thread-MPI, unless set on the command line, the number",
"of MPI threads will automatically be set to the number of GPUs detected.",
"To use a subset of the available GPUs, or to manually provide a mapping of",
"GPUs to PP ranks, you can use the [TT]-gpu_id[tt] option. The argument of [TT]-gpu_id[tt] is",
"a string of digits (without delimiter) representing device id-s of the GPUs to be used.",
"For example, \"[TT]02[tt]\" specifies using GPUs 0 and 2 in the first and second PP ranks per compute node",
"respectively. To select different sets of GPU-s",
"on different nodes of a compute cluster, use the [TT]GMX_GPU_ID[tt] environment",
"variable instead. The format for [TT]GMX_GPU_ID[tt] is identical to ",
"[TT]-gpu_id[tt], with the difference that an environment variable can have",
"different values on different compute nodes. Multiple MPI ranks on each node",
"can share GPUs. This is accomplished by specifying the id(s) of the GPU(s)",
"multiple times, e.g. \"[TT]0011[tt]\" for four ranks sharing two GPUs in this node.",
"This works within a single simulation, or a multi-simulation, with any form of MPI.",
"[PAR]",
"When using PME with separate PME nodes or with a GPU, the two major",
"compute tasks, the non-bonded force calculation and the PME calculation",
"run on different compute resources. If this load is not balanced,",
"some of the resources will be idle part of time. With the Verlet",
"cut-off scheme this load is automatically balanced when the PME load",
"is too high (but not when it is too low). This is done by scaling",
"the Coulomb cut-off and PME grid spacing by the same amount. In the first",
"few hundred steps different settings are tried and the fastest is chosen",
"for the rest of the simulation. This does not affect the accuracy of",
"the results, but it does affect the decomposition of the Coulomb energy",
"into particle and mesh contributions. The auto-tuning can be turned off",
"with the option [TT]-notunepme[tt].",
"[PAR]",
"[TT]mdrun[tt] pins (sets affinity of) threads to specific cores,",
"when all (logical) cores on a compute node are used by [TT]mdrun[tt],",
"even when no multi-threading is used,",
"as this usually results in significantly better performance.",
"If the queuing systems or the OpenMP library pinned threads, we honor",
"this and don't pin again, even though the layout may be sub-optimal.",
"If you want to have [TT]mdrun[tt] override an already set thread affinity",
"or pin threads when using less cores, use [TT]-pin on[tt].",
"With SMT (simultaneous multithreading), e.g. Intel Hyper-Threading,",
"there are multiple logical cores per physical core.",
"The option [TT]-pinstride[tt] sets the stride in logical cores for",
"pinning consecutive threads. Without SMT, 1 is usually the best choice.",
"With Intel Hyper-Threading 2 is best when using half or less of the",
"logical cores, 1 otherwise. The default value of 0 do exactly that:",
"it minimizes the threads per logical core, to optimize performance.",
"If you want to run multiple mdrun jobs on the same physical node,"
"you should set [TT]-pinstride[tt] to 1 when using all logical cores.",
"When running multiple mdrun (or other) simulations on the same physical",
"node, some simulations need to start pinning from a non-zero core",
"to avoid overloading cores; with [TT]-pinoffset[tt] you can specify",
"the offset in logical cores for pinning.",
"[PAR]",
"When [TT]mdrun[tt] is started using MPI with more than 1 process",
"or with thread-MPI with more than 1 thread, MPI parallelization is used.",
"By default domain decomposition is used, unless the [TT]-pd[tt]",
"option is set, which selects particle decomposition.",
"[PAR]",
"With domain decomposition, the spatial decomposition can be set",
"with option [TT]-dd[tt]. By default [TT]mdrun[tt] selects a good decomposition.",
"The user only needs to change this when the system is very inhomogeneous.",
"Dynamic load balancing is set with the option [TT]-dlb[tt],",
"which can give a significant performance improvement,",
"especially for inhomogeneous systems. The only disadvantage of",
"dynamic load balancing is that runs are no longer binary reproducible,",
"but in most cases this is not important.",
"By default the dynamic load balancing is automatically turned on",
"when the measured performance loss due to load imbalance is 5% or more.",
"At low parallelization these are the only important options",
"for domain decomposition.",
"At high parallelization the options in the next two sections",
"could be important for increasing the performace.",
"[PAR]",
"When PME is used with domain decomposition, separate nodes can",
"be assigned to do only the PME mesh calculation;",
"this is computationally more efficient starting at about 12 nodes.",
"The number of PME nodes is set with option [TT]-npme[tt],",
"this can not be more than half of the nodes.",
"By default [TT]mdrun[tt] makes a guess for the number of PME",
"nodes when the number of nodes is larger than 11 or performance wise",
"not compatible with the PME grid x dimension.",
"But the user should optimize npme. Performance statistics on this issue",
"are written at the end of the log file.",
"For good load balancing at high parallelization, the PME grid x and y",
"dimensions should be divisible by the number of PME nodes",
"(the simulation will run correctly also when this is not the case).",
"[PAR]",
"This section lists all options that affect the domain decomposition.",
"[PAR]",
"Option [TT]-rdd[tt] can be used to set the required maximum distance",
"for inter charge-group bonded interactions.",
"Communication for two-body bonded interactions below the non-bonded",
"cut-off distance always comes for free with the non-bonded communication.",
"Atoms beyond the non-bonded cut-off are only communicated when they have",
"missing bonded interactions; this means that the extra cost is minor",
"and nearly indepedent of the value of [TT]-rdd[tt].",
"With dynamic load balancing option [TT]-rdd[tt] also sets",
"the lower limit for the domain decomposition cell sizes.",
"By default [TT]-rdd[tt] is determined by [TT]mdrun[tt] based on",
"the initial coordinates. The chosen value will be a balance",
"between interaction range and communication cost.",
"[PAR]",
"When inter charge-group bonded interactions are beyond",
"the bonded cut-off distance, [TT]mdrun[tt] terminates with an error message.",
"For pair interactions and tabulated bonds",
"that do not generate exclusions, this check can be turned off",
"with the option [TT]-noddcheck[tt].",
"[PAR]",
"When constraints are present, option [TT]-rcon[tt] influences",
"the cell size limit as well.",
"Atoms connected by NC constraints, where NC is the LINCS order plus 1,",
"should not be beyond the smallest cell size. A error message is",
"generated when this happens and the user should change the decomposition",
"or decrease the LINCS order and increase the number of LINCS iterations.",
"By default [TT]mdrun[tt] estimates the minimum cell size required for P-LINCS",
"in a conservative fashion. For high parallelization it can be useful",
"to set the distance required for P-LINCS with the option [TT]-rcon[tt].",
"[PAR]",
"The [TT]-dds[tt] option sets the minimum allowed x, y and/or z scaling",
"of the cells with dynamic load balancing. [TT]mdrun[tt] will ensure that",
"the cells can scale down by at least this factor. This option is used",
"for the automated spatial decomposition (when not using [TT]-dd[tt])",
"as well as for determining the number of grid pulses, which in turn",
"sets the minimum allowed cell size. Under certain circumstances",
"the value of [TT]-dds[tt] might need to be adjusted to account for",
"high or low spatial inhomogeneity of the system.",
"[PAR]",
"The option [TT]-gcom[tt] can be used to only do global communication",
"every n steps.",
"This can improve performance for highly parallel simulations",
"where this global communication step becomes the bottleneck.",
"For a global thermostat and/or barostat the temperature",
"and/or pressure will also only be updated every [TT]-gcom[tt] steps.",
"By default it is set to the minimum of nstcalcenergy and nstlist.[PAR]",
"With [TT]-rerun[tt] an input trajectory can be given for which ",
"forces and energies will be (re)calculated. Neighbor searching will be",
"performed for every frame, unless [TT]nstlist[tt] is zero",
"(see the [TT].mdp[tt] file).[PAR]",
"ED (essential dynamics) sampling and/or additional flooding potentials",
"are switched on by using the [TT]-ei[tt] flag followed by an [TT].edi[tt]",
"file. The [TT].edi[tt] file can be produced with the [TT]make_edi[tt] tool",
"or by using options in the essdyn menu of the WHAT IF program.",
"[TT]mdrun[tt] produces a [TT].xvg[tt] output file that",
"contains projections of positions, velocities and forces onto selected",
"eigenvectors.[PAR]",
"When user-defined potential functions have been selected in the",
"[TT].mdp[tt] file the [TT]-table[tt] option is used to pass [TT]mdrun[tt]",
"a formatted table with potential functions. The file is read from",
"either the current directory or from the [TT]GMXLIB[tt] directory.",
"A number of pre-formatted tables are presented in the [TT]GMXLIB[tt] dir,",
"for 6-8, 6-9, 6-10, 6-11, 6-12 Lennard-Jones potentials with",
"normal Coulomb.",
"When pair interactions are present, a separate table for pair interaction",
"functions is read using the [TT]-tablep[tt] option.[PAR]",
"When tabulated bonded functions are present in the topology,",
"interaction functions are read using the [TT]-tableb[tt] option.",
"For each different tabulated interaction type the table file name is",
"modified in a different way: before the file extension an underscore is",
"appended, then a 'b' for bonds, an 'a' for angles or a 'd' for dihedrals",
"and finally the table number of the interaction type.[PAR]",
"The options [TT]-px[tt] and [TT]-pf[tt] are used for writing pull COM",
"coordinates and forces when pulling is selected",
"in the [TT].mdp[tt] file.[PAR]",
"With [TT]-multi[tt] or [TT]-multidir[tt], multiple systems can be ",
"simulated in parallel.",
"As many input files/directories are required as the number of systems. ",
"The [TT]-multidir[tt] option takes a list of directories (one for each ",
"system) and runs in each of them, using the input/output file names, ",
"such as specified by e.g. the [TT]-s[tt] option, relative to these ",
"directories.",
"With [TT]-multi[tt], the system number is appended to the run input ",
"and each output filename, for instance [TT]topol.tpr[tt] becomes",
"[TT]topol0.tpr[tt], [TT]topol1.tpr[tt] etc.",
"The number of nodes per system is the total number of nodes",
"divided by the number of systems.",
"One use of this option is for NMR refinement: when distance",
"or orientation restraints are present these can be ensemble averaged",
"over all the systems.[PAR]",
"With [TT]-replex[tt] replica exchange is attempted every given number",
"of steps. The number of replicas is set with the [TT]-multi[tt] or ",
"[TT]-multidir[tt] option, described above.",
"All run input files should use a different coupling temperature,",
"the order of the files is not important. The random seed is set with",
"[TT]-reseed[tt]. The velocities are scaled and neighbor searching",
"is performed after every exchange.[PAR]",
"Finally some experimental algorithms can be tested when the",
"appropriate options have been given. Currently under",
"investigation are: polarizability and X-ray bombardments.",
"[PAR]",
"The option [TT]-membed[tt] does what used to be g_membed, i.e. embed",
"a protein into a membrane. The data file should contain the options",
"that where passed to g_membed before. The [TT]-mn[tt] and [TT]-mp[tt]",
"both apply to this as well.",
"[PAR]",
"The option [TT]-pforce[tt] is useful when you suspect a simulation",
"crashes due to too large forces. With this option coordinates and",
"forces of atoms with a force larger than a certain value will",
"be printed to stderr.",
"[PAR]",
"Checkpoints containing the complete state of the system are written",
"at regular intervals (option [TT]-cpt[tt]) to the file [TT]-cpo[tt],",
"unless option [TT]-cpt[tt] is set to -1.",
"The previous checkpoint is backed up to [TT]state_prev.cpt[tt] to",
"make sure that a recent state of the system is always available,",
"even when the simulation is terminated while writing a checkpoint.",
"With [TT]-cpnum[tt] all checkpoint files are kept and appended",
"with the step number.",
"A simulation can be continued by reading the full state from file",
"with option [TT]-cpi[tt]. This option is intelligent in the way that",
"if no checkpoint file is found, Gromacs just assumes a normal run and",
"starts from the first step of the [TT].tpr[tt] file. By default the output",
"will be appending to the existing output files. The checkpoint file",
"contains checksums of all output files, such that you will never",
"loose data when some output files are modified, corrupt or removed.",
"There are three scenarios with [TT]-cpi[tt]:[PAR]",
"[TT]*[tt] no files with matching names are present: new output files are written[PAR]",
"[TT]*[tt] all files are present with names and checksums matching those stored",
"in the checkpoint file: files are appended[PAR]",
"[TT]*[tt] otherwise no files are modified and a fatal error is generated[PAR]",
"With [TT]-noappend[tt] new output files are opened and the simulation",
"part number is added to all output file names.",
"Note that in all cases the checkpoint file itself is not renamed",
"and will be overwritten, unless its name does not match",
"the [TT]-cpo[tt] option.",
"[PAR]",
"With checkpointing the output is appended to previously written",
"output files, unless [TT]-noappend[tt] is used or none of the previous",
"output files are present (except for the checkpoint file).",
"The integrity of the files to be appended is verified using checksums",
"which are stored in the checkpoint file. This ensures that output can",
"not be mixed up or corrupted due to file appending. When only some",
"of the previous output files are present, a fatal error is generated",
"and no old output files are modified and no new output files are opened.",
"The result with appending will be the same as from a single run.",
"The contents will be binary identical, unless you use a different number",
"of nodes or dynamic load balancing or the FFT library uses optimizations",
"through timing.",
"[PAR]",
"With option [TT]-maxh[tt] a simulation is terminated and a checkpoint",
"file is written at the first neighbor search step where the run time",
"exceeds [TT]-maxh[tt]*0.99 hours.",
"[PAR]",
"When [TT]mdrun[tt] receives a TERM signal, it will set nsteps to the current",
"step plus one. When [TT]mdrun[tt] receives an INT signal (e.g. when ctrl+C is",
"pressed), it will stop after the next neighbor search step ",
"(with nstlist=0 at the next step).",
"In both cases all the usual output will be written to file.",
"When running with MPI, a signal to one of the [TT]mdrun[tt] processes",
"is sufficient, this signal should not be sent to mpirun or",
"the [TT]mdrun[tt] process that is the parent of the others.",
"[PAR]",
"When [TT]mdrun[tt] is started with MPI, it does not run niced by default."
};
t_commrec *cr;
t_filenm fnm[] = {
{ efTPX, NULL, NULL, ffREAD },
{ efTRN, "-o", NULL, ffWRITE },
{ efXTC, "-x", NULL, ffOPTWR },
{ efCPT, "-cpi", NULL, ffOPTRD },
{ efCPT, "-cpo", NULL, ffOPTWR },
{ efSTO, "-c", "confout", ffWRITE },
{ efEDR, "-e", "ener", ffWRITE },
{ efLOG, "-g", "md", ffWRITE },
{ efXVG, "-dhdl", "dhdl", ffOPTWR },
{ efXVG, "-field", "field", ffOPTWR },
{ efXVG, "-table", "table", ffOPTRD },
{ efXVG, "-tabletf", "tabletf", ffOPTRD },
{ efXVG, "-tablep", "tablep", ffOPTRD },
{ efXVG, "-tableb", "table", ffOPTRD },
{ efTRX, "-rerun", "rerun", ffOPTRD },
{ efXVG, "-tpi", "tpi", ffOPTWR },
{ efXVG, "-tpid", "tpidist", ffOPTWR },
{ efEDI, "-ei", "sam", ffOPTRD },
{ efXVG, "-eo", "edsam", ffOPTWR },
{ efGCT, "-j", "wham", ffOPTRD },
{ efGCT, "-jo", "bam", ffOPTWR },
{ efXVG, "-ffout", "gct", ffOPTWR },
{ efXVG, "-devout", "deviatie", ffOPTWR },
{ efXVG, "-runav", "runaver", ffOPTWR },
{ efXVG, "-px", "pullx", ffOPTWR },
{ efXVG, "-pf", "pullf", ffOPTWR },
{ efXVG, "-ro", "rotation", ffOPTWR },
{ efLOG, "-ra", "rotangles", ffOPTWR },
{ efLOG, "-rs", "rotslabs", ffOPTWR },
{ efLOG, "-rt", "rottorque", ffOPTWR },
{ efMTX, "-mtx", "nm", ffOPTWR },
{ efNDX, "-dn", "dipole", ffOPTWR },
{ efRND, "-multidir", NULL, ffOPTRDMULT},
{ efDAT, "-plumed", "plumed", ffOPTRD }, /* PLUMED */
{ efDAT, "-membed", "membed", ffOPTRD },
{ efTOP, "-mp", "membed", ffOPTRD },
{ efNDX, "-mn", "membed", ffOPTRD }
};
#define NFILE asize(fnm)
/* Command line options ! */
gmx_bool bCart = FALSE;
gmx_bool bPPPME = FALSE;
gmx_bool bPartDec = FALSE;
gmx_bool bDDBondCheck = TRUE;
gmx_bool bDDBondComm = TRUE;
gmx_bool bTunePME = TRUE;
gmx_bool bTestVerlet = FALSE;
gmx_bool bVerbose = FALSE;
gmx_bool bCompact = TRUE;
gmx_bool bSepPot = FALSE;
gmx_bool bRerunVSite = FALSE;
gmx_bool bIonize = FALSE;
gmx_bool bConfout = TRUE;
gmx_bool bReproducible = FALSE;
int npme = -1;
int nmultisim = 0;
int nstglobalcomm = -1;
int repl_ex_nst = 0;
int repl_ex_seed = -1;
int repl_ex_nex = 0;
int nstepout = 100;
int resetstep = -1;
gmx_large_int_t nsteps = -2; /* the value -2 means that the mdp option will be used */
rvec realddxyz = {0, 0, 0};
const char *ddno_opt[ddnoNR+1] =
{ NULL, "interleave", "pp_pme", "cartesian", NULL };
const char *dddlb_opt[] =
{ NULL, "auto", "no", "yes", NULL };
const char *thread_aff_opt[threadaffNR+1] =
{ NULL, "auto", "on", "off", NULL };
const char *nbpu_opt[] =
{ NULL, "auto", "cpu", "gpu", "gpu_cpu", NULL };
real rdd = 0.0, rconstr = 0.0, dlb_scale = 0.8, pforce = -1;
char *ddcsx = NULL, *ddcsy = NULL, *ddcsz = NULL;
real cpt_period = 15.0, max_hours = -1;
gmx_bool bAppendFiles = TRUE;
gmx_bool bKeepAndNumCPT = FALSE;
gmx_bool bResetCountersHalfWay = FALSE;
output_env_t oenv = NULL;
const char *deviceOptions = "";
/* Non transparent initialization of a complex gmx_hw_opt_t struct.
* But unfortunately we are not allowed to call a function here,
* since declarations follow below.
*/
gmx_hw_opt_t hw_opt = { 0, 0, 0, 0, threadaffSEL, 0, 0,
{ NULL, FALSE, 0, NULL } };
t_pargs pa[] = {
{ "-pd", FALSE, etBOOL, {&bPartDec},
"Use particle decompostion" },
{ "-dd", FALSE, etRVEC, {&realddxyz},
"Domain decomposition grid, 0 is optimize" },
{ "-ddorder", FALSE, etENUM, {ddno_opt},
"DD node order" },
{ "-npme", FALSE, etINT, {&npme},
"Number of separate nodes to be used for PME, -1 is guess" },
{ "-nt", FALSE, etINT, {&hw_opt.nthreads_tot},
"Total number of threads to start (0 is guess)" },
{ "-ntmpi", FALSE, etINT, {&hw_opt.nthreads_tmpi},
"Number of thread-MPI threads to start (0 is guess)" },
{ "-ntomp", FALSE, etINT, {&hw_opt.nthreads_omp},
"Number of OpenMP threads per MPI process/thread to start (0 is guess)" },
{ "-ntomp_pme", FALSE, etINT, {&hw_opt.nthreads_omp_pme},
"Number of OpenMP threads per MPI process/thread to start (0 is -ntomp)" },
{ "-pin", FALSE, etENUM, {thread_aff_opt},
"Fix threads (or processes) to specific cores" },
{ "-pinoffset", FALSE, etINT, {&hw_opt.core_pinning_offset},
"The starting logical core number for pinning to cores; used to avoid pinning threads from different mdrun instances to the same core" },
{ "-pinstride", FALSE, etINT, {&hw_opt.core_pinning_stride},
"Pinning distance in logical cores for threads, use 0 to minimize the number of threads per physical core" },
{ "-gpu_id", FALSE, etSTR, {&hw_opt.gpu_opt.gpu_id},
"List of GPU device id-s to use, specifies the per-node PP rank to GPU mapping" },
{ "-ddcheck", FALSE, etBOOL, {&bDDBondCheck},
"Check for all bonded interactions with DD" },
{ "-ddbondcomm", FALSE, etBOOL, {&bDDBondComm},
"HIDDENUse special bonded atom communication when [TT]-rdd[tt] > cut-off" },
{ "-rdd", FALSE, etREAL, {&rdd},
"The maximum distance for bonded interactions with DD (nm), 0 is determine from initial coordinates" },
{ "-rcon", FALSE, etREAL, {&rconstr},
"Maximum distance for P-LINCS (nm), 0 is estimate" },
{ "-dlb", FALSE, etENUM, {dddlb_opt},
"Dynamic load balancing (with DD)" },
{ "-dds", FALSE, etREAL, {&dlb_scale},
"Minimum allowed dlb scaling of the DD cell size" },
{ "-ddcsx", FALSE, etSTR, {&ddcsx},
"HIDDENThe DD cell sizes in x" },
{ "-ddcsy", FALSE, etSTR, {&ddcsy},
"HIDDENThe DD cell sizes in y" },
{ "-ddcsz", FALSE, etSTR, {&ddcsz},
"HIDDENThe DD cell sizes in z" },
{ "-gcom", FALSE, etINT, {&nstglobalcomm},
"Global communication frequency" },
{ "-nb", FALSE, etENUM, {&nbpu_opt},
"Calculate non-bonded interactions on" },
{ "-tunepme", FALSE, etBOOL, {&bTunePME},
"Optimize PME load between PP/PME nodes or GPU/CPU" },
{ "-testverlet", FALSE, etBOOL, {&bTestVerlet},
"Test the Verlet non-bonded scheme" },
{ "-v", FALSE, etBOOL, {&bVerbose},
"Be loud and noisy" },
{ "-compact", FALSE, etBOOL, {&bCompact},
"Write a compact log file" },
{ "-seppot", FALSE, etBOOL, {&bSepPot},
"Write separate V and dVdl terms for each interaction type and node to the log file(s)" },
{ "-pforce", FALSE, etREAL, {&pforce},
"Print all forces larger than this (kJ/mol nm)" },
{ "-reprod", FALSE, etBOOL, {&bReproducible},
"Try to avoid optimizations that affect binary reproducibility" },
{ "-cpt", FALSE, etREAL, {&cpt_period},
"Checkpoint interval (minutes)" },
{ "-cpnum", FALSE, etBOOL, {&bKeepAndNumCPT},
"Keep and number checkpoint files" },
{ "-append", FALSE, etBOOL, {&bAppendFiles},
"Append to previous output files when continuing from checkpoint instead of adding the simulation part number to all file names" },
{ "-nsteps", FALSE, etGMX_LARGE_INT, {&nsteps},
"Run this number of steps, overrides .mdp file option" },
{ "-maxh", FALSE, etREAL, {&max_hours},
"Terminate after 0.99 times this time (hours)" },
{ "-multi", FALSE, etINT, {&nmultisim},
"Do multiple simulations in parallel" },
{ "-replex", FALSE, etINT, {&repl_ex_nst},
"Attempt replica exchange periodically with this period (steps)" },
{ "-nex", FALSE, etINT, {&repl_ex_nex},
"Number of random exchanges to carry out each exchange interval (N^3 is one suggestion). -nex zero or not specified gives neighbor replica exchange." },
{ "-reseed", FALSE, etINT, {&repl_ex_seed},
"Seed for replica exchange, -1 is generate a seed" },
{ "-rerunvsite", FALSE, etBOOL, {&bRerunVSite},
"HIDDENRecalculate virtual site coordinates with [TT]-rerun[tt]" },
{ "-ionize", FALSE, etBOOL, {&bIonize},
"Do a simulation including the effect of an X-Ray bombardment on your system" },
{ "-confout", FALSE, etBOOL, {&bConfout},
"HIDDENWrite the last configuration with [TT]-c[tt] and force checkpointing at the last step" },
{ "-stepout", FALSE, etINT, {&nstepout},
"HIDDENFrequency of writing the remaining runtime" },
{ "-resetstep", FALSE, etINT, {&resetstep},
"HIDDENReset cycle counters after these many time steps" },
{ "-resethway", FALSE, etBOOL, {&bResetCountersHalfWay},
"HIDDENReset the cycle counters after half the number of steps or halfway [TT]-maxh[tt]" }
};
gmx_edsam_t ed;
unsigned long Flags, PCA_Flags;
ivec ddxyz;
int dd_node_order;
gmx_bool bAddPart;
FILE *fplog, *fpmulti;
int sim_part, sim_part_fn;
const char *part_suffix = ".part";
char suffix[STRLEN];
int rc;
char **multidir = NULL;
cr = init_par(&argc, &argv);
if (MASTER(cr))
{
CopyRight(stderr, argv[0]);
}
PCA_Flags = (PCA_CAN_SET_DEFFNM | (MASTER(cr) ? 0 : PCA_QUIET));
/* Comment this in to do fexist calls only on master
* works not with rerun or tables at the moment
* also comment out the version of init_forcerec in md.c
* with NULL instead of opt2fn
*/
/*
if (!MASTER(cr))
{
PCA_Flags |= PCA_NOT_READ_NODE;
}
*/
parse_common_args(&argc, argv, PCA_Flags, NFILE, fnm, asize(pa), pa,
asize(desc), desc, 0, NULL, &oenv);
/* we set these early because they might be used in init_multisystem()
Note that there is the potential for npme>nnodes until the number of
threads is set later on, if there's thread parallelization. That shouldn't
lead to problems. */
dd_node_order = nenum(ddno_opt);
cr->npmenodes = npme;
hw_opt.thread_affinity = nenum(thread_aff_opt);
/* now check the -multi and -multidir option */
if (opt2bSet("-multidir", NFILE, fnm))
{
int i;
if (nmultisim > 0)
{
gmx_fatal(FARGS, "mdrun -multi and -multidir options are mutually exclusive.");
}
nmultisim = opt2fns(&multidir, "-multidir", NFILE, fnm);
}
if (repl_ex_nst != 0 && nmultisim < 2)
{
gmx_fatal(FARGS, "Need at least two replicas for replica exchange (option -multi)");
}
if (repl_ex_nex < 0)
{
gmx_fatal(FARGS, "Replica exchange number of exchanges needs to be positive");
}
if (nmultisim > 1)
{
#ifndef GMX_THREAD_MPI
gmx_bool bParFn = (multidir == NULL);
init_multisystem(cr, nmultisim, multidir, NFILE, fnm, bParFn);
#else
gmx_fatal(FARGS, "mdrun -multi is not supported with the thread library.Please compile GROMACS with MPI support");
#endif
}
bAddPart = !bAppendFiles;
/* Check if there is ANY checkpoint file available */
sim_part = 1;
sim_part_fn = sim_part;
if (opt2bSet("-cpi", NFILE, fnm))
{
if (bSepPot && bAppendFiles)
{
gmx_fatal(FARGS, "Output file appending is not supported with -seppot");
}
bAppendFiles =
read_checkpoint_simulation_part(opt2fn_master("-cpi", NFILE,
fnm, cr),
&sim_part_fn, NULL, cr,
bAppendFiles, NFILE, fnm,
part_suffix, &bAddPart);
if (sim_part_fn == 0 && MULTIMASTER(cr))
{
fprintf(stdout, "No previous checkpoint file present, assuming this is a new run.\n");
}
else
{
sim_part = sim_part_fn + 1;
}
if (MULTISIM(cr) && MASTER(cr))
{
if (MULTIMASTER(cr))
{
/* Log file is not yet available, so if there's a
* problem we can only write to stderr. */
fpmulti = stderr;
}
else
{
fpmulti = NULL;
}
check_multi_int(fpmulti, cr->ms, sim_part, "simulation part", TRUE);
}
}
else
{
bAppendFiles = FALSE;
}
if (!bAppendFiles)
{
sim_part_fn = sim_part;
}
if (bAddPart)
{
/* Rename all output files (except checkpoint files) */
/* create new part name first (zero-filled) */
sprintf(suffix, "%s%04d", part_suffix, sim_part_fn);
add_suffix_to_output_names(fnm, NFILE, suffix);
if (MULTIMASTER(cr))
{
fprintf(stdout, "Checkpoint file is from part %d, new output files will be suffixed '%s'.\n", sim_part-1, suffix);
}
}
Flags = opt2bSet("-rerun", NFILE, fnm) ? MD_RERUN : 0;
Flags = Flags | (bSepPot ? MD_SEPPOT : 0);
Flags = Flags | (bIonize ? MD_IONIZE : 0);
Flags = Flags | (bPartDec ? MD_PARTDEC : 0);
Flags = Flags | (bDDBondCheck ? MD_DDBONDCHECK : 0);
Flags = Flags | (bDDBondComm ? MD_DDBONDCOMM : 0);
Flags = Flags | (bTunePME ? MD_TUNEPME : 0);
Flags = Flags | (bTestVerlet ? MD_TESTVERLET : 0);
Flags = Flags | (bConfout ? MD_CONFOUT : 0);
Flags = Flags | (bRerunVSite ? MD_RERUN_VSITE : 0);
Flags = Flags | (bReproducible ? MD_REPRODUCIBLE : 0);
Flags = Flags | (bAppendFiles ? MD_APPENDFILES : 0);
Flags = Flags | (opt2parg_bSet("-append", asize(pa), pa) ? MD_APPENDFILESSET : 0);
Flags = Flags | (bKeepAndNumCPT ? MD_KEEPANDNUMCPT : 0);
Flags = Flags | (sim_part > 1 ? MD_STARTFROMCPT : 0);
Flags = Flags | (bResetCountersHalfWay ? MD_RESETCOUNTERSHALFWAY : 0);
/* We postpone opening the log file if we are appending, so we can
first truncate the old log file and append to the correct position
there instead. */
if ((MASTER(cr) || bSepPot) && !bAppendFiles)
{
gmx_log_open(ftp2fn(efLOG, NFILE, fnm), cr,
!bSepPot, Flags & MD_APPENDFILES, &fplog);
CopyRight(fplog, argv[0]);
please_cite(fplog, "Hess2008b");
please_cite(fplog, "Spoel2005a");
please_cite(fplog, "Lindahl2001a");
please_cite(fplog, "Berendsen95a");
}
else if (!MASTER(cr) && bSepPot)
{
gmx_log_open(ftp2fn(efLOG, NFILE, fnm), cr, !bSepPot, Flags, &fplog);
}
else
{
fplog = NULL;
}
ddxyz[XX] = (int)(realddxyz[XX] + 0.5);
ddxyz[YY] = (int)(realddxyz[YY] + 0.5);
ddxyz[ZZ] = (int)(realddxyz[ZZ] + 0.5);
/* PLUMED */
plumedswitch=0;
if (opt2bSet("-plumed",NFILE,fnm)) plumedswitch=1;
if(plumedswitch){ plumedcmd=plumed_cmd;
int plumed_is_there=0;
int real_precision=sizeof(real);
real energyUnits=1.0;
real lengthUnits=1.0;
real timeUnits=1.0;
if(!plumed_installed()){
gmx_fatal(FARGS,"Plumed is not available. Check your PLUMED_KERNEL variable.");
}
plumedmain=plumed_create();
plumed_cmd(plumedmain,"setRealPrecision",&real_precision);
// this is not necessary for gromacs units:
plumed_cmd(plumedmain,"setMDEnergyUnits",&energyUnits);
plumed_cmd(plumedmain,"setMDLengthUnits",&lengthUnits);
plumed_cmd(plumedmain,"setMDTimeUnits",&timeUnits);
//
plumed_cmd(plumedmain,"setPlumedDat",ftp2fn(efDAT,NFILE,fnm));
plumedswitch=1;
}
/* END PLUMED */
rc = mdrunner(&hw_opt, fplog, cr, NFILE, fnm, oenv, bVerbose, bCompact,
nstglobalcomm, ddxyz, dd_node_order, rdd, rconstr,
dddlb_opt[0], dlb_scale, ddcsx, ddcsy, ddcsz,
nbpu_opt[0],
nsteps, nstepout, resetstep,
nmultisim, repl_ex_nst, repl_ex_nex, repl_ex_seed,
pforce, cpt_period, max_hours, deviceOptions, Flags);
/* PLUMED */
if(plumedswitch){
plumed_finalize(plumedmain);
}
/* END PLUMED */
gmx_finalize_par();
if (MULTIMASTER(cr))
{
thanx(stderr);
}
/* Log file has to be closed in mdrunner if we are appending to it
(fplog not set here) */
if (MASTER(cr) && !bAppendFiles)
{
gmx_log_close(fplog);
}
return rc;
}
Extern void SUFFIX(cubafork)(Spin **pspin)
{
char out[128];
int cores, core, *pfd;
Spin *spin;
VerboseInit();
EnvInit(cubaworkers_.paccel, "CUBAACCELMAX", 1000);
EnvInit(cubaworkers_.pcores, "CUBACORESMAX", 10000);
EnvInit(cubaworkers_.naccel, "CUBAACCEL", 0);
EnvInit(cubaworkers_.ncores, "CUBACORES", -sysconf(_SC_NPROCESSORS_ONLN));
#ifdef HAVE_GETLOADAVG
if( cubaworkers_.ncores < 0 ) {
static int load = uninitialized;
if( load == uninitialized ) {
double loadavg;
getloadavg(&loadavg, 1);
load = floor(loadavg);
}
cubaworkers_.ncores = IMax(-cubaworkers_.ncores - load, 0);
}
#else
cubaworkers_.ncores = abs(cubaworkers_.ncores);
#endif
cores = cubaworkers_.naccel + cubaworkers_.ncores;
if( cores < MINCORES ) {
*pspin = NULL;
return;
}
if( cubaverb_ ) {
sprintf(out, "using %d cores %d accelerators via "
#ifdef HAVE_SHMGET
"shared memory",
#else
"pipes",
#endif
cubaworkers_.ncores, cubaworkers_.naccel);
Print(out);
}
fflush(NULL); /* make sure all buffers are flushed,
or else buffered content will be written
out multiply, at each child's exit(0) */
MemAlloc(spin, sizeof *spin + cores*sizeof(int));
spin->spec = cubaworkers_;
pfd = spin->fd;
for( core = -spin->spec.naccel; core < spin->spec.ncores; ++core ) {
int fd[2];
pid_t pid;
assert(
socketpair(AF_LOCAL, SOCK_STREAM, 0, fd) != -1 &&
(pid = fork()) != -1 );
if( pid == 0 ) {
close(fd[0]);
free(spin);
Child(fd[1], core);
exit(0);
}
MASTER("forked pid %d pipe %d(master) -> %d(worker)",
pid, fd[0], fd[1]);
close(fd[1]);
*pfd++ = fd[0];
}
*pspin = spin;
}
/*! \brief Do test particle insertion.
\copydoc integrator_t (FILE *fplog, t_commrec *cr, const gmx::MDLogger &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv, gmx_bool bVerbose,
int nstglobalcomm,
gmx_vsite_t *vsite, gmx_constr_t constr,
int stepout,
t_inputrec *inputrec,
gmx_mtop_t *top_global, t_fcdata *fcd,
t_state *state_global,
t_mdatoms *mdatoms,
t_nrnb *nrnb, gmx_wallcycle_t wcycle,
gmx_edsam_t ed,
t_forcerec *fr,
int repl_ex_nst, int repl_ex_nex, int repl_ex_seed,
real cpt_period, real max_hours,
int imdport,
unsigned long Flags,
gmx_walltime_accounting_t walltime_accounting)
*/
double do_tpi(FILE *fplog, t_commrec *cr, const gmx::MDLogger gmx_unused &mdlog,
int nfile, const t_filenm fnm[],
const gmx_output_env_t *oenv, gmx_bool bVerbose,
int gmx_unused nstglobalcomm,
gmx_vsite_t gmx_unused *vsite, gmx_constr_t gmx_unused constr,
int gmx_unused stepout,
t_inputrec *inputrec,
gmx_mtop_t *top_global, t_fcdata *fcd,
t_state *state_global,
t_mdatoms *mdatoms,
t_nrnb *nrnb, gmx_wallcycle_t wcycle,
gmx_edsam_t gmx_unused ed,
t_forcerec *fr,
int gmx_unused repl_ex_nst, int gmx_unused repl_ex_nex, int gmx_unused repl_ex_seed,
real gmx_unused cpt_period, real gmx_unused max_hours,
int gmx_unused imdport,
unsigned long gmx_unused Flags,
gmx_walltime_accounting_t walltime_accounting)
{
gmx_localtop_t *top;
gmx_groups_t *groups;
gmx_enerdata_t *enerd;
rvec *f;
real lambda, t, temp, beta, drmax, epot;
double embU, sum_embU, *sum_UgembU, V, V_all, VembU_all;
t_trxstatus *status;
t_trxframe rerun_fr;
gmx_bool bDispCorr, bCharge, bRFExcl, bNotLastFrame, bStateChanged, bNS;
tensor force_vir, shake_vir, vir, pres;
int cg_tp, a_tp0, a_tp1, ngid, gid_tp, nener, e;
rvec *x_mol;
rvec mu_tot, x_init, dx, x_tp;
int nnodes, frame;
gmx_int64_t frame_step_prev, frame_step;
gmx_int64_t nsteps, stepblocksize = 0, step;
gmx_int64_t seed;
int i;
FILE *fp_tpi = NULL;
char *ptr, *dump_pdb, **leg, str[STRLEN], str2[STRLEN];
double dbl, dump_ener;
gmx_bool bCavity;
int nat_cavity = 0, d;
real *mass_cavity = NULL, mass_tot;
int nbin;
double invbinw, *bin, refvolshift, logV, bUlogV;
real prescorr, enercorr, dvdlcorr;
gmx_bool bEnergyOutOfBounds;
const char *tpid_leg[2] = {"direct", "reweighted"};
/* Since there is no upper limit to the insertion energies,
* we need to set an upper limit for the distribution output.
*/
real bU_bin_limit = 50;
real bU_logV_bin_limit = bU_bin_limit + 10;
if (inputrec->cutoff_scheme == ecutsVERLET)
{
gmx_fatal(FARGS, "TPI does not work (yet) with the Verlet cut-off scheme");
}
nnodes = cr->nnodes;
top = gmx_mtop_generate_local_top(top_global, inputrec->efep != efepNO);
groups = &top_global->groups;
bCavity = (inputrec->eI == eiTPIC);
if (bCavity)
{
ptr = getenv("GMX_TPIC_MASSES");
if (ptr == NULL)
{
nat_cavity = 1;
}
else
{
/* Read (multiple) masses from env var GMX_TPIC_MASSES,
* The center of mass of the last atoms is then used for TPIC.
*/
nat_cavity = 0;
while (sscanf(ptr, "%20lf%n", &dbl, &i) > 0)
{
srenew(mass_cavity, nat_cavity+1);
mass_cavity[nat_cavity] = dbl;
fprintf(fplog, "mass[%d] = %f\n",
nat_cavity+1, mass_cavity[nat_cavity]);
nat_cavity++;
ptr += i;
}
if (nat_cavity == 0)
{
gmx_fatal(FARGS, "Found %d masses in GMX_TPIC_MASSES", nat_cavity);
}
}
}
/*
init_em(fplog,TPI,inputrec,&lambda,nrnb,mu_tot,
state_global->box,fr,mdatoms,top,cr,nfile,fnm,NULL,NULL);*/
/* We never need full pbc for TPI */
fr->ePBC = epbcXYZ;
/* Determine the temperature for the Boltzmann weighting */
temp = inputrec->opts.ref_t[0];
if (fplog)
{
for (i = 1; (i < inputrec->opts.ngtc); i++)
{
if (inputrec->opts.ref_t[i] != temp)
{
fprintf(fplog, "\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n");
fprintf(stderr, "\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n");
}
}
fprintf(fplog,
"\n The temperature for test particle insertion is %.3f K\n\n",
temp);
}
beta = 1.0/(BOLTZ*temp);
/* Number of insertions per frame */
nsteps = inputrec->nsteps;
/* Use the same neighborlist with more insertions points
* in a sphere of radius drmax around the initial point
*/
/* This should be a proper mdp parameter */
drmax = inputrec->rtpi;
/* An environment variable can be set to dump all configurations
* to pdb with an insertion energy <= this value.
*/
dump_pdb = getenv("GMX_TPI_DUMP");
dump_ener = 0;
if (dump_pdb)
{
sscanf(dump_pdb, "%20lf", &dump_ener);
}
atoms2md(top_global, inputrec, 0, NULL, top_global->natoms, mdatoms);
update_mdatoms(mdatoms, inputrec->fepvals->init_lambda);
snew(enerd, 1);
init_enerdata(groups->grps[egcENER].nr, inputrec->fepvals->n_lambda, enerd);
snew(f, top_global->natoms);
/* Print to log file */
walltime_accounting_start(walltime_accounting);
wallcycle_start(wcycle, ewcRUN);
print_start(fplog, cr, walltime_accounting, "Test Particle Insertion");
/* The last charge group is the group to be inserted */
cg_tp = top->cgs.nr - 1;
a_tp0 = top->cgs.index[cg_tp];
a_tp1 = top->cgs.index[cg_tp+1];
if (debug)
{
fprintf(debug, "TPI cg %d, atoms %d-%d\n", cg_tp, a_tp0, a_tp1);
}
GMX_RELEASE_ASSERT(inputrec->rcoulomb <= inputrec->rlist && inputrec->rvdw <= inputrec->rlist, "Twin-range interactions are not supported with TPI");
snew(x_mol, a_tp1-a_tp0);
bDispCorr = (inputrec->eDispCorr != edispcNO);
bCharge = FALSE;
for (i = a_tp0; i < a_tp1; i++)
{
/* Copy the coordinates of the molecule to be insterted */
copy_rvec(state_global->x[i], x_mol[i-a_tp0]);
/* Check if we need to print electrostatic energies */
bCharge |= (mdatoms->chargeA[i] != 0 ||
(mdatoms->chargeB && mdatoms->chargeB[i] != 0));
}
bRFExcl = (bCharge && EEL_RF(fr->eeltype));
calc_cgcm(fplog, cg_tp, cg_tp+1, &(top->cgs), state_global->x, fr->cg_cm);
if (bCavity)
{
if (norm(fr->cg_cm[cg_tp]) > 0.5*inputrec->rlist && fplog)
{
fprintf(fplog, "WARNING: Your TPI molecule is not centered at 0,0,0\n");
fprintf(stderr, "WARNING: Your TPI molecule is not centered at 0,0,0\n");
}
}
else
{
/* Center the molecule to be inserted at zero */
for (i = 0; i < a_tp1-a_tp0; i++)
{
rvec_dec(x_mol[i], fr->cg_cm[cg_tp]);
}
}
if (fplog)
{
fprintf(fplog, "\nWill insert %d atoms %s partial charges\n",
a_tp1-a_tp0, bCharge ? "with" : "without");
fprintf(fplog, "\nWill insert %d times in each frame of %s\n",
(int)nsteps, opt2fn("-rerun", nfile, fnm));
}
if (!bCavity)
{
if (inputrec->nstlist > 1)
{
if (drmax == 0 && a_tp1-a_tp0 == 1)
{
gmx_fatal(FARGS, "Re-using the neighborlist %d times for insertions of a single atom in a sphere of radius %f does not make sense", inputrec->nstlist, drmax);
}
if (fplog)
{
fprintf(fplog, "Will use the same neighborlist for %d insertions in a sphere of radius %f\n", inputrec->nstlist, drmax);
}
}
}
else
{
if (fplog)
{
fprintf(fplog, "Will insert randomly in a sphere of radius %f around the center of the cavity\n", drmax);
}
}
ngid = groups->grps[egcENER].nr;
gid_tp = GET_CGINFO_GID(fr->cginfo[cg_tp]);
nener = 1 + ngid;
if (bDispCorr)
{
nener += 1;
}
if (bCharge)
{
nener += ngid;
if (bRFExcl)
{
nener += 1;
}
if (EEL_FULL(fr->eeltype))
{
nener += 1;
}
}
snew(sum_UgembU, nener);
/* Copy the random seed set by the user */
seed = inputrec->ld_seed;
gmx::ThreeFry2x64<16> rng(seed, gmx::RandomDomain::TestParticleInsertion); // 16 bits internal counter => 2^16 * 2 = 131072 values per stream
gmx::UniformRealDistribution<real> dist;
if (MASTER(cr))
{
fp_tpi = xvgropen(opt2fn("-tpi", nfile, fnm),
"TPI energies", "Time (ps)",
"(kJ mol\\S-1\\N) / (nm\\S3\\N)", oenv);
xvgr_subtitle(fp_tpi, "f. are averages over one frame", oenv);
snew(leg, 4+nener);
e = 0;
sprintf(str, "-kT log(<Ve\\S-\\betaU\\N>/<V>)");
leg[e++] = gmx_strdup(str);
sprintf(str, "f. -kT log<e\\S-\\betaU\\N>");
leg[e++] = gmx_strdup(str);
sprintf(str, "f. <e\\S-\\betaU\\N>");
leg[e++] = gmx_strdup(str);
sprintf(str, "f. V");
leg[e++] = gmx_strdup(str);
sprintf(str, "f. <Ue\\S-\\betaU\\N>");
leg[e++] = gmx_strdup(str);
for (i = 0; i < ngid; i++)
{
sprintf(str, "f. <U\\sVdW %s\\Ne\\S-\\betaU\\N>",
*(groups->grpname[groups->grps[egcENER].nm_ind[i]]));
leg[e++] = gmx_strdup(str);
}
if (bDispCorr)
{
sprintf(str, "f. <U\\sdisp c\\Ne\\S-\\betaU\\N>");
leg[e++] = gmx_strdup(str);
}
if (bCharge)
{
for (i = 0; i < ngid; i++)
{
sprintf(str, "f. <U\\sCoul %s\\Ne\\S-\\betaU\\N>",
*(groups->grpname[groups->grps[egcENER].nm_ind[i]]));
leg[e++] = gmx_strdup(str);
}
if (bRFExcl)
{
sprintf(str, "f. <U\\sRF excl\\Ne\\S-\\betaU\\N>");
leg[e++] = gmx_strdup(str);
}
if (EEL_FULL(fr->eeltype))
{
sprintf(str, "f. <U\\sCoul recip\\Ne\\S-\\betaU\\N>");
leg[e++] = gmx_strdup(str);
}
}
xvgr_legend(fp_tpi, 4+nener, (const char**)leg, oenv);
for (i = 0; i < 4+nener; i++)
{
sfree(leg[i]);
}
sfree(leg);
}
clear_rvec(x_init);
V_all = 0;
VembU_all = 0;
invbinw = 10;
nbin = 10;
snew(bin, nbin);
/* Avoid frame step numbers <= -1 */
frame_step_prev = -1;
bNotLastFrame = read_first_frame(oenv, &status, opt2fn("-rerun", nfile, fnm),
&rerun_fr, TRX_NEED_X);
frame = 0;
if (rerun_fr.natoms - (bCavity ? nat_cavity : 0) !=
mdatoms->nr - (a_tp1 - a_tp0))
{
gmx_fatal(FARGS, "Number of atoms in trajectory (%d)%s "
"is not equal the number in the run input file (%d) "
"minus the number of atoms to insert (%d)\n",
rerun_fr.natoms, bCavity ? " minus one" : "",
mdatoms->nr, a_tp1-a_tp0);
}
refvolshift = log(det(rerun_fr.box));
switch (inputrec->eI)
{
case eiTPI:
stepblocksize = inputrec->nstlist;
break;
case eiTPIC:
stepblocksize = 1;
break;
default:
gmx_fatal(FARGS, "Unknown integrator %s", ei_names[inputrec->eI]);
}
while (bNotLastFrame)
{
frame_step = rerun_fr.step;
if (frame_step <= frame_step_prev)
{
/* We don't have step number in the trajectory file,
* or we have constant or decreasing step numbers.
* Ensure we have increasing step numbers, since we use
* the step numbers as a counter for random numbers.
*/
frame_step = frame_step_prev + 1;
}
frame_step_prev = frame_step;
lambda = rerun_fr.lambda;
t = rerun_fr.time;
sum_embU = 0;
for (e = 0; e < nener; e++)
{
sum_UgembU[e] = 0;
}
/* Copy the coordinates from the input trajectory */
for (i = 0; i < rerun_fr.natoms; i++)
{
copy_rvec(rerun_fr.x[i], state_global->x[i]);
}
copy_mat(rerun_fr.box, state_global->box);
V = det(state_global->box);
logV = log(V);
bStateChanged = TRUE;
bNS = TRUE;
step = cr->nodeid*stepblocksize;
while (step < nsteps)
{
/* Restart random engine using the frame and insertion step
* as counters.
* Note that we need to draw several random values per iteration,
* but by using the internal subcounter functionality of ThreeFry2x64
* we can draw 131072 unique 64-bit values before exhausting
* the stream. This is a huge margin, and if something still goes
* wrong you will get an exception when the stream is exhausted.
*/
rng.restart(frame_step, step);
dist.reset(); // erase any memory in the distribution
if (!bCavity)
{
/* Random insertion in the whole volume */
bNS = (step % inputrec->nstlist == 0);
if (bNS)
{
/* Generate a random position in the box */
for (d = 0; d < DIM; d++)
{
x_init[d] = dist(rng)*state_global->box[d][d];
}
}
if (inputrec->nstlist == 1)
{
copy_rvec(x_init, x_tp);
}
else
{
/* Generate coordinates within |dx|=drmax of x_init */
do
{
for (d = 0; d < DIM; d++)
{
dx[d] = (2*dist(rng) - 1)*drmax;
}
}
while (norm2(dx) > drmax*drmax);
rvec_add(x_init, dx, x_tp);
}
}
else
{
/* Random insertion around a cavity location
* given by the last coordinate of the trajectory.
*/
if (step == 0)
{
if (nat_cavity == 1)
{
/* Copy the location of the cavity */
copy_rvec(rerun_fr.x[rerun_fr.natoms-1], x_init);
}
else
{
/* Determine the center of mass of the last molecule */
clear_rvec(x_init);
mass_tot = 0;
for (i = 0; i < nat_cavity; i++)
{
for (d = 0; d < DIM; d++)
{
x_init[d] +=
mass_cavity[i]*rerun_fr.x[rerun_fr.natoms-nat_cavity+i][d];
}
mass_tot += mass_cavity[i];
}
for (d = 0; d < DIM; d++)
{
x_init[d] /= mass_tot;
}
}
}
/* Generate coordinates within |dx|=drmax of x_init */
do
{
for (d = 0; d < DIM; d++)
{
dx[d] = (2*dist(rng) - 1)*drmax;
}
}
while (norm2(dx) > drmax*drmax);
rvec_add(x_init, dx, x_tp);
}
if (a_tp1 - a_tp0 == 1)
{
/* Insert a single atom, just copy the insertion location */
copy_rvec(x_tp, state_global->x[a_tp0]);
}
else
{
/* Copy the coordinates from the top file */
for (i = a_tp0; i < a_tp1; i++)
{
copy_rvec(x_mol[i-a_tp0], state_global->x[i]);
}
/* Rotate the molecule randomly */
rotate_conf(a_tp1-a_tp0, state_global->x+a_tp0, NULL,
2*M_PI*dist(rng),
2*M_PI*dist(rng),
2*M_PI*dist(rng));
/* Shift to the insertion location */
for (i = a_tp0; i < a_tp1; i++)
{
rvec_inc(state_global->x[i], x_tp);
}
}
/* Clear some matrix variables */
clear_mat(force_vir);
clear_mat(shake_vir);
clear_mat(vir);
clear_mat(pres);
/* Set the charge group center of mass of the test particle */
copy_rvec(x_init, fr->cg_cm[top->cgs.nr-1]);
/* Calc energy (no forces) on new positions.
* Since we only need the intermolecular energy
* and the RF exclusion terms of the inserted molecule occur
* within a single charge group we can pass NULL for the graph.
* This also avoids shifts that would move charge groups
* out of the box. */
/* Make do_force do a single node force calculation */
cr->nnodes = 1;
do_force(fplog, cr, inputrec,
step, nrnb, wcycle, top, &top_global->groups,
state_global->box, state_global->x, &state_global->hist,
f, force_vir, mdatoms, enerd, fcd,
state_global->lambda,
NULL, fr, NULL, mu_tot, t, NULL, NULL, FALSE,
GMX_FORCE_NONBONDED | GMX_FORCE_ENERGY |
(bNS ? GMX_FORCE_DYNAMICBOX | GMX_FORCE_NS : 0) |
(bStateChanged ? GMX_FORCE_STATECHANGED : 0));
cr->nnodes = nnodes;
bStateChanged = FALSE;
bNS = FALSE;
/* Calculate long range corrections to pressure and energy */
calc_dispcorr(inputrec, fr, state_global->box,
lambda, pres, vir, &prescorr, &enercorr, &dvdlcorr);
/* figure out how to rearrange the next 4 lines MRS 8/4/2009 */
enerd->term[F_DISPCORR] = enercorr;
enerd->term[F_EPOT] += enercorr;
enerd->term[F_PRES] += prescorr;
enerd->term[F_DVDL_VDW] += dvdlcorr;
epot = enerd->term[F_EPOT];
bEnergyOutOfBounds = FALSE;
/* If the compiler doesn't optimize this check away
* we catch the NAN energies.
* The epot>GMX_REAL_MAX check catches inf values,
* which should nicely result in embU=0 through the exp below,
* but it does not hurt to check anyhow.
*/
/* Non-bonded Interaction usually diverge at r=0.
* With tabulated interaction functions the first few entries
* should be capped in a consistent fashion between
* repulsion, dispersion and Coulomb to avoid accidental
* negative values in the total energy.
* The table generation code in tables.c does this.
* With user tbales the user should take care of this.
*/
if (epot != epot || epot > GMX_REAL_MAX)
{
bEnergyOutOfBounds = TRUE;
}
if (bEnergyOutOfBounds)
{
if (debug)
{
fprintf(debug, "\n time %.3f, step %d: non-finite energy %f, using exp(-bU)=0\n", t, (int)step, epot);
}
embU = 0;
}
else
{
embU = exp(-beta*epot);
sum_embU += embU;
/* Determine the weighted energy contributions of each energy group */
e = 0;
sum_UgembU[e++] += epot*embU;
if (fr->bBHAM)
{
for (i = 0; i < ngid; i++)
{
sum_UgembU[e++] +=
enerd->grpp.ener[egBHAMSR][GID(i, gid_tp, ngid)]*embU;
}
}
else
{
for (i = 0; i < ngid; i++)
{
sum_UgembU[e++] +=
enerd->grpp.ener[egLJSR][GID(i, gid_tp, ngid)]*embU;
}
}
if (bDispCorr)
{
sum_UgembU[e++] += enerd->term[F_DISPCORR]*embU;
}
if (bCharge)
{
for (i = 0; i < ngid; i++)
{
sum_UgembU[e++] += enerd->grpp.ener[egCOULSR][GID(i, gid_tp, ngid)] * embU;
}
if (bRFExcl)
{
sum_UgembU[e++] += enerd->term[F_RF_EXCL]*embU;
}
if (EEL_FULL(fr->eeltype))
{
sum_UgembU[e++] += enerd->term[F_COUL_RECIP]*embU;
}
}
}
if (embU == 0 || beta*epot > bU_bin_limit)
{
bin[0]++;
}
else
{
i = (int)((bU_logV_bin_limit
- (beta*epot - logV + refvolshift))*invbinw
+ 0.5);
if (i < 0)
{
i = 0;
}
if (i >= nbin)
{
realloc_bins(&bin, &nbin, i+10);
}
bin[i]++;
}
if (debug)
{
fprintf(debug, "TPI %7d %12.5e %12.5f %12.5f %12.5f\n",
(int)step, epot, x_tp[XX], x_tp[YY], x_tp[ZZ]);
}
if (dump_pdb && epot <= dump_ener)
{
sprintf(str, "t%g_step%d.pdb", t, (int)step);
sprintf(str2, "t: %f step %d ener: %f", t, (int)step, epot);
write_sto_conf_mtop(str, str2, top_global, state_global->x, state_global->v,
inputrec->ePBC, state_global->box);
}
step++;
if ((step/stepblocksize) % cr->nnodes != cr->nodeid)
{
/* Skip all steps assigned to the other MPI ranks */
step += (cr->nnodes - 1)*stepblocksize;
}
}
if (PAR(cr))
{
/* When running in parallel sum the energies over the processes */
gmx_sumd(1, &sum_embU, cr);
gmx_sumd(nener, sum_UgembU, cr);
}
frame++;
V_all += V;
VembU_all += V*sum_embU/nsteps;
if (fp_tpi)
{
if (bVerbose || frame%10 == 0 || frame < 10)
{
fprintf(stderr, "mu %10.3e <mu> %10.3e\n",
-log(sum_embU/nsteps)/beta, -log(VembU_all/V_all)/beta);
}
fprintf(fp_tpi, "%10.3f %12.5e %12.5e %12.5e %12.5e",
t,
VembU_all == 0 ? 20/beta : -log(VembU_all/V_all)/beta,
sum_embU == 0 ? 20/beta : -log(sum_embU/nsteps)/beta,
sum_embU/nsteps, V);
for (e = 0; e < nener; e++)
{
fprintf(fp_tpi, " %12.5e", sum_UgembU[e]/nsteps);
}
fprintf(fp_tpi, "\n");
fflush(fp_tpi);
}
bNotLastFrame = read_next_frame(oenv, status, &rerun_fr);
} /* End of the loop */
walltime_accounting_end(walltime_accounting);
close_trj(status);
if (fp_tpi != NULL)
{
xvgrclose(fp_tpi);
}
if (fplog != NULL)
{
fprintf(fplog, "\n");
fprintf(fplog, " <V> = %12.5e nm^3\n", V_all/frame);
fprintf(fplog, " <mu> = %12.5e kJ/mol\n", -log(VembU_all/V_all)/beta);
}
/* Write the Boltzmann factor histogram */
if (PAR(cr))
{
/* When running in parallel sum the bins over the processes */
i = nbin;
global_max(cr, &i);
realloc_bins(&bin, &nbin, i);
gmx_sumd(nbin, bin, cr);
}
if (MASTER(cr))
{
fp_tpi = xvgropen(opt2fn("-tpid", nfile, fnm),
"TPI energy distribution",
"\\betaU - log(V/<V>)", "count", oenv);
sprintf(str, "number \\betaU > %g: %9.3e", bU_bin_limit, bin[0]);
xvgr_subtitle(fp_tpi, str, oenv);
xvgr_legend(fp_tpi, 2, (const char **)tpid_leg, oenv);
for (i = nbin-1; i > 0; i--)
{
bUlogV = -i/invbinw + bU_logV_bin_limit - refvolshift + log(V_all/frame);
fprintf(fp_tpi, "%6.2f %10d %12.5e\n",
bUlogV,
(int)(bin[i]+0.5),
bin[i]*exp(-bUlogV)*V_all/VembU_all);
}
xvgrclose(fp_tpi);
}
sfree(bin);
sfree(sum_UgembU);
walltime_accounting_set_nsteps_done(walltime_accounting, frame*inputrec->nsteps);
return 0;
}
void do_coupling(FILE *log,int nfile,t_filenm fnm[],
t_coupl_rec *tcr,real t,int step,real ener[],
t_forcerec *fr,t_inputrec *ir,bool bMaster,
t_mdatoms *md,t_idef *idef,real mu_aver,int nmols,
t_commrec *cr,matrix box,tensor virial,
tensor pres,rvec mu_tot,
rvec x[],rvec f[],bool bDoIt)
{
#define enm2Debye 48.0321
#define d2e(x) (x)/enm2Debye
#define enm2kjmol(x) (x)*0.0143952 /* = 2.0*4.0*M_PI*EPSILON0 */
static real *f6,*f12,*fa,*fb,*fc,*fq;
static bool bFirst = TRUE;
int i,j,ati,atj,atnr2,type,ftype;
real deviation[eoObsNR],prdev[eoObsNR],epot0,dist,rmsf;
real ff6,ff12,ffa,ffb,ffc,ffq,factor,dt,mu_ind;
real Epol,Eintern,Virial,muabs,xiH=-1,xiS=-1,xi6,xi12;
rvec fmol[2];
bool bTest,bPrint;
t_coupl_LJ *tclj;
t_coupl_BU *tcbu;
t_coupl_Q *tcq;
t_coupl_iparams *tip;
atnr2 = idef->atnr * idef->atnr;
if (bFirst) {
if (PAR(cr))
fprintf(log,"GCT: this is parallel\n");
else
fprintf(log,"GCT: this is not parallel\n");
fflush(log);
snew(f6, atnr2);
snew(f12,atnr2);
snew(fa, atnr2);
snew(fb, atnr2);
snew(fc, atnr2);
snew(fq, idef->atnr);
if (tcr->bVirial) {
int nrdf = 0;
real TTT = 0;
real Vol = det(box);
for(i=0; (i<ir->opts.ngtc); i++) {
nrdf += ir->opts.nrdf[i];
TTT += ir->opts.nrdf[i]*ir->opts.ref_t[i];
}
TTT /= nrdf;
/* Calculate reference virial from reference temperature and pressure */
tcr->ref_value[eoVir] = 0.5*BOLTZ*nrdf*TTT - (3.0/2.0)*
Vol*tcr->ref_value[eoPres];
fprintf(log,"GCT: TTT = %g, nrdf = %d, vir0 = %g, Vol = %g\n",
TTT,nrdf,tcr->ref_value[eoVir],Vol);
fflush(log);
}
bFirst = FALSE;
}
bPrint = MASTER(cr) && do_per_step(step,ir->nstlog);
dt = ir->delta_t;
/* Initiate coupling to the reference pressure and temperature to start
* coupling slowly.
*/
if (step == 0) {
for(i=0; (i<eoObsNR); i++)
tcr->av_value[i] = tcr->ref_value[i];
if ((tcr->ref_value[eoDipole]) != 0.0) {
mu_ind = mu_aver - d2e(tcr->ref_value[eoDipole]); /* in e nm */
Epol = mu_ind*mu_ind/(enm2kjmol(tcr->ref_value[eoPolarizability]));
tcr->av_value[eoEpot] -= Epol;
fprintf(log,"GCT: mu_aver = %g(D), mu_ind = %g(D), Epol = %g (kJ/mol)\n",
mu_aver*enm2Debye,mu_ind*enm2Debye,Epol);
}
}
/* We want to optimize the LJ params, usually to the Vaporization energy
* therefore we only count intermolecular degrees of freedom.
* Note that this is now optional. switch UseEinter to yes in your gct file
* if you want this.
*/
dist = calc_dist(log,x);
muabs = norm(mu_tot);
Eintern = Ecouple(tcr,ener)/nmols;
Virial = virial[XX][XX]+virial[YY][YY]+virial[ZZ][ZZ];
/*calc_force(md->nr,f,fmol);*/
clear_rvec(fmol[0]);
/* Use a memory of tcr->nmemory steps, so we actually couple to the
* average observable over the last tcr->nmemory steps. This may help
* in avoiding local minima in parameter space.
*/
set_act_value(tcr,eoPres, ener[F_PRES],step);
set_act_value(tcr,eoEpot, Eintern, step);
set_act_value(tcr,eoVir, Virial, step);
set_act_value(tcr,eoDist, dist, step);
set_act_value(tcr,eoMu, muabs, step);
set_act_value(tcr,eoFx, fmol[0][XX], step);
set_act_value(tcr,eoFy, fmol[0][YY], step);
set_act_value(tcr,eoFz, fmol[0][ZZ], step);
set_act_value(tcr,eoPx, pres[XX][XX],step);
set_act_value(tcr,eoPy, pres[YY][YY],step);
set_act_value(tcr,eoPz, pres[ZZ][ZZ],step);
epot0 = tcr->ref_value[eoEpot];
/* If dipole != 0.0 assume we want to use polarization corrected coupling */
if ((tcr->ref_value[eoDipole]) != 0.0) {
mu_ind = mu_aver - d2e(tcr->ref_value[eoDipole]); /* in e nm */
Epol = mu_ind*mu_ind/(enm2kjmol(tcr->ref_value[eoPolarizability]));
epot0 -= Epol;
if (debug) {
fprintf(debug,"mu_ind: %g (%g D) mu_aver: %g (%g D)\n",
mu_ind,mu_ind*enm2Debye,mu_aver,mu_aver*enm2Debye);
fprintf(debug,"Eref %g Epol %g Erunav %g Eact %g\n",
tcr->ref_value[eoEpot],Epol,tcr->av_value[eoEpot],
tcr->act_value[eoEpot]);
}
}
if (bPrint) {
pr_ff(tcr,t,idef,cr,nfile,fnm);
}
/* Calculate the deviation of average value from the target value */
for(i=0; (i<eoObsNR); i++) {
deviation[i] = calc_deviation(tcr->av_value[i],tcr->act_value[i],
tcr->ref_value[i]);
prdev[i] = tcr->ref_value[i] - tcr->act_value[i];
}
deviation[eoEpot] = calc_deviation(tcr->av_value[eoEpot],tcr->act_value[eoEpot],
epot0);
prdev[eoEpot] = epot0 - tcr->act_value[eoEpot];
if (bPrint)
pr_dev(tcr,t,prdev,cr,nfile,fnm);
/* First set all factors to 1 */
for(i=0; (i<atnr2); i++) {
f6[i] = f12[i] = fa[i] = fb[i] = fc[i] = 1.0;
}
for(i=0; (i<idef->atnr); i++)
fq[i] = 1.0;
/* Now compute the actual coupling compononents */
if (!fr->bBHAM) {
if (bDoIt) {
for(i=0; (i<tcr->nLJ); i++) {
tclj=&(tcr->tcLJ[i]);
ati=tclj->at_i;
atj=tclj->at_j;
ff6 = ff12 = 1.0;
if (tclj->eObs == eoForce) {
gmx_fatal(FARGS,"Hack code for this to work again ");
if (debug)
fprintf(debug,"Have computed derivatives: xiH = %g, xiS = %g\n",xiH,xiS);
if (ati == 1) {
/* Hydrogen */
ff12 += xiH;
}
else if (ati == 2) {
/* Shell */
ff12 += xiS;
}
else
gmx_fatal(FARGS,"No H, no Shell, edit code at %s, line %d\n",
__FILE__,__LINE__);
if (ff6 > 0)
set_factor_matrix(idef->atnr,f6, sqrt(ff6), ati,atj);
if (ff12 > 0)
set_factor_matrix(idef->atnr,f12,sqrt(ff12),ati,atj);
}
else {
if (debug)
fprintf(debug,"tcr->tcLJ[%d].xi_6 = %g, xi_12 = %g deviation = %g\n",i,
tclj->xi_6,tclj->xi_12,deviation[tclj->eObs]);
factor=deviation[tclj->eObs];
upd_f_value(log,idef->atnr,tclj->xi_6, dt,factor,f6, ati,atj);
upd_f_value(log,idef->atnr,tclj->xi_12,dt,factor,f12,ati,atj);
}
}
}
if (PAR(cr)) {
gprod(cr,atnr2,f6);
gprod(cr,atnr2,f12);
#ifdef DEBUGGCT
dump_fm(log,idef->atnr,f6,"f6");
dump_fm(log,idef->atnr,f12,"f12");
#endif
}
upd_nbfplj(log,fr->nbfp,idef->atnr,f6,f12,tcr->combrule);
/* Copy for printing */
for(i=0; (i<tcr->nLJ); i++) {
tclj=&(tcr->tcLJ[i]);
ati = tclj->at_i;
atj = tclj->at_j;
if (atj == -1)
atj = ati;
tclj->c6 = C6(fr->nbfp,fr->ntype,ati,atj);
tclj->c12 = C12(fr->nbfp,fr->ntype,ati,atj);
}
}
else {
if (bDoIt) {
for(i=0; (i<tcr->nBU); i++) {
tcbu = &(tcr->tcBU[i]);
factor = deviation[tcbu->eObs];
ati = tcbu->at_i;
atj = tcbu->at_j;
upd_f_value(log,idef->atnr,tcbu->xi_a,dt,factor,fa,ati,atj);
upd_f_value(log,idef->atnr,tcbu->xi_b,dt,factor,fb,ati,atj);
upd_f_value(log,idef->atnr,tcbu->xi_c,dt,factor,fc,ati,atj);
}
}
if (PAR(cr)) {
gprod(cr,atnr2,fa);
gprod(cr,atnr2,fb);
gprod(cr,atnr2,fc);
}
upd_nbfpbu(log,fr->nbfp,idef->atnr,fa,fb,fc);
/* Copy for printing */
for(i=0; (i<tcr->nBU); i++) {
tcbu=&(tcr->tcBU[i]);
ati = tcbu->at_i;
atj = tcbu->at_j;
if (atj == -1)
atj = ati;
tcbu->a = BHAMA(fr->nbfp,fr->ntype,ati,atj);
tcbu->b = BHAMB(fr->nbfp,fr->ntype,ati,atj);
tcbu->c = BHAMC(fr->nbfp,fr->ntype,ati,atj);
if (debug)
fprintf(debug,"buck (type=%d) = %e, %e, %e\n",
tcbu->at_i,tcbu->a,tcbu->b,tcbu->c);
}
}
if (bDoIt) {
for(i=0; (i<tcr->nQ); i++) {
tcq=&(tcr->tcQ[i]);
if (tcq->xi_Q)
ffq = 1.0 + (dt/tcq->xi_Q) * deviation[tcq->eObs];
else
ffq = 1.0;
fq[tcq->at_i] *= ffq;
}
}
if (PAR(cr))
gprod(cr,idef->atnr,fq);
for(j=0; (j<md->nr); j++) {
md->chargeA[j] *= fq[md->typeA[j]];
}
for(i=0; (i<tcr->nQ); i++) {
tcq=&(tcr->tcQ[i]);
for(j=0; (j<md->nr); j++) {
if (md->typeA[j] == tcq->at_i) {
tcq->Q = md->chargeA[j];
break;
}
}
if (j == md->nr)
gmx_fatal(FARGS,"Coupling type %d not found",tcq->at_i);
}
for(i=0; (i<tcr->nIP); i++) {
tip = &(tcr->tIP[i]);
type = tip->type;
ftype = idef->functype[type];
factor = dt*deviation[tip->eObs];
switch(ftype) {
case F_BONDS:
if (tip->xi.harmonic.krA) idef->iparams[type].harmonic.krA *= (1+factor/tip->xi.harmonic.krA);
if (tip->xi.harmonic.rA) idef->iparams[type].harmonic.rA *= (1+factor/tip->xi.harmonic.rA);
break;
default:
break;
}
tip->iprint=idef->iparams[type];
}
}
/*
* Reuse existing connection
*/
static bool connect_using_existing_connection(POOL_CONNECTION *frontend,
POOL_CONNECTION_POOL *backend,
StartupPacket *sp)
{
int i, freed = 0;
/*
* Save startup packet info
*/
for (i = 0; i < NUM_BACKENDS; i++)
{
if (VALID_BACKEND(i))
{
if (!freed)
{
pool_free_startup_packet(backend->slots[i]->sp);
freed = 1;
}
backend->slots[i]->sp = sp;
}
}
/* Reuse existing connection to backend */
if (pool_do_reauth(frontend, backend))
{
pool_close(frontend);
connection_count_down();
return false;
}
if (MAJOR(backend) == 3)
{
char command_buf[1024];
/* If we have received application_name in the start up
* packet, we send SET command to backend. Also we add or
* replace existing application_name data.
*/
if (sp->application_name)
{
snprintf(command_buf, sizeof(command_buf), "SET application_name TO '%s'", sp->application_name);
for (i=0;i<NUM_BACKENDS;i++)
{
if (VALID_BACKEND(i))
if (do_command(frontend, CONNECTION(backend, i),
command_buf, MAJOR(backend),
MASTER_CONNECTION(backend)->pid,
MASTER_CONNECTION(backend)->key, 0) != POOL_CONTINUE)
{
pool_error("connect_using_existing_connection: do_command failed. command: %s", command_buf);
return false;
}
}
pool_add_param(&MASTER(backend)->params, "application_name", sp->application_name);
}
if (send_params(frontend, backend))
{
pool_close(frontend);
connection_count_down();
return false;
}
}
/* Send ReadyForQuery to frontend */
pool_write(frontend, "Z", 1);
if (MAJOR(backend) == 3)
{
int len;
char tstate;
len = htonl(5);
pool_write(frontend, &len, sizeof(len));
tstate = TSTATE(backend, MASTER_NODE_ID);
pool_write(frontend, &tstate, 1);
}
if (pool_flush(frontend) < 0)
{
pool_close(frontend);
connection_count_down();
return false;
}
return true;
}
real dd_choose_grid(FILE *fplog,
t_commrec *cr, gmx_domdec_t *dd, t_inputrec *ir,
gmx_mtop_t *mtop, matrix box, gmx_ddbox_t *ddbox,
gmx_bool bDynLoadBal, real dlb_scale,
real cellsize_limit, real cutoff_dd,
gmx_bool bInterCGBondeds)
{
gmx_int64_t nnodes_div, ldiv;
real limit;
if (MASTER(cr))
{
nnodes_div = cr->nnodes;
if (EEL_PME(ir->coulombtype))
{
if (cr->npmenodes > 0)
{
if (cr->npmenodes >= cr->nnodes)
{
gmx_fatal(FARGS,
"Cannot have %d separate PME ranks with just %d total ranks",
cr->npmenodes, cr->nnodes);
}
/* If the user purposely selected the number of PME nodes,
* only check for large primes in the PP node count.
*/
nnodes_div -= cr->npmenodes;
}
}
else
{
cr->npmenodes = 0;
}
if (nnodes_div > 12)
{
ldiv = largest_divisor(nnodes_div);
/* Check if the largest divisor is more than nnodes^2/3 */
if (ldiv*ldiv*ldiv > nnodes_div*nnodes_div)
{
gmx_fatal(FARGS, "The number of ranks you selected (%d) contains a large prime factor %d. In most cases this will lead to bad performance. Choose a number with smaller prime factors or set the decomposition (option -dd) manually.",
nnodes_div, ldiv);
}
}
if (EEL_PME(ir->coulombtype))
{
if (cr->npmenodes < 0)
{
/* Use PME nodes when the number of nodes is more than 16 */
if (cr->nnodes <= 18)
{
cr->npmenodes = 0;
if (fplog)
{
fprintf(fplog, "Using %d separate PME ranks, as there are too few total\n ranks for efficient splitting\n", cr->npmenodes);
}
}
else
{
cr->npmenodes = guess_npme(fplog, mtop, ir, box, cr->nnodes);
if (fplog)
{
fprintf(fplog, "Using %d separate PME ranks, as guessed by mdrun\n", cr->npmenodes);
}
}
}
else
{
if (fplog)
{
fprintf(fplog, "Using %d separate PME ranks, per user request\n", cr->npmenodes);
}
}
}
limit = optimize_ncells(fplog, cr->nnodes, cr->npmenodes,
bDynLoadBal, dlb_scale,
mtop, box, ddbox, ir, dd,
cellsize_limit, cutoff_dd,
bInterCGBondeds,
dd->nc);
}
else
{
limit = 0;
}
/* Communicate the information set by the master to all nodes */
gmx_bcast(sizeof(dd->nc), dd->nc, cr);
if (EEL_PME(ir->coulombtype))
{
gmx_bcast(sizeof(ir->nkx), &ir->nkx, cr);
gmx_bcast(sizeof(ir->nky), &ir->nky, cr);
gmx_bcast(sizeof(cr->npmenodes), &cr->npmenodes, cr);
}
else
{
cr->npmenodes = 0;
}
return limit;
}
/*
* Get or return cached transaction isolation mode
*/
POOL_TRANSACTION_ISOLATION pool_get_transaction_isolation(void)
{
POOL_STATUS status;
POOL_SELECT_RESULT *res;
POOL_TRANSACTION_ISOLATION ret;
if (!session_context)
{
pool_error("pool_get_transaction_isolation: session context is not initialized");
return POOL_UNKNOWN;
}
/* It seems cached result is usable. Return it. */
if (session_context->transaction_isolation != POOL_UNKNOWN)
return session_context->transaction_isolation;
/* No cached data is available. Ask backend. */
status = do_query(MASTER(session_context->backend),
"SELECT current_setting('transaction_isolation')", &res, MAJOR(session_context->backend));
if (res->numrows <= 0)
{
pool_error("pool_get_transaction_isolation: do_query returns no rows");
free_select_result(res);
return POOL_UNKNOWN;
}
if (res->data[0] == NULL)
{
pool_error("pool_get_transaction_isolation: do_query returns no data");
free_select_result(res);
return POOL_UNKNOWN;
}
if (res->nullflags[0] == -1)
{
pool_error("pool_get_transaction_isolation: do_query returns NULL");
free_select_result(res);
return POOL_UNKNOWN;
}
if (!strcmp(res->data[0], "read uncommitted"))
ret = POOL_READ_UNCOMMITTED;
else if (!strcmp(res->data[0], "read committed"))
ret = POOL_READ_COMMITTED;
else if (!strcmp(res->data[0], "repeatable read"))
ret = POOL_REPEATABLE_READ;
else if (!strcmp(res->data[0], "serializable"))
ret = POOL_SERIALIZABLE;
else
{
pool_error("pool_get_transaction_isolation: unknown transaction isolation level:%s",
res->data[0]);
ret = POOL_UNKNOWN;
}
free_select_result(res);
if (ret != POOL_UNKNOWN)
session_context->transaction_isolation = ret;
return ret;
}
